From abcd8a8f5bef55e434b1baec0e0871b42f8e348a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Jun 2022 16:11:56 +0800 Subject: [PATCH 01/94] q --- source/client/test/smlTest.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 832564e0db..25cbb42ee9 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -913,7 +913,8 @@ TEST(testCase, smlParseTelnetLine_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -982,7 +983,8 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -1069,7 +1071,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1106,7 +1108,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1257,7 +1259,7 @@ TEST(testCase, sml_16368_Test) { "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]" }; - pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + pRes = taos_schemaless_insert(taos, (char**)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1283,5 +1285,5 @@ TEST(testCase, sml_dup_time_Test) { pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); -} -*/ +}*/ + From 88ed8d4cf5d73bb70407fcd891dde28f5b3d9db5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 13 Jul 2022 14:04:18 +0800 Subject: [PATCH 02/94] refactor code --- source/client/inc/clientInt.h | 5 +++++ source/client/src/clientImpl.c | 11 +++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 700a4d9daf..9a139f8442 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -313,6 +313,11 @@ int taos_options_imp(TSDB_OPTION option, const char* str); void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +typedef struct AsyncArg { + SRpcMsg msg; + SEpSet* pEpset; +} AsyncArg; + bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 32c11983c2..b7a8ba7202 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1269,13 +1269,8 @@ void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, } } -typedef struct SchedArg { - SRpcMsg msg; - SEpSet* pEpset; -} SchedArg; - int32_t doProcessMsgFromServer(void* param) { - SchedArg* arg = (SchedArg*)param; + AsyncArg* arg = (AsyncArg*)param; SRpcMsg* pMsg = &arg->msg; SEpSet* pEpSet = arg->pEpset; @@ -1338,7 +1333,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { memcpy((void*)tEpSet, (void*)pEpSet, sizeof(SEpSet)); } - SchedArg* arg = taosMemoryCalloc(1, sizeof(SchedArg)); + AsyncArg* arg = taosMemoryCalloc(1, sizeof(AsyncArg)); arg->msg = *pMsg; arg->pEpset = tEpSet; @@ -1478,7 +1473,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU tsem_wait(&pParam->sem); } - if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { return NULL; } else { if (setupOneRowPtr) { From 3769e0770877b6f858c97ac3edf7d50f2ee7bcc3 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 13 Jul 2022 18:13:22 +0800 Subject: [PATCH 03/94] refactor(sync): optimized replication --- source/libs/sync/src/syncMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 918496c894..92d03ae7c9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1853,8 +1853,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde syncNodeBecomeLeader(pSyncNode, tmpbuf); // Raft 3.6.2 Committing entries from previous terms - syncNodeReplicate(pSyncNode); syncNodeAppendNoop(pSyncNode); + syncNodeReplicate(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); } else { @@ -2029,8 +2029,8 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); // Raft 3.6.2 Committing entries from previous terms - syncNodeReplicate(pSyncNode); syncNodeAppendNoop(pSyncNode); + syncNodeReplicate(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); } From a0f7cc6790f9d5f39cb95bd72ca2179bd6d8ef78 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 14 Jul 2022 14:03:27 +0800 Subject: [PATCH 04/94] enh: add schedule policy --- include/libs/scheduler/scheduler.h | 8 +- source/client/src/clientEnv.c | 3 +- source/client/src/clientMain.c | 1 + source/libs/command/inc/commandInt.h | 4 + source/libs/command/src/command.c | 37 ++++++++ source/libs/nodes/src/nodesUtilFuncs.c | 6 ++ source/libs/qworker/inc/qwMsg.h | 2 +- source/libs/qworker/src/qwMsg.c | 2 +- source/libs/qworker/src/qworker.c | 2 +- source/libs/scheduler/inc/schInt.h | 85 ++++++++++--------- source/libs/scheduler/src/schTask.c | 62 ++++++++++++-- source/libs/scheduler/src/scheduler.c | 41 ++++++--- source/libs/scheduler/test/schedulerTests.cpp | 2 +- tests/script/api/batchprepare.c | 2 + 14 files changed, 189 insertions(+), 68 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 70ac7a6304..1b664d4ac5 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -26,8 +26,10 @@ extern "C" { extern tsem_t schdRspSem; typedef struct SSchedulerCfg { - uint32_t maxJobNum; - int32_t maxNodeTableNum; + uint32_t maxJobNum; + int32_t maxNodeTableNum; + SCH_POLICY schPolicy; + bool enableReSchedule; } SSchedulerCfg; typedef struct SQueryProfileSummary { @@ -96,6 +98,8 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub); void schedulerStopQueryHb(void *pTrans); +int32_t schedulerUpdatePolicy(int32_t policy); +int32_t schedulerEnableReSchedule(bool enableResche); /** * Cancel query job diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 207ac01a2c..26eaad16b9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -359,8 +359,7 @@ void taos_init_imp(void) { SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; catalogInit(&cfg); - SSchedulerCfg scfg = {.maxJobNum = 100}; - schedulerInit(&scfg); + schedulerInit(); tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 73def5b9b1..208c02e3fa 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -129,6 +129,7 @@ void taos_close(TAOS *taos) { STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { + taosMemoryFree(taos); return; } diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 6aca581f45..7012c889e9 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -77,6 +77,10 @@ extern "C" { #define EXPLAIN_MODE_FORMAT "mode=%s" #define EXPLAIN_STRING_TYPE_FORMAT "%s" +#define COMMAND_RESET_LOG "resetLog" +#define COMMAND_SCHEDULE_POLICY "schedulePolicy" +#define COMMAND_ENABLE_RESCHEDULE "enableReSchedule" + typedef struct SExplainGroup { int32_t nodeNum; int32_t physiPlanExecNum; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 034778e5bf..94d19be0a4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -479,7 +479,42 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR return execShowCreateTable(pStmt, pRsp); } +static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { + int32_t code = 0; + + if (0 == strcasecmp(cmd, COMMAND_RESET_LOG)) { + taosResetLog(); + cfgDumpCfg(tsCfg, 0, false); + } else if (0 == strcasecmp(cmd, COMMAND_SCHEDULE_POLICY)) { + code = schedulerUpdatePolicy(atoi(value)); + } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) { + code = schedulerEnableReSchedule(atoi(value)); + } else { + goto _return; + } + + *processed = true; + +_return: + + if (code) { + terrno = code; + } + + return code; +} + static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { + bool processed = false; + + if (execAlterCmd(pStmt->config, pStmt->value, &processed)) { + return terrno; + } + + if (processed) { + goto _return; + } + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { return terrno; } @@ -488,6 +523,8 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return terrno; } +_return: + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4abb8f5c2c..db582d4fa6 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -388,6 +388,11 @@ static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode* static void destroyExprNode(SExprNode* pExpr) { taosArrayDestroy(pExpr->pAssociation); } +static void nodesDestroyNodePointer(void* node) { + SNode* pNode = *(SNode**)node; + nodesDestroyNode(pNode); +} + void nodesDestroyNode(SNode* pNode) { if (NULL == pNode) { return; @@ -718,6 +723,7 @@ void nodesDestroyNode(SNode* pNode) { } taosArrayDestroy(pQuery->pDbList); taosArrayDestroy(pQuery->pTableList); + taosArrayDestroyEx(pQuery->pPlaceholderValues, nodesDestroyNodePointer); break; } case QUERY_NODE_LOGIC_PLAN_SCAN: { diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 704cd31428..acb7004a51 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -24,7 +24,7 @@ extern "C" { #include "dataSinkMgt.h" int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF); -int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); +int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 73110472f7..b937c44810 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -318,7 +318,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { SQWMsg qwMsg = {.msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; QW_SCH_TASK_DLOG("prerocessQuery start, handle:%p", pMsg->info.handle); - QW_ERR_RET(qwPrerocessQuery(QW_FPARAMS(), &qwMsg)); + QW_ERR_RET(qwPreprocessQuery(QW_FPARAMS(), &qwMsg)); QW_SCH_TASK_DLOG("prerocessQuery end, handle:%p", pMsg->info.handle); return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3e8ced318c..b22729fb9c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -469,7 +469,7 @@ int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF) { } -int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { +int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; bool queryRsped = false; SSubplan *plan = NULL; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 0a0d319025..ef817a557c 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -28,15 +28,6 @@ extern "C" { #include "trpc.h" #include "command.h" -#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT - -#define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 -#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 - -#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA - enum { SCH_READ = 1, SCH_WRITE, @@ -54,6 +45,24 @@ typedef enum { SCH_OP_GET_STATUS, } SCH_OP_TYPE; +typedef enum { + SCH_LOAD_SEQ = 1, + SCH_RANDOM, + SCH_ALL, +} SCH_POLICY; + +#define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 +#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT +#define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ + +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 +#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 +#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA + + + + typedef struct SSchDebug { bool lockEnable; bool apiEnable; @@ -184,34 +193,36 @@ typedef struct SSchLevel { typedef struct SSchTaskProfile { int64_t startTs; - int64_t* execTime; + SArray* execTime; int64_t waitTime; int64_t endTs; } SSchTaskProfile; typedef struct SSchTask { - uint64_t taskId; // task id - SRWLatch lock; // task reentrant lock - int32_t maxExecTimes; // task may exec times - int32_t execId; // task current execute try index - SSchLevel *level; // level - SRWLatch planLock; // task update plan lock - SSubplan *plan; // subplan - char *msg; // operator tree - int32_t msgLen; // msg length - int8_t status; // task status - int32_t lastMsgType; // last sent msg type - int64_t timeoutUsec; // taks timeout useconds before reschedule - SQueryNodeAddr succeedAddr; // task executed success node address - int8_t candidateIdx; // current try condidation index - SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo - SSchTaskProfile profile; // task execution profile - int32_t childReady; // child task ready number - SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* - SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* - void* handle; // task send handle - bool registerdHb; // registered in hb + uint64_t taskId; // task id + SRWLatch lock; // task reentrant lock + int32_t maxExecTimes; // task max exec times + int32_t maxRetryTimes; // task max retry times + int32_t retryTimes; // task retry times + int32_t execId; // task current execute index + SSchLevel *level; // level + SRWLatch planLock; // task update plan lock + SSubplan *plan; // subplan + char *msg; // operator tree + int32_t msgLen; // msg length + int8_t status; // task status + int32_t lastMsgType; // last sent msg type + int64_t timeoutUsec; // task timeout useconds before reschedule + SQueryNodeAddr succeedAddr; // task executed success node address + int8_t candidateIdx; // current try condidation index + SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr + SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo + SSchTaskProfile profile; // task execution profile + int32_t childReady; // child task ready number + SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* + SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* + void* handle; // task send handle + bool registerdHb; // registered in hb } SSchTask; typedef struct SSchJobAttr { @@ -298,7 +309,6 @@ extern SSchedulerMgmt schMgmt; #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) -#define SCH_TASK_MAX_EXEC_TIMES(_levelIdx, _levelNum) (SCH_MAX_CANDIDATE_EP_NUM * ((_levelNum) - (_levelIdx))) #define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) @@ -320,8 +330,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_START_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ - (_task)->profile.execTime[idx] = us; \ + taosArrayPush((_task)->profile.execTime, &us); \ if (0 == (_task)->execId) { \ (_task)->profile.startTs = us; \ } \ @@ -330,8 +339,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOG_TASK_WAIT_TS(_task) \ do { \ int64_t us = taosGetTimestampUs(); \ - int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ - (_task)->profile.waitTime += us - (_task)->profile.execTime[idx]; \ + (_task)->profile.waitTime += us - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId); \ } while (0) @@ -339,7 +347,8 @@ extern SSchedulerMgmt schMgmt; do { \ int64_t us = taosGetTimestampUs(); \ int32_t idx = (_task)->execId % (_task)->maxExecTimes; \ - (_task)->profile.execTime[idx] = us - (_task)->profile.execTime[idx]; \ + int64_t *startts = taosArrayGet((_task)->profile.execTime, (_task)->execId); \ + *startts = us - *startts; \ (_task)->profile.endTs = us; \ } while (0) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 71d123b799..ebe4bcab7b 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -42,7 +42,18 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { taosHashCleanup(pTask->execNodes); } - taosMemoryFree(pTask->profile.execTime); + taosArrayDestroy(pTask->profile.execTime); +} + +void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel, int32_t levelNum) { + if (SCH_IS_DATA_BIND_TASK(pTask) || (!SCH_IS_QUERY_JOB(pJob)) || (SCH_ALL != schMgmt.cfg.schPolicy)) { + pTask->maxRetryTimes = SCH_MAX_CANDIDATE_EP_NUM; + } else { + int32_t nodeNum = taosArrayGetSize(pJob->nodeList); + pTask->maxRetryTimes = TMAX(nodeNum, SCH_MAX_CANDIDATE_EP_NUM); + } + + pTask->maxExecTimes = pTask->maxRetryTimes * (levelNum - pLevel->level); } int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum) { @@ -51,12 +62,14 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->plan = pPlan; pTask->level = pLevel; pTask->execId = -1; - pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES(pLevel->level, levelNum); pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; pTask->taskId = schGenTaskId(); pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - pTask->profile.execTime = taosMemoryCalloc(pTask->maxExecTimes, sizeof(int64_t)); + + schInitTaskRetryTimes(pJob, pTask, pLevel, levelNum); + + pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -67,7 +80,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * _return: - taosMemoryFreeClear(pTask->profile.execTime); + taosArrayDestroy(pTask->profile.execTime); taosHashCleanup(pTask->execNodes); SCH_RET(code); @@ -285,6 +298,10 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { + if (!schMgmt.cfg.enableReSchedule) { + return TSDB_CODE_SUCCESS; + } + if (SCH_IS_DATA_BIND_TASK(pTask)) { return TSDB_CODE_SUCCESS; } @@ -320,6 +337,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; + pTask->retryTimes = 0; memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); if (SCH_IS_DATA_BIND_TASK(pTask)) { @@ -493,9 +511,15 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } } + if ((pTask->retryTimes + 1) > pTask->maxRetryTimes) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, pTask->maxRetryTimes); + return TSDB_CODE_SUCCESS; + } + if ((pTask->execId + 1) >= pTask->maxExecTimes) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since reach max try times, execId:%d", pTask->execId); + SCH_TASK_DLOG("task no more retry since reach max exec times, execId:%d/%d", pTask->execId, pTask->maxExecTimes); return TSDB_CODE_SUCCESS; } @@ -647,10 +671,31 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); - if (++pTask->candidateIdx >= candidateNum) { - pTask->candidateIdx = 0; + if (candidateNum <= 1) { + goto _return; } - SCH_TASK_DLOG("switch task candiateIdx to %d", pTask->candidateIdx); + + switch (schMgmt.cfg.schPolicy) { + case SCH_LOAD_SEQ: + case SCH_ALL: + default: + if (++pTask->candidateIdx >= candidateNum) { + pTask->candidateIdx = 0; + } + break; + case SCH_RANDOM: { + int32_t lastIdx = pTask->candidateIdx; + while (lastIdx == pTask->candidateIdx) { + pTask->candidateIdx = taosRand() % candidateNum; + } + break; + } + } + +_return: + + SCH_TASK_DLOG("switch task candiateIdx to %d/%d", pTask->candidateIdx, candidateNum); + return TSDB_CODE_SUCCESS; } @@ -737,6 +782,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; + pTask->retryTimes++; SCH_TASK_DLOG("start to launch task's %dth exec", pTask->execId); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 39465f3064..81c1ff9468 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -22,26 +22,19 @@ SSchedulerMgmt schMgmt = { .jobRef = -1, }; -int32_t schedulerInit(SSchedulerCfg *cfg) { +int32_t schedulerInit() { if (schMgmt.jobRef >= 0) { qError("scheduler already initialized"); return TSDB_CODE_QRY_INVALID_INPUT; } - if (cfg) { - schMgmt.cfg = *cfg; - - if (schMgmt.cfg.maxJobNum == 0) { - schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; - } - if (schMgmt.cfg.maxNodeTableNum <= 0) { - schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; - } - } else { - schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; - schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; - } + schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; + schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; + schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; + schMgmt.cfg.enableReSchedule = true; + qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy); + schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); @@ -130,6 +123,26 @@ void schedulerStopQueryHb(void *pTrans) { schCleanClusterHb(pTrans); } +int32_t schedulerUpdatePolicy(int32_t policy) { + switch (policy) { + case SCH_LOAD_SEQ: + case SCH_RANDOM: + case SCH_ALL: + schMgmt.cfg.schPolicy = policy; + qDebug("schedule policy updated to %d", schMgmt.cfg.schPolicy); + break; + default: + return TSDB_CODE_TSC_INVALID_INPUT; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schedulerEnableReSchedule(bool enableResche) { + schMgmt.cfg.enableReSchedule = enableResche; + return TSDB_CODE_SUCCESS; +} + void schedulerFreeJob(int64_t* jobId, int32_t errCode) { if (0 == *jobId) { return; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index d6b1baf978..cbfa8215dc 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -477,7 +477,7 @@ void* schtRunJobThread(void *aa) { schtInitLogFile(); - int32_t code = schedulerInit(NULL); + int32_t code = schedulerInit(); assert(code == 0); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index b31c39718c..29c1fdb015 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -2685,6 +2685,8 @@ int main(int argc, char *argv[]) runAll(taos); + taos_close(taos); + return 0; } From cace9b6be909a0f8b6d5d408285613273fdad241 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 14:51:39 +0800 Subject: [PATCH 05/94] feat: add lru to index --- include/libs/index/index.h | 2 +- source/dnode/vnode/src/meta/metaOpen.c | 6 +- source/libs/index/inc/indexFstFile.h | 4 +- source/libs/index/inc/indexInt.h | 23 ++---- source/libs/index/inc/indexTfile.h | 7 +- source/libs/index/src/index.c | 68 +++++++++++------ source/libs/index/src/indexFst.c | 5 +- source/libs/index/src/indexFstFile.c | 100 +++++++++++++++++++------ source/libs/index/src/indexTfile.c | 54 +++++-------- source/libs/index/test/fstTest.cc | 12 +-- source/libs/index/test/fstUT.cc | 5 +- source/libs/index/test/indexTests.cc | 53 ++++++------- source/libs/index/test/jsonUT.cc | 41 +++++----- 13 files changed, 220 insertions(+), 160 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c6641f8b02..08d64699d4 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -173,7 +173,7 @@ void indexMultiTermDestroy(SIndexMultiTerm* terms); * @param: * @param: */ -SIndexOpts* indexOptsCreate(); +SIndexOpts* indexOptsCreate(int32_t cacheSize); void indexOptsDestroy(SIndexOpts* opts); /* diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 1022f6796b..0a27f0bc4e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -99,12 +99,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } - // open pTagIdx - // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); taosMkDir(indexFullPath); - ret = indexOpen(indexOptsCreate(), indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + + SIndexOpts *opts = indexOptsCreate(8 * 1024 * 1024); + ret = indexOpen(opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); if (ret < 0) { metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/libs/index/inc/indexFstFile.h b/source/libs/index/inc/indexFstFile.h index a161c4aee1..0ddffe7df0 100644 --- a/source/libs/index/inc/indexFstFile.h +++ b/source/libs/index/inc/indexFstFile.h @@ -27,7 +27,7 @@ extern "C" { #define DefaultMem 1024 * 1024 static char tmpFile[] = "./index"; -typedef enum WriterType { TMemory, TFile } WriterType; +typedef enum WriterType { TMEMORY, TFILE } WriterType; typedef struct IFileCtx { int (*write)(struct IFileCtx* ctx, uint8_t* buf, int len); @@ -35,6 +35,8 @@ typedef struct IFileCtx { int (*flush)(struct IFileCtx* ctx); int (*readFrom)(struct IFileCtx* ctx, uint8_t* buf, int len, int32_t offset); int (*size)(struct IFileCtx* ctx); + + SLRUCache* lru; WriterType type; union { struct { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index d50fa0e917..9370f7c708 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -24,12 +24,9 @@ #include "tchecksum.h" #include "thash.h" #include "tlog.h" +#include "tlrucache.h" #include "tutil.h" -#ifdef USE_LUCENE -#include -#endif - #ifdef __cplusplus extern "C" { #endif @@ -61,28 +58,22 @@ struct SIndex { void* tindex; SHashObj* colObj; // < field name, field id> - int64_t suid; // current super table id, -1 is normal table - int32_t cVersion; // current version allocated to cache - - char* path; + int64_t suid; // current super table id, -1 is normal table + int32_t cVersion; // current version allocated to cache + SLRUCache* lru; + char* path; int8_t status; SIndexStat stat; TdThreadMutex mtx; tsem_t sem; bool quit; + void* opts; }; struct SIndexOpts { -#ifdef USE_LUCENE - void* opts; -#endif - -#ifdef USE_INVERTED_INDEX int32_t cacheSize; // MB - // add cache module later -#endif - int32_t cacheOpt; // MB + int32_t cacheOpt; // MB }; struct SIndexMultiTermQuery { diff --git a/source/libs/index/inc/indexTfile.h b/source/libs/index/inc/indexTfile.h index ca5c688162..7425b13ce9 100644 --- a/source/libs/index/inc/indexTfile.h +++ b/source/libs/index/inc/indexTfile.h @@ -71,6 +71,7 @@ typedef struct TFileReader { IFileCtx* ctx; TFileHeader header; bool remove; + void* lru; } TFileReader; typedef struct IndexTFile { @@ -95,14 +96,14 @@ typedef struct TFileReaderOpt { } TFileReaderOpt; // tfile cache, manage tindex reader -TFileCache* tfileCacheCreate(const char* path); +TFileCache* tfileCacheCreate(SIndex* idx, const char* path); void tfileCacheDestroy(TFileCache* tcache); TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key); void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader); TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName); -TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const char* colName); +TFileReader* tfileReaderOpen(SIndex* idx, uint64_t suid, int64_t version, const char* colName); TFileReader* tfileReaderCreate(IFileCtx* ctx); void tfileReaderDestroy(TFileReader* reader); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr); @@ -117,7 +118,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // -IndexTFile* idxTFileCreate(const char* path); +IndexTFile* idxTFileCreate(SIndex* idx, const char* path); void idxTFileDestroy(IndexTFile* tfile); int idxTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); int idxTFileSearch(void* tfile, SIndexTermQuery* query, SIdxTRslt* tr); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e3d367e59c..c710858b82 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -103,44 +103,61 @@ static void indexWait(void* idx) { int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { int ret = TSDB_CODE_SUCCESS; taosThreadOnce(&isInit, indexInit); - SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); - if (sIdx == NULL) { + SIndex* idx = taosMemoryCalloc(1, sizeof(SIndex)); + if (idx == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - sIdx->tindex = idxTFileCreate(path); - if (sIdx->tindex == NULL) { + idx->lru = taosLRUCacheInit(opts->cacheSize, -1, .5); + if (idx->lru == NULL) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } + taosLRUCacheSetStrictCapacity(idx->lru, true); + + idx->tindex = idxTFileCreate(idx, path); + if (idx->tindex == NULL) { ret = TSDB_CODE_OUT_OF_MEMORY; goto END; } - sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - sIdx->cVersion = 1; - sIdx->path = tstrdup(path); - taosThreadMutexInit(&sIdx->mtx, NULL); - tsem_init(&sIdx->sem, 0, 0); + idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + idx->cVersion = 1; + idx->path = tstrdup(path); + taosThreadMutexInit(&idx->mtx, NULL); + tsem_init(&idx->sem, 0, 0); - sIdx->refId = idxAddRef(sIdx); - idxAcquireRef(sIdx->refId); + idx->refId = idxAddRef(idx); + idx->opts = opts; + idxAcquireRef(idx->refId); - *index = sIdx; + *index = idx; return ret; END: - if (sIdx != NULL) { - indexClose(sIdx); + if (idx != NULL) { + indexClose(idx); } *index = NULL; return ret; } void indexDestroy(void* handle) { - SIndex* sIdx = handle; - taosThreadMutexDestroy(&sIdx->mtx); - tsem_destroy(&sIdx->sem); - idxTFileDestroy(sIdx->tindex); - taosMemoryFree(sIdx->path); - taosMemoryFree(sIdx); + SIndex* idx = handle; + taosThreadMutexDestroy(&idx->mtx); + tsem_destroy(&idx->sem); + idxTFileDestroy(idx->tindex); + taosMemoryFree(idx->path); + + SLRUCache* lru = idx->lru; + if (lru != NULL) { + taosLRUCacheEraseUnrefEntries(lru); + taosLRUCacheCleanup(lru); + } + idx->lru = NULL; + + indexOptsDestroy(idx->opts); + taosMemoryFree(idx); return; } void indexClose(SIndex* sIdx) { @@ -159,6 +176,7 @@ void indexClose(SIndex* sIdx) { taosHashCleanup(sIdx->colObj); sIdx->colObj = NULL; } + idxReleaseRef(sIdx->refId); idxRemoveRef(sIdx->refId); } @@ -234,8 +252,12 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { return 1; } // int indexRebuild(SIndex* index, SIndexOpts* opts) { return 0; } -SIndexOpts* indexOptsCreate() { return NULL; } -void indexOptsDestroy(SIndexOpts* opts) { return; } +SIndexOpts* indexOptsCreate(int32_t cacheSize) { + SIndexOpts* opts = taosMemoryCalloc(1, sizeof(SIndexOpts)); + opts->cacheSize = cacheSize; + return opts; +} +void indexOptsDestroy(SIndexOpts* opts) { return taosMemoryFree(opts); } /* * @param: oper * @@ -641,7 +663,7 @@ static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { } tfileWriterClose(tw); - TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); + TFileReader* reader = tfileReaderOpen(sIdx, cache->suid, version, cache->colName); if (reader == NULL) { return -1; } diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index c4b83f8a07..15152cef55 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -772,6 +772,7 @@ void fstBuilderDestroy(FstBuilder* b) { if (b == NULL) { return; } + fstBuilderFinish(b); idxFileDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); @@ -1074,8 +1075,8 @@ FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx) { } FstNode* fstGetRoot(Fst* fst) { - CompiledAddr rAddr = fstGetRootAddr(fst); - return fstGetNode(fst, rAddr); + CompiledAddr addr = fstGetRootAddr(fst); + return fstGetNode(fst, addr); } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 9106caebd6..c8023d51c9 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -4,8 +4,7 @@ * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. * @@ -14,13 +13,32 @@ */ #include "indexFstFile.h" +#include "indexComm.h" #include "indexFstUtil.h" #include "indexInt.h" +#include "indexUtil.h" #include "os.h" #include "tutil.h" +static int32_t kBlockSize = 4096; + +typedef struct { + int32_t blockId; + int32_t nread; + char buf[0]; +} SDataBlock; + +static void deleteDataBlockFromLRU(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } + +static void idxGenLRUKey(char* buf, const char* path, int32_t blockId) { + char* p = buf; + SERIALIZE_STR_VAR_TO_BUF(p, path, strlen(path)); + SERIALIZE_VAR_TO_BUF(p, '_', char); + idxInt2str(blockId, p, 0); + return; +} static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { - if (ctx->type == TFile) { + if (ctx->type == TFILE) { assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); @@ -30,7 +48,7 @@ static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { } static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { int nRead = 0; - if (ctx->type == TFile) { + if (ctx->type == TFILE) { #ifdef USE_MMAP nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); @@ -45,24 +63,59 @@ static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { return nRead; } static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset) { - int nRead = 0; - if (ctx->type == TFile) { - // tfLseek(ctx->file.pFile, offset, 0); + int32_t total = 0, nread = 0; + int32_t blkId = offset / kBlockSize; + int32_t blkOffset = offset % kBlockSize; + int32_t blkLeft = kBlockSize - blkOffset; + + do { + char key[128] = {0}; + idxGenLRUKey(key, ctx->file.buf, blkId); + LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key)); + + if (h) { + SDataBlock* blk = taosLRUCacheValue(ctx->lru, h); + nread = MIN(blkLeft, len); + memcpy(buf + total, blk->buf + blkOffset, nread); + taosLRUCacheRelease(ctx->lru, h, false); + } else { + int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; + SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); + + blk->blockId = blkId; + blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); + assert(blk->nread <= kBlockSize); + nread = MIN(blkLeft, len); + if (blk->nread < kBlockSize && blk->nread < len) { + break; + } + memcpy(buf + total, blk->buf + blkOffset, nread); + + LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, + TAOS_LRU_PRIORITY_LOW); + if (s != TAOS_LRU_STATUS_OK) { + return -1; + } + } + total += nread; + len -= nread; + offset += nread; + + blkId = offset / kBlockSize; + blkOffset = offset % kBlockSize; + blkLeft = kBlockSize - blkOffset; + + } while (len > 0); + #ifdef USE_MMAP - int32_t last = ctx->file.size - offset; - nRead = last >= len ? len : last; - memcpy(buf, ctx->file.ptr + offset, nRead); -#else - nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); + int32_t last = ctx->file.size - offset; + nRead = last >= len ? len : last; + memcpy(buf, ctx->file.ptr + offset, nRead); #endif - } else { - // refactor later - assert(0); - } - return nRead; + return total; } static int idxFileCtxGetSize(IFileCtx* ctx) { - if (ctx->type == TFile) { + if (ctx->type == TFILE) { int64_t file_size = 0; taosStatFile(ctx->file.buf, &file_size, NULL); return (int)file_size; @@ -70,7 +123,7 @@ static int idxFileCtxGetSize(IFileCtx* ctx) { return 0; } static int idxFileCtxDoFlush(IFileCtx* ctx) { - if (ctx->type == TFile) { + if (ctx->type == TFILE) { taosFsyncFile(ctx->file.pFile); } else { // do nothing @@ -85,7 +138,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int } ctx->type = type; - if (ctx->type == TFile) { + if (ctx->type == TFILE) { // ugly code, refactor later ctx->file.readOnly = readOnly; memcpy(ctx->file.buf, path, strlen(path)); @@ -109,7 +162,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int indexError("failed to open file, error %d", errno); goto END; } - } else if (ctx->type == TMemory) { + } else if (ctx->type == TMEMORY) { ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); ctx->mem.cap = capacity; } @@ -124,14 +177,14 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int return ctx; END: - if (ctx->type == TMemory) { + if (ctx->type == TMEMORY) { taosMemoryFree(ctx->mem.buf); } taosMemoryFree(ctx); return NULL; } void idxFileCtxDestroy(IFileCtx* ctx, bool remove) { - if (ctx->type == TMemory) { + if (ctx->type == TMEMORY) { taosMemoryFree(ctx->mem.buf); } else { ctx->flush(ctx); @@ -183,6 +236,7 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { write->summer = taosCalcChecksum(write->summer, buf, len); return len; } + int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len) { if (write == NULL) { return 0; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 56ebd9eb18..b4e59d0f63 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -90,7 +90,7 @@ static int32_t (*tfSearch[][QUERY_MAX])(void* reader, SIndexTerm* tem, SIdxTRslt {tfSearchEqual_JSON, tfSearchPrefix_JSON, tfSearchSuffix_JSON, tfSearchRegex_JSON, tfSearchLessThan_JSON, tfSearchLessEqual_JSON, tfSearchGreaterThan_JSON, tfSearchGreaterEqual_JSON, tfSearchRange_JSON}}; -TFileCache* tfileCacheCreate(const char* path) { +TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); if (tcache == NULL) { return NULL; @@ -103,17 +103,20 @@ TFileCache* tfileCacheCreate(const char* path) { for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); - IFileCtx* wc = idxFileCtxCreate(TFile, file, true, 1024 * 1024 * 64); - if (wc == NULL) { + IFileCtx* ctx = idxFileCtxCreate(TFILE, file, true, 1024 * 1024 * 64); + if (ctx == NULL) { indexError("failed to open index:%s", file); goto End; } + ctx->lru = idx->lru; - TFileReader* reader = tfileReaderCreate(wc); + TFileReader* reader = tfileReaderCreate(ctx); if (reader == NULL) { indexInfo("skip invalid file: %s", file); continue; } + reader->lru = idx->lru; + TFileHeader* header = &reader->header; ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = (int32_t)strlen(header->colName)}; @@ -160,9 +163,8 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { return *reader; } void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { - char buf[128] = {0}; - int32_t sz = idxSerialCacheKey(key, buf); - // remove last version index reader + char buf[128] = {0}; + int32_t sz = idxSerialCacheKey(key, buf); TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL && *p != NULL) { TFileReader* oldRdr = *p; @@ -493,7 +495,7 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c char fullname[256] = {0}; tfileGenFileFullName(fullname, path, suid, colName, version); // indexInfo("open write file name %s", fullname); - IFileCtx* wcx = idxFileCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); + IFileCtx* wcx = idxFileCtxCreate(TFILE, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; } @@ -506,16 +508,17 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c return tfileWriterCreate(wcx, &tfh); } -TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const char* colName) { +TFileReader* tfileReaderOpen(SIndex* idx, uint64_t suid, int64_t version, const char* colName) { char fullname[256] = {0}; - tfileGenFileFullName(fullname, path, suid, colName, version); + tfileGenFileFullName(fullname, idx->path, suid, colName, version); - IFileCtx* wc = idxFileCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); + IFileCtx* wc = idxFileCtxCreate(TFILE, fullname, true, 1024 * 1024 * 1024); if (wc == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr()); return NULL; } + wc->lru = idx->lru; indexTrace("open read file name:%s, file size: %" PRId64 "", wc->file.buf, wc->file.size); TFileReader* reader = tfileReaderCreate(wc); @@ -598,17 +601,11 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } else { - // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, - // (int)taosArrayGetSize(v->tableId)); - - // indexInfo("tfile write data size: %d", tw->ctx->size(tw->ctx)); + indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, + (int)taosArrayGetSize(v->tableId)); } } - - fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); - tw->fb = NULL; - tfileWriteFooter(tw); return 0; } @@ -627,8 +624,8 @@ void tfileWriterDestroy(TFileWriter* tw) { taosMemoryFree(tw); } -IndexTFile* idxTFileCreate(const char* path) { - TFileCache* cache = tfileCacheCreate(path); +IndexTFile* idxTFileCreate(SIndex* idx, const char* path) { + TFileCache* cache = tfileCacheCreate(idx, path); if (cache == NULL) { return NULL; } @@ -859,18 +856,6 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { return 0; } return -1; - - // if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { - // FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); - // if (fstBuilderInsert(write->fb, key, tval->offset)) { - // fstSliceDestroy(&key); - // return 0; - // } - // fstSliceDestroy(&key); - // return -1; - //} else { - // // handle other type later - //} } static int tfileWriteFooter(TFileWriter* write) { char buf[sizeof(FILE_MAGIC_NUMBER) + 1] = {0}; @@ -887,6 +872,7 @@ static int tfileReaderLoadHeader(TFileReader* reader) { char buf[TFILE_HEADER_SIZE] = {0}; int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); + if (nread == -1) { indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, reader->ctx->file.buf); @@ -914,7 +900,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { int64_t cost = taosGetTimestampUs() - ts; indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %" PRId64 ", time cost: %" PRId64 "us", - nread, reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost); + nread, reader->header.fstOffset, fstSize, ctx->file.buf, size, cost); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread <= fstSize); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 7109c65e85..4e9a853302 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -19,7 +19,7 @@ class FstWriter { public: FstWriter() { taosRemoveFile(fileName.c_str()); - _wc = idxFileCtxCreate(TFile, fileName.c_str(), false, 64 * 1024 * 1024); + _wc = idxFileCtxCreate(TFILE, fileName.c_str(), false, 64 * 1024 * 1024); _b = fstBuilderCreate(_wc, 0); } bool Put(const std::string& key, uint64_t val) { @@ -34,7 +34,7 @@ class FstWriter { return ok; } ~FstWriter() { - fstBuilderFinish(_b); + // fstBuilderFinish(_b); fstBuilderDestroy(_b); idxFileCtxDestroy(_wc, false); @@ -48,7 +48,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(int32_t size, const std::string& fileName = TD_TMP_DIR_PATH "tindex.tindex") { - _wc = idxFileCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); + _wc = idxFileCtxCreate(TFILE, fileName.c_str(), true, 64 * 1024); _w = idxFileCreate(_wc); _size = size; memset((void*)&_s, 0, sizeof(_s)); @@ -598,7 +598,9 @@ void fst_get(Fst* fst) { void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); + SIndex* index = (SIndex*)taosMemoryCalloc(1, sizeof(SIndex)); + index->path = strdup(arg); + TFileReader* reader = tfileReaderOpen(index, 0, 20000000, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -617,7 +619,7 @@ void iterTFileReader(char* path, char* uid, char* colName, char* ver) { uint64_t suid = atoi(uid); int version = atoi(ver); - TFileReader* reader = tfileReaderOpen(path, suid, version, colName); + TFileReader* reader = tfileReaderOpen(NULL, suid, version, colName); Iterate* iter = tfileIteratorCreate(reader); bool tn = iter ? iter->next(iter) : false; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index b8663dd9f2..37c9d1b97b 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -39,7 +39,7 @@ static void EnvCleanup() {} class FstWriter { public: FstWriter() { - _wc = idxFileCtxCreate(TFile, tindex, false, 64 * 1024 * 1024); + _wc = idxFileCtxCreate(TFILE, tindex, false, 64 * 1024 * 1024); _b = fstBuilderCreate(_wc, 0); } bool Put(const std::string& key, uint64_t val) { @@ -54,7 +54,6 @@ class FstWriter { return ok; } ~FstWriter() { - fstBuilderFinish(_b); fstBuilderDestroy(_b); idxFileCtxDestroy(_wc, false); @@ -68,7 +67,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _wc = idxFileCtxCreate(TFile, tindex, true, 64 * 1024); + _wc = idxFileCtxCreate(TFILE, tindex, true, 64 * 1024); _w = idxFileCreate(_wc); _size = size; memset((void*)&_s, 0, sizeof(_s)); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 6b20205014..42127242b6 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -50,7 +50,7 @@ class DebugInfo { class FstWriter { public: FstWriter() { - _wc = idxFileCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024); + _wc = idxFileCtxCreate(TFILE, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024); _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string& key, uint64_t val) { @@ -60,7 +60,7 @@ class FstWriter { return ok; } ~FstWriter() { - fstBuilderFinish(_b); + // fstBuilderFinish(_b); fstBuilderDestroy(_b); idxFileCtxDestroy(_wc, false); @@ -74,7 +74,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _wc = idxFileCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024); + _wc = idxFileCtxCreate(TFILE, TD_TMP_DIR_PATH "tindex", true, 64 * 1024); _w = idxFileCreate(_wc); _size = size; memset((void*)&_s, 0, sizeof(_s)); @@ -292,7 +292,7 @@ class IndexEnv : public ::testing::Test { virtual void SetUp() { initLog(); taosRemoveDir(path); - opts = indexOptsCreate(); + opts = indexOptsCreate(1024 * 1024 * 8); int ret = indexOpen(opts, path, &index); assert(ret == 0); } @@ -391,13 +391,15 @@ class TFileObj { fileName_ = path; - IFileCtx* ctx = idxFileCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024); + IFileCtx* ctx = idxFileCtxCreate(TFILE, path.c_str(), false, 64 * 1024 * 1024); + ctx->lru = taosLRUCacheInit(1024 * 1024 * 4, -1, .5); writer_ = tfileWriterCreate(ctx, &header); return writer_ != NULL ? true : false; } bool InitReader() { - IFileCtx* ctx = idxFileCtxCreate(TFile, fileName_.c_str(), true, 64 * 1024 * 1024); + IFileCtx* ctx = idxFileCtxCreate(TFILE, fileName_.c_str(), true, 64 * 1024 * 1024); + ctx->lru = taosLRUCacheInit(1024 * 1024 * 4, -1, .5); reader_ = tfileReaderCreate(ctx); return reader_ != NULL ? true : false; } @@ -657,7 +659,7 @@ TEST_F(IndexCacheEnv, cache_test) { { std::string colVal("v3"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -672,7 +674,7 @@ TEST_F(IndexCacheEnv, cache_test) { { std::string colVal("v2"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -698,7 +700,8 @@ class IndexObj { taosMkDir(dir.c_str()); } taosMkDir(dir.c_str()); - int ret = indexOpen(&opts, dir.c_str(), &idx); + opts = indexOptsCreate(1024 * 1024 * 4); + int ret = indexOpen(opts, dir.c_str(), &idx); if (ret != 0) { // opt std::cout << "failed to open index: %s" << dir << std::endl; @@ -707,7 +710,7 @@ class IndexObj { } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { SIndexTerm* term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); Put(terms, uid); @@ -716,7 +719,7 @@ class IndexObj { int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < numOfTable; i++) { @@ -738,7 +741,7 @@ class IndexObj { tColVal[taosRand() % colValSize] = 'a' + k % 26; } SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - tColVal.c_str(), tColVal.size()); + tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t j = 0; j < skip; j++) { @@ -774,7 +777,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -796,7 +799,7 @@ class IndexObj { int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -821,7 +824,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); indexMultiTermDestroy(terms); @@ -829,7 +832,7 @@ class IndexObj { void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); indexMultiTermDestroy(terms); @@ -845,10 +848,10 @@ class IndexObj { } private: - SIndexOpts opts; - SIndex* idx; - int numOfWrite; - int numOfRead; + SIndexOpts* opts; + SIndex* idx; + int numOfWrite; + int numOfRead; }; class IndexEnv2 : public ::testing::Test { @@ -875,7 +878,7 @@ TEST_F(IndexEnv2, testIndexOpen) { std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { @@ -890,7 +893,7 @@ TEST_F(IndexEnv2, testIndexOpen) { std::string colName("tag1"), colVal("hello"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < size; i++) { @@ -905,7 +908,7 @@ TEST_F(IndexEnv2, testIndexOpen) { std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = size * 3; i < size * 4; i++) { @@ -920,7 +923,7 @@ TEST_F(IndexEnv2, testIndexOpen) { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -943,7 +946,7 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) { std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index c65949277e..1911514d97 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -54,13 +54,12 @@ class JsonEnv : public ::testing::Test { printf("set up\n"); initLog(); - opts = indexOptsCreate(); + opts = indexOptsCreate(1024 * 1024 * 4); int ret = indexJsonOpen(opts, dir.c_str(), &index); assert(ret == 0); } virtual void TearDown() { indexJsonClose(index); - indexOptsDestroy(opts); printf("destory\n"); taosMsleep(1000); } @@ -71,7 +70,7 @@ class JsonEnv : public ::testing::Test { static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = ADD_VALUE) { SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), - (const char*)data, dlen); + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); indexJsonPut(index, terms, (int64_t)tableId); @@ -82,7 +81,7 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = DEL_VALUE) { SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), - (const char*)data, dlen); + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); indexJsonPut(index, terms, (int64_t)tableId); @@ -108,7 +107,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); for (int i = 0; i < 100; i++) { SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); indexJsonPut(index, terms, i); @@ -147,7 +146,7 @@ TEST_F(JsonEnv, testWrite) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -205,7 +204,7 @@ TEST_F(JsonEnv, testWriteMillonData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -220,7 +219,7 @@ TEST_F(JsonEnv, testWriteMillonData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -235,7 +234,7 @@ TEST_F(JsonEnv, testWriteMillonData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -305,7 +304,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -319,7 +318,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -334,7 +333,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -349,7 +348,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -364,7 +363,7 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -407,7 +406,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -421,7 +420,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -436,7 +435,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -450,7 +449,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -464,7 +463,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -493,7 +492,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -521,7 +520,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); From a9e64f29380062871c6c6dbfd4e52d56c8b1be19 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 15:17:05 +0800 Subject: [PATCH 06/94] feat: add lru to index --- include/libs/index/index.h | 4 +++- source/dnode/vnode/src/meta/metaOpen.c | 4 ++-- source/dnode/vnode/src/tsdb/.tsdbCache.c.swo | Bin 0 -> 16384 bytes source/libs/index/inc/indexInt.h | 7 +------ source/libs/index/src/index.c | 4 +--- source/libs/index/test/indexTests.cc | 13 +++++-------- 6 files changed, 12 insertions(+), 20 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/.tsdbCache.c.swo diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 08d64699d4..c1fdc4df52 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -28,7 +28,6 @@ extern "C" { typedef struct SIndex SIndex; typedef struct SIndexTerm SIndexTerm; -typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; @@ -62,6 +61,9 @@ typedef enum { QUERY_MAX } EIndexQueryType; +typedef struct SIndexOpts { + int32_t cacheSize; // MB +} SIndexOpts; /* * create multi query * @param oper (input, relation between querys) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 0a27f0bc4e..59df35d554 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -103,8 +103,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); taosMkDir(indexFullPath); - SIndexOpts *opts = indexOptsCreate(8 * 1024 * 1024); - ret = indexOpen(opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; + ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); if (ret < 0) { metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/tsdb/.tsdbCache.c.swo b/source/dnode/vnode/src/tsdb/.tsdbCache.c.swo new file mode 100644 index 0000000000000000000000000000000000000000..9feb547e0573668075490648079b7daa25b63e0c GIT binary patch literal 16384 zcmeI2Yit}>6~~8n+w>8U3M3Hp#A%AXUGMslI1eW#TfbuB#MbUQsfExoyL;C=cs#Sr z+_mF8N=PY0JW_%9Kp_YULO{#w0}_0oAR!16Dx?Sy;sZ3O67U6-540prOZcBVcXnsj zPMkynDl{wo_HiHQo_p@|oY`W}!HNB}J=JUAwcapJReo9AHhr)0m+u;e=em-U*LrOm zD2c7+4pAA{w)I+NA7X*&CD)rz=SPGyXE`G6yLGQ5(q+dji}a#;_q|eD`sJc}8#YT- zkt$tFY@8QcAhy6-7O2Zg_x3xC{=VKGF1o2>BW-x&%v!bL{MZ7q1!4=t7KkknTOhVT zY=PJUu?7B*EFf!l8XrJ^Uak9dLVv!h?emI$9nt9@Y|HYF{$-tiTIV0p3WxPy)cGIQ`M10#_!8z{()o|-{L^ju7j-_{f?scYZSWb^ z|EJCm`*)@-{{@|YBB*9;d|hk(=XL%`o&R`S{y*B<-}L&{`dVG%ox1*)+VXXB{1ICq zwm@uw*aEQyVhh9;h%FFXAhtkkf!G4E1y;3ySvHK#$jPujjpzR-bg7r`GK?RC9{~vx z;Lmp&#&h7)U>~>-Yy!W#!!RBN9|Io+hrtr4fCs@C*a>!k4dA8Q4dXoc6!;`4fN`)1 z+zr-)yTJK%hVd-;J$MRy3w#4S2~GkJjDs<-8EgdifVYF6+-4Z3!C|ln>fl~*;mgZ1D70t3GV zXTcA__rdqT$*RO;4x3yT9)}NIu$goHJ0ozpo&vgui?Qg~Js7*N6JU$B3|n6rH)r?2P(N ziJJOMk}MYLRvESNo}_$k-{>rfePX$bjvB^sNcV)SdyW!5Xy7-e5v9|#=vrlxrt617 zM?KSTX@g4qXOkrAI=&3bC(x%P`PLDECWHQUaUVjt)FLqpwOMAUY&A(0)AmJJg#>;; zQLevcP3hIcidKa>EoafRt+FYF>M9Eey4L(8Dtf}45A@>#%{s6%%L}tuAX|-wke>Gs zns)b)^jR=hBg##hz8rRKN(8y6j=>1iy4Z>t&oh_Dh3u?xdXg*@*Zr)x9_uT62VgW# zSEC$!+eHo%2I>C$RRtb3ZZIs(nXGH1fgFqjb5DIQr;wJu*^CHk+jySOLZMnoijkl&ETj!kUw=J72Fg23gIJ$kdnem{t7F%R&GCUrwr?6|+R`bXS{H-hXpedG68;Pb^=W z(kcZzZ&@7y&6T8P>a};~q~5c!Enw@Uqs?t(MEKHkmpg-;LF}ByT5YlZH__aFvEX{k zW1bMv#(1Yr*y?QLv&5}nQ*h+KmV(5-gS=oTPc%{~(S=3L)2xxyg>9sZj#1Cy?LDn8 z-t~^_qeYFsjYW+#G})-#6xe7O^0+4NdFtHDFV}D!V_#v*@qx)=3NS;9saDLb)&m*6 z3Qkd$hLe=<0Kpl`Icx&nUX7?dq=(c_bXRS`zEks1w89BV>cf8 z!fHQlsEk_Cp={SL*V(FIFrPXKL zzCJEgAKB!=6@iTr+)Mc8W4nHEYOGv14Xd?WZ-{DnRO}wIy}CMXFk=G2gGNDk-cFud zQN?pY=AceV4x2TzWXa{mDW-25C?9lp&Xq2)jNnYwceQ+Oh2_@;!M1C)(sn#ENQX7|N+)O(%=Cy9Y;_8v+*X~X>AwI@gHzxm;2>as|E=IHU>&#(T!x>230wp}0AB-N1rLGazyUUx2Ptp? zeg8Z78wle8*XR4jrD6-j7KkknTOhVTY=PJUH)jFu@314Fe2sC@^v`n2*MZ-mKH+$T z0q&?f6|E3DM{S;AmkN5YZx^iye=u~xIkPW&$Fp<~A)U2F|V zsMwZ@dPNJXsBScF5f+mi%YTPYVxh(kt4D~5xG+m(BC`|N-D|L7I*PC$yg9)ki9ST& z@SI95Mg;$xX4j@en616a8Z}y;ZUhfDn(+*!^cn)u(`Msg6kE^l(3)X5k>f`!^ClJm zTlkg!ECqcF76fKt^KnX$8hxU2U7HU!VIJ|lD}#6KPc5UNq?VmYPwT7hPrQuOkcaW? V8lp}r0VCQ7_(f)>T1MBb^>4m-J}dwL literal 0 HcmV?d00001 diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 9370f7c708..065f4acb57 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -68,12 +68,7 @@ struct SIndex { TdThreadMutex mtx; tsem_t sem; bool quit; - void* opts; -}; - -struct SIndexOpts { - int32_t cacheSize; // MB - int32_t cacheOpt; // MB + SIndexOpts opts; }; struct SIndexMultiTermQuery { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index c710858b82..283d59df4b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -128,7 +128,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { tsem_init(&idx->sem, 0, 0); idx->refId = idxAddRef(idx); - idx->opts = opts; + idx->opts = *opts; idxAcquireRef(idx->refId); *index = idx; @@ -155,8 +155,6 @@ void indexDestroy(void* handle) { taosLRUCacheCleanup(lru); } idx->lru = NULL; - - indexOptsDestroy(idx->opts); taosMemoryFree(idx); return; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 42127242b6..3c1f148ae4 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -292,14 +292,11 @@ class IndexEnv : public ::testing::Test { virtual void SetUp() { initLog(); taosRemoveDir(path); - opts = indexOptsCreate(1024 * 1024 * 8); - int ret = indexOpen(opts, path, &index); + SIndexOpts opts = {.cacheSize = 1024 * 1024 * 4}; + int ret = indexOpen(&opts, path, &index); assert(ret == 0); } - virtual void TearDown() { - indexClose(index); - indexOptsDestroy(opts); - } + virtual void TearDown() { indexClose(index); } const char* path = TD_TMP_DIR_PATH "tindex"; SIndexOpts* opts; @@ -700,8 +697,8 @@ class IndexObj { taosMkDir(dir.c_str()); } taosMkDir(dir.c_str()); - opts = indexOptsCreate(1024 * 1024 * 4); - int ret = indexOpen(opts, dir.c_str(), &idx); + SIndexOpts opts = {.cacheSize = 1024 * 1024 * 4}; + int ret = indexOpen(&opts, dir.c_str(), &idx); if (ret != 0) { // opt std::cout << "failed to open index: %s" << dir << std::endl; From fe8ccabf7a30c7121080996885a7e3afd4a37218 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 17:14:53 +0800 Subject: [PATCH 07/94] add lru to index --- source/dnode/vnode/src/tsdb/.tsdbCache.c.swo | Bin 16384 -> 0 bytes source/libs/index/src/index.c | 2 +- source/libs/index/src/indexFilter.c | 12 ++++++------ source/libs/index/src/indexFstFile.c | 9 +-------- source/libs/transport/src/transCli.c | 2 ++ 5 files changed, 10 insertions(+), 15 deletions(-) delete mode 100644 source/dnode/vnode/src/tsdb/.tsdbCache.c.swo diff --git a/source/dnode/vnode/src/tsdb/.tsdbCache.c.swo b/source/dnode/vnode/src/tsdb/.tsdbCache.c.swo deleted file mode 100644 index 9feb547e0573668075490648079b7daa25b63e0c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI2Yit}>6~~8n+w>8U3M3Hp#A%AXUGMslI1eW#TfbuB#MbUQsfExoyL;C=cs#Sr z+_mF8N=PY0JW_%9Kp_YULO{#w0}_0oAR!16Dx?Sy;sZ3O67U6-540prOZcBVcXnsj zPMkynDl{wo_HiHQo_p@|oY`W}!HNB}J=JUAwcapJReo9AHhr)0m+u;e=em-U*LrOm zD2c7+4pAA{w)I+NA7X*&CD)rz=SPGyXE`G6yLGQ5(q+dji}a#;_q|eD`sJc}8#YT- zkt$tFY@8QcAhy6-7O2Zg_x3xC{=VKGF1o2>BW-x&%v!bL{MZ7q1!4=t7KkknTOhVT zY=PJUu?7B*EFf!l8XrJ^Uak9dLVv!h?emI$9nt9@Y|HYF{$-tiTIV0p3WxPy)cGIQ`M10#_!8z{()o|-{L^ju7j-_{f?scYZSWb^ z|EJCm`*)@-{{@|YBB*9;d|hk(=XL%`o&R`S{y*B<-}L&{`dVG%ox1*)+VXXB{1ICq zwm@uw*aEQyVhh9;h%FFXAhtkkf!G4E1y;3ySvHK#$jPujjpzR-bg7r`GK?RC9{~vx z;Lmp&#&h7)U>~>-Yy!W#!!RBN9|Io+hrtr4fCs@C*a>!k4dA8Q4dXoc6!;`4fN`)1 z+zr-)yTJK%hVd-;J$MRy3w#4S2~GkJjDs<-8EgdifVYF6+-4Z3!C|ln>fl~*;mgZ1D70t3GV zXTcA__rdqT$*RO;4x3yT9)}NIu$goHJ0ozpo&vgui?Qg~Js7*N6JU$B3|n6rH)r?2P(N ziJJOMk}MYLRvESNo}_$k-{>rfePX$bjvB^sNcV)SdyW!5Xy7-e5v9|#=vrlxrt617 zM?KSTX@g4qXOkrAI=&3bC(x%P`PLDECWHQUaUVjt)FLqpwOMAUY&A(0)AmJJg#>;; zQLevcP3hIcidKa>EoafRt+FYF>M9Eey4L(8Dtf}45A@>#%{s6%%L}tuAX|-wke>Gs zns)b)^jR=hBg##hz8rRKN(8y6j=>1iy4Z>t&oh_Dh3u?xdXg*@*Zr)x9_uT62VgW# zSEC$!+eHo%2I>C$RRtb3ZZIs(nXGH1fgFqjb5DIQr;wJu*^CHk+jySOLZMnoijkl&ETj!kUw=J72Fg23gIJ$kdnem{t7F%R&GCUrwr?6|+R`bXS{H-hXpedG68;Pb^=W z(kcZzZ&@7y&6T8P>a};~q~5c!Enw@Uqs?t(MEKHkmpg-;LF}ByT5YlZH__aFvEX{k zW1bMv#(1Yr*y?QLv&5}nQ*h+KmV(5-gS=oTPc%{~(S=3L)2xxyg>9sZj#1Cy?LDn8 z-t~^_qeYFsjYW+#G})-#6xe7O^0+4NdFtHDFV}D!V_#v*@qx)=3NS;9saDLb)&m*6 z3Qkd$hLe=<0Kpl`Icx&nUX7?dq=(c_bXRS`zEks1w89BV>cf8 z!fHQlsEk_Cp={SL*V(FIFrPXKL zzCJEgAKB!=6@iTr+)Mc8W4nHEYOGv14Xd?WZ-{DnRO}wIy}CMXFk=G2gGNDk-cFud zQN?pY=AceV4x2TzWXa{mDW-25C?9lp&Xq2)jNnYwceQ+Oh2_@;!M1C)(sn#ENQX7|N+)O(%=Cy9Y;_8v+*X~X>AwI@gHzxm;2>as|E=IHU>&#(T!x>230wp}0AB-N1rLGazyUUx2Ptp? zeg8Z78wle8*XR4jrD6-j7KkknTOhVTY=PJUH)jFu@314Fe2sC@^v`n2*MZ-mKH+$T z0q&?f6|E3DM{S;AmkN5YZx^iye=u~xIkPW&$Fp<~A)U2F|V zsMwZ@dPNJXsBScF5f+mi%YTPYVxh(kt4D~5xG+m(BC`|N-D|L7I*PC$yg9)ki9ST& z@SI95Mg;$xX4j@en616a8Z}y;ZUhfDn(+*!^cn)u(`Msg6kE^l(3)X5k>f`!^ClJm zTlkg!ECqcF76fKt^KnX$8hxU2U7HU!VIJ|lD}#6KPc5UNq?VmYPwT7hPrQuOkcaW? V8lp}r0VCQ7_(f)>T1MBb^>4m-J}dwL diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 283d59df4b..2468dca86c 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -113,7 +113,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { ret = TSDB_CODE_OUT_OF_MEMORY; goto END; } - taosLRUCacheSetStrictCapacity(idx->lru, true); + taosLRUCacheSetStrictCapacity(idx->lru, false); idx->tindex = idxTFileCreate(idx, path); if (idx->tindex == NULL) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 7bed059dfd..eadccba35f 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -31,7 +31,7 @@ typedef struct SIFParam { SHashObj *pFilter; SArray *result; - char * condValue; + char *condValue; SIdxFltStatus status; uint8_t colValType; @@ -45,7 +45,7 @@ typedef struct SIFParam { typedef struct SIFCtx { int32_t code; - SHashObj * pRes; /* element is SIFParam */ + SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; // SIdxFltStatus st; @@ -137,7 +137,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { // covert data From snode; SValueNode *vn = (SValueNode *)node; - char * pData = nodesGetValueFromNode(vn); + char *pData = nodesGetValueFromNode(vn); SDataType *pType = &vn->node.resType; int32_t type = pType->type; int32_t valLen = 0; @@ -175,7 +175,7 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SOperatorNode *nd = (SOperatorNode *)node; assert(nodeType(node) == QUERY_NODE_OPERATOR); SColumnNode *l = (SColumnNode *)nd->pLeft; - SValueNode * r = (SValueNode *)nd->pRight; + SValueNode *r = (SValueNode *)nd->pRight; param->colId = l->colId; param->colValType = l->node.resType.type; @@ -357,7 +357,7 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { int ret = 0; - SIndexMetaArg * arg = &output->arg; + SIndexMetaArg *arg = &output->arg; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); if (left->colValType == TSDB_DATA_TYPE_JSON) { @@ -749,7 +749,7 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SFilterInfo *filter = NULL; - SArray * output = taosArrayInit(8, sizeof(uint64_t)); + SArray *output = taosArrayInit(8, sizeof(uint64_t)); SIFParam param = {.arg = *metaArg, .result = output}; SIF_ERR_RET(sifCalculate((SNode *)pFilterNode, ¶m)); diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index c8023d51c9..f380833622 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -106,12 +106,6 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of blkLeft = kBlockSize - blkOffset; } while (len > 0); - -#ifdef USE_MMAP - int32_t last = ctx->file.size - offset; - nRead = last >= len ? len : last; - memcpy(buf, ctx->file.ptr + offset, nRead); -#endif return total; } static int idxFileCtxGetSize(IFileCtx* ctx) { @@ -146,8 +140,6 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); taosFtruncateFile(ctx->file.pFile, 0); taosStatFile(path, &ctx->file.size, NULL); - // ctx->file.size = (int)size; - } else { ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); @@ -166,6 +158,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity); ctx->mem.cap = capacity; } + ctx->write = idxFileCtxDoWrite; ctx->read = idxFileCtxDoRead; ctx->flush = idxFileCtxDoFlush; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5d087d5769..fc14a0f277 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -753,6 +753,8 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); if (exh == NULL) { tDebug("%" PRId64 " already release", refId); + destroyCmsg(pMsg); + return; } SCliConn* conn = exh->handle; From 9c4842ef103de33f4445225495418eaa448f633e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 17:30:48 +0800 Subject: [PATCH 08/94] feat: add lru to index --- source/libs/index/src/indexFstFile.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index f380833622..9f6168a2c1 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -79,13 +79,14 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of memcpy(buf + total, blk->buf + blkOffset, nread); taosLRUCacheRelease(ctx->lru, h, false); } else { - int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; - SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); + int32_t cacheMemSize = sizeof(SDataBlock) + kBlockSize; + SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize); blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); assert(blk->nread <= kBlockSize); nread = MIN(blkLeft, len); + if (blk->nread < kBlockSize && blk->nread < len) { break; } From 12b961c7f5905686b57434a6e1e41b0fa9881ebe Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 14 Jul 2022 18:42:26 +0800 Subject: [PATCH 09/94] enh: add schedule policy --- include/libs/scheduler/scheduler.h | 9 +------- source/client/src/clientImpl.c | 1 + source/libs/command/CMakeLists.txt | 4 ++-- source/libs/command/src/command.c | 2 ++ source/libs/qworker/inc/qwInt.h | 1 + source/libs/qworker/src/qwDbg.c | 13 ++++++----- source/libs/qworker/src/qwMsg.c | 2 +- source/libs/qworker/src/qworker.c | 2 ++ source/libs/scheduler/inc/schInt.h | 14 ++++++++++-- source/libs/scheduler/src/schJob.c | 26 ++++++++++++++-------- source/libs/scheduler/src/schTask.c | 31 ++++++++++++++++----------- source/libs/scheduler/src/scheduler.c | 2 +- source/libs/transport/src/transCli.c | 1 + source/util/src/terror.c | 2 +- 14 files changed, 66 insertions(+), 44 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 1b664d4ac5..e6973cd390 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -25,13 +25,6 @@ extern "C" { extern tsem_t schdRspSem; -typedef struct SSchedulerCfg { - uint32_t maxJobNum; - int32_t maxNodeTableNum; - SCH_POLICY schPolicy; - bool enableReSchedule; -} SSchedulerCfg; - typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed @@ -86,7 +79,7 @@ typedef struct SSchedulerReq { } SSchedulerReq; -int32_t schedulerInit(SSchedulerCfg *cfg); +int32_t schedulerInit(void); int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 32c11983c2..87b94af392 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -834,6 +834,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; + schedulerFreeJob(&pRequest->body.queryJob, 0); doAsyncQuery(pRequest, true); return; } diff --git a/source/libs/command/CMakeLists.txt b/source/libs/command/CMakeLists.txt index 51118f4a34..a890972d14 100644 --- a/source/libs/command/CMakeLists.txt +++ b/source/libs/command/CMakeLists.txt @@ -8,9 +8,9 @@ target_include_directories( target_link_libraries( command - PRIVATE os util nodes catalog function transport qcom + PRIVATE os util nodes catalog function transport qcom scheduler ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 94d19be0a4..d22b3d88b4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -17,6 +17,8 @@ #include "catalog.h" #include "tdatablock.h" #include "tglobal.h" +#include "commandInt.h" +#include "scheduler.h" extern SConfig* tsCfg; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 539643c390..d8d7c5a0ea 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -378,6 +378,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet); int32_t qwAddTaskCtx(QW_FPARAMS_DEF); +int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 869eedf8f6..fa63cf2c3a 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -9,7 +9,7 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = true}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = false}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -147,9 +147,9 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int return TSDB_CODE_SUCCESS; } -int32_t qwDbgResponseREdirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { +int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { if (gQWDebug.tmp) { - if (TDMT_SCH_QUERY == qwMsg->msgType) { + if (TDMT_SCH_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { SEpSet epSet = {0}; epSet.inUse = 1; epSet.numOfEps = 3; @@ -159,16 +159,15 @@ int32_t qwDbgResponseREdirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { epSet.eps[1].port = 7200; strcpy(epSet.eps[2].fqdn, "localhost"); epSet.eps[2].port = 7300; - + + ctx->phase = QW_PHASE_POST_QUERY; qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); - gQWDebug.tmp = false; return TSDB_CODE_SUCCESS; } - if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType) { + if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { ctx->phase = QW_PHASE_POST_QUERY; qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); - gQWDebug.tmp = false; return TSDB_CODE_SUCCESS; } } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index b937c44810..93268e1bcc 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -315,7 +315,7 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { int64_t rId = msg->refId; int32_t eId = msg->execId; - SQWMsg qwMsg = {.msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; + SQWMsg qwMsg = {.msgType = pMsg->msgType, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; QW_SCH_TASK_DLOG("prerocessQuery start, handle:%p", pMsg->info.handle); QW_ERR_RET(qwPreprocessQuery(QW_FPARAMS(), &qwMsg)); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b22729fb9c..1b58dc2824 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -488,6 +488,8 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); + qwDbgResponseRedirect(qwMsg, ctx); + _return: if (ctx) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index ef817a557c..7979db081b 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -135,6 +135,13 @@ typedef struct SSchStatusFps { schStatusEventFp eventFp; } SSchStatusFps; +typedef struct SSchedulerCfg { + uint32_t maxJobNum; + int32_t maxNodeTableNum; + SCH_POLICY schPolicy; + bool enableReSchedule; +} SSchedulerCfg; + typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId @@ -275,7 +282,7 @@ typedef struct SSchJob { extern SSchedulerMgmt schMgmt; -#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - (_task)->profile.execTime[(_task)->execId % (_task)->maxExecTimes]) > (_task)->timeoutUsec) +#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - *(int64_t*)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) @@ -479,9 +486,12 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); -int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum); +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); +int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); +int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); + extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e482814ee7..f1799b04c7 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -343,7 +343,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_ERR_JRET(schInitTask(pJob, pTask, plan, pLevel, levelNum)); + SCH_ERR_JRET(schInitTask(pJob, pTask, plan, pLevel)); SCH_ERR_JRET(schAppendJobDataSrc(pJob, pTask)); @@ -464,7 +464,7 @@ void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { } } -int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { schUpdateJobErrCode(pJob, errCode); int32_t code = atomic_load_32(&pJob->errCode); @@ -477,21 +477,29 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod SCH_RET(TSDB_CODE_SCH_IGNORE_ERROR); } -// Note: no more task error processing, handled in function internal -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { +int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode) { if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_FAIL, errCode); + schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, &errCode); return TSDB_CODE_SCH_IGNORE_ERROR; } -// Note: no more error processing, handled in function internal int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { - SCH_RET(schProcessOnJobFailureImpl(pJob, JOB_TASK_STATUS_DROP, errCode)); + SCH_RET(schProcessOnJobFailure(pJob, errCode)); } +int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { + if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { + return TSDB_CODE_SCH_IGNORE_ERROR; + } + + schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, &errCode); + return TSDB_CODE_SCH_IGNORE_ERROR; +} + + int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_EXEC); @@ -804,7 +812,7 @@ void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq* pReq, int } if (errCode) { - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void*)&errCode); + schHandleJobFailure(pJob, errCode); } SCH_JOB_DLOG("job end %s operation with code %s", schGetOpStr(type), tstrerror(errCode)); @@ -877,7 +885,7 @@ void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { } if (errCode) { - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void*)&errCode); + schHandleJobFailure(pJob, errCode); } if (pJob) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ebe4bcab7b..9226ea25c9 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -45,7 +45,7 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { taosArrayDestroy(pTask->profile.execTime); } -void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel, int32_t levelNum) { +void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) { if (SCH_IS_DATA_BIND_TASK(pTask) || (!SCH_IS_QUERY_JOB(pJob)) || (SCH_ALL != schMgmt.cfg.schPolicy)) { pTask->maxRetryTimes = SCH_MAX_CANDIDATE_EP_NUM; } else { @@ -53,10 +53,10 @@ void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel, in pTask->maxRetryTimes = TMAX(nodeNum, SCH_MAX_CANDIDATE_EP_NUM); } - pTask->maxExecTimes = pTask->maxRetryTimes * (levelNum - pLevel->level); + pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1); } -int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum) { +int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { int32_t code = 0; pTask->plan = pPlan; @@ -67,7 +67,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - schInitTaskRetryTimes(pJob, pTask, pLevel, levelNum); + schInitTaskRetryTimes(pJob, pTask, pLevel); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { @@ -76,6 +76,8 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); + SCH_TASK_DLOG("task initialized, max times %d:%d", pTask->maxRetryTimes, pTask->maxExecTimes); + return TSDB_CODE_SUCCESS; _return: @@ -118,7 +120,7 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_ } if (taosHashRemove(pTask->execNodes, &execId, sizeof(execId))) { - SCH_TASK_ELOG("fail to remove execId %d from execNodeList", execId); + SCH_TASK_DLOG("execId %d already not in execNodeList", execId); } else { SCH_TASK_DLOG("execId %d removed from execNodeList", execId); } @@ -248,7 +250,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } if (pTask->level->taskFailed > 0) { - SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, NULL)); + SCH_RET(schHandleJobFailure(pJob, pJob->errCode)); } else { SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } @@ -321,13 +323,17 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; - if ((pTask->execId + 1) >= pTask->maxExecTimes) { - SCH_TASK_DLOG("task no more retry since reach max try times, execId:%d", pTask->execId); - schSwitchJobStatus(pJob, JOB_TASK_STATUS_FAIL, (void *)&rspCode); - return TSDB_CODE_SUCCESS; + SCH_TASK_DLOG("task will be redirected now, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + + if (NULL == pData) { + pTask->retryTimes = 0; } - SCH_TASK_DLOG("task will be redirected now, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) { + SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, pTask->maxExecTimes, pTask->execId); + schHandleJobFailure(pJob, rspCode); + return TSDB_CODE_SUCCESS; + } schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); @@ -337,7 +343,6 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; - pTask->retryTimes = 0; memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); if (SCH_IS_DATA_BIND_TASK(pTask)) { @@ -784,7 +789,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { pTask->execId++; pTask->retryTimes++; - SCH_TASK_DLOG("start to launch task's %dth exec", pTask->execId); + SCH_TASK_DLOG("start to launch task, execId %d, retry %d", pTask->execId, pTask->retryTimes); SCH_LOG_TASK_START_TS(pTask); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 81c1ff9468..3a15523040 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -154,7 +154,7 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { return; } - schSwitchJobStatus(pJob, JOB_TASK_STATUS_DROP, (void*)&errCode); + schHandleJobDrop(pJob, errCode); schReleaseJob(*jobId); *jobId = 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5d087d5769..ac32f97258 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -753,6 +753,7 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); if (exh == NULL) { tDebug("%" PRId64 " already release", refId); + return; } SCliConn* conn = exh->handle; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2364c53a9a..95334b5535 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -135,7 +135,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_API_ERROR, "Stmt API usage error" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") // mnode-common From ca7ca2f1e33a86322890b87beb92e7ba37434d4a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 19:23:01 +0800 Subject: [PATCH 10/94] refactor code --- source/libs/index/src/indexCache.c | 18 +++++++++--------- source/libs/index/test/indexTests.cc | 11 +++++++---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 05ce418037..1e9ec4a841 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -462,8 +462,8 @@ Iterate* idxCacheIteratorCreate(IndexCache* cache) { if (cache->imm == NULL) { return NULL; } - Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); - if (iiter == NULL) { + Iterate* iter = taosMemoryCalloc(1, sizeof(Iterate)); + if (iter == NULL) { return NULL; } taosThreadMutexLock(&cache->mtx); @@ -471,15 +471,15 @@ Iterate* idxCacheIteratorCreate(IndexCache* cache) { idxMemRef(cache->imm); MemTable* tbl = cache->imm; - iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); - iiter->val.colVal = NULL; - iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; - iiter->next = idxCacheIteratorNext; - iiter->getValue = idxCacheIteratorGetValue; + iter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iter->val.colVal = NULL; + iter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; + iter->next = idxCacheIteratorNext; + iter->getValue = idxCacheIteratorGetValue; taosThreadMutexUnlock(&cache->mtx); - return iiter; + return iter; } void idxCacheIteratorDestroy(Iterate* iter) { if (iter == NULL) { @@ -564,13 +564,13 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { idxMemUnRef(tbl); taosThreadMutexUnlock(&pCache->mtx); - idxCacheUnRef(pCache); return 0; // encode end } void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; + idxCacheRef(pCache); taosThreadMutexLock(&pCache->mtx); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 3c1f148ae4..5b76de2ef8 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -292,8 +292,9 @@ class IndexEnv : public ::testing::Test { virtual void SetUp() { initLog(); taosRemoveDir(path); - SIndexOpts opts = {.cacheSize = 1024 * 1024 * 4}; - int ret = indexOpen(&opts, path, &index); + SIndexOpts opts; + opts.cacheSize = 1024 * 1024 * 4; + int ret = indexOpen(&opts, path, &index); assert(ret == 0); } virtual void TearDown() { indexClose(index); } @@ -697,8 +698,10 @@ class IndexObj { taosMkDir(dir.c_str()); } taosMkDir(dir.c_str()); - SIndexOpts opts = {.cacheSize = 1024 * 1024 * 4}; - int ret = indexOpen(&opts, dir.c_str(), &idx); + SIndexOpts opts; + opts.cacheSize = 1024 * 1024 * 4; + + int ret = indexOpen(&opts, dir.c_str(), &idx); if (ret != 0) { // opt std::cout << "failed to open index: %s" << dir << std::endl; From e01297d09dae23ed8a254461e16beb24e6f2f4e2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 14 Jul 2022 19:29:23 +0800 Subject: [PATCH 11/94] refactor(query): optimize table scan by reusing reader --- source/libs/executor/src/scanoperator.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f44ac48a4..f5b6b0c7a6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -516,10 +516,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); - tsdbReaderClose(pInfo->dataReader); - int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, - GET_TASKID(pTaskInfo)); + int32_t code; + if (pInfo->dataReader != NULL) { + code = tsdbReaderReset(pInfo->dataReader, &pInfo->cond); + ASSERT(code == 0); + } else { + code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, + GET_TASKID(pTaskInfo)); + ASSERT(code == 0); + } } SSDataBlock* result = doTableScanGroup(pOperator); @@ -871,6 +877,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } resetTableScanInfo(pInfo->pTableScanOp->info, &win); + pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1193,8 +1200,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } } - ASSERT(pInfo->pRes->pDataBlock != NULL); - // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, From 7e661a9e9ccfe26e8038297fde1774d3b37ff55f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 14 Jul 2022 20:02:21 +0800 Subject: [PATCH 12/94] feat(query): add avg function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 96 +++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 8b08785ed5..7c23184d93 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -101,6 +101,7 @@ int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t sumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t minScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t maxScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bc915132de..d0bd6917f2 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1999,6 +1999,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunction, + .sprocessFunc = avgScalarFunction, .finalizeFunc = avgFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 47ab4c614a..2de893380c 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1919,6 +1919,101 @@ int32_t maxScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * return doMinMaxScalarFunction(pInput, inputNum, pOutput, false); } +int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + int64_t count = 0, sum = 0; + bool hasNull = false; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + + switch(type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *in = (int8_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *in = (int16_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *in = (int32_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *in = (int64_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *in = (uint8_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *in = (uint16_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *in = (uint32_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *in = (uint64_t *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *in = (float *)pInputData->pData; + sum += in[i]; + count++; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *in = (double *)pInputData->pData; + sum += in[i]; + count++; + break; + } + } + } + + double *out = (double *)pOutputData->pData; + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + *out = (int64_t)sum / (double)count; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + *out = (uint64_t)sum / (double)count; + } else if (IS_FLOAT_TYPE(type)) { + *out = (double)sum / (double)count; + } + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -2031,3 +2126,4 @@ int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + From de495eb7ecbe653b976fca78b6c0119ead524212 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 14 Jul 2022 20:38:37 +0800 Subject: [PATCH 13/94] feat: add lru to index --- source/libs/index/src/indexFstFile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 9f6168a2c1..6036a06eaa 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -75,7 +75,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of if (h) { SDataBlock* blk = taosLRUCacheValue(ctx->lru, h); - nread = MIN(blkLeft, len); + nread = TMIN(blkLeft, len); memcpy(buf + total, blk->buf + blkOffset, nread); taosLRUCacheRelease(ctx->lru, h, false); } else { @@ -85,7 +85,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); assert(blk->nread <= kBlockSize); - nread = MIN(blkLeft, len); + nread = TMIN(blkLeft, len); if (blk->nread < kBlockSize && blk->nread < len) { break; From 39038de27f3f814f3cc531969afc696c0845f3a0 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 15 Jul 2022 01:28:27 +0000 Subject: [PATCH 14/94] update sysinfo.py --- tests/system-test/0-others/sysinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index d74c4f6db9..ae16c7d4db 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -44,9 +44,9 @@ class TDTestCase: tdSql.query('select server_status()') tdSql.checkData(0,0,1) tdDnodes.stoptaosd(1) + sleep(10) + tdSql.error('select server_status()') - tdSql.query('select server_status()') - print(tdSql.queryResult) def run(self): self.get_database_info() self.check_version() From eea76d8a7874b4f1cf2da9d3fc9c5ba6bd0029b8 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 15 Jul 2022 01:28:44 +0000 Subject: [PATCH 15/94] update --- tests/system-test/0-others/sysinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index ae16c7d4db..7027e1ef75 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -43,6 +43,7 @@ class TDTestCase: def get_server_status(self): tdSql.query('select server_status()') tdSql.checkData(0,0,1) + #!for bug tdDnodes.stoptaosd(1) sleep(10) tdSql.error('select server_status()') From 21a32b36e9368dbaf6e95db4b7a02a1dd2c4a2f2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 15 Jul 2022 09:32:54 +0800 Subject: [PATCH 16/94] fix: multi result function is not allowed when appear in clauses other than select --- source/libs/parser/src/parTranslater.c | 36 +++++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d108e86df1..be75b67d10 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1177,6 +1177,29 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p "%s is only supported in single table query", pFunc->functionName); } +static bool isStar(SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) && + (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); +} + +static bool isTableStar(SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) && + (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); +} + +static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsMultiResFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (SQL_CLAUSE_SELECT != pCxt->currClause ) { + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (isStar(pPara) || isTableStar(pPara)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s(*) is only supported in SELECTed list", pFunc->functionName); + } + } + return TSDB_CODE_SUCCESS; +} static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { if (NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt)) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; @@ -1311,6 +1334,9 @@ static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateRepeatScanFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateMultiResFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { setFuncClassification(pCxt->pCurrStmt, pFunc); } @@ -1892,16 +1918,6 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, bo return code; } -static bool isStar(SNode* pNode) { - return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) && - (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); -} - -static bool isTableStar(SNode* pNode) { - return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) && - (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); -} - static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrcParas, SNodeList** pOutput) { int32_t code = TSDB_CODE_SUCCESS; From ad383f3a6097dce1303154ce1a95934940d4f3b8 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 09:43:47 +0800 Subject: [PATCH 17/94] doc: move cache to internals and totally rewrite it --- docs/zh/21-tdinternal/03-cache.md | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/zh/21-tdinternal/03-cache.md diff --git a/docs/zh/21-tdinternal/03-cache.md b/docs/zh/21-tdinternal/03-cache.md new file mode 100644 index 0000000000..8009c3d712 --- /dev/null +++ b/docs/zh/21-tdinternal/03-cache.md @@ -0,0 +1,49 @@ +--- +sidebar_label: 缓存 +title: 缓存 +description: "TDengine 内部的缓存设计" +--- + +为了实现高效的写入和查询，TDengine 充分利用了各种缓存技术，本节将对 TDengine 中对缓存的使用做详细的说明。 + +## 写缓存 + +TDengine 采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine 充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 + +每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。 + +```sql +create database db0 vgroups 100 buffer 16 +``` + +理论上缓存越大越好，但超过一定阈值后再增加缓存对写入性能提升并无帮助，一般情况下使用默认值即可。 + +## 读缓存 + +在创建数据库时可以选择是否缓存该数据库中每个子表的最新数据。由参数 cachelast 设置，分为三种情况： +- 0: 不缓存 +- 1: 缓存子表最近一行数据，这将显著改善 last_row 函数的性能 +- 2: 缓存子表每一列最近的非 NULL 值，这将显著改善无特殊影响（比如 WHERE, ORDER BY, GROUP BY, INTERVAL）时的 last 函数的性能 +- 3: 同时缓存行和列，即等同于上述 cachelast 值为 1 或 2 时的行为同时生效 + +## 元数据缓存 + +为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。 + +```sql +create database db0 pages 128 pagesize 16kb +``` + +TBD: 元数据缓存中的替换策略需要补充 + +## 文件系统缓存 + +TDengine 利用 WAL 技术来提供基本的数据可靠性。写入 WAL 本质上是以顺序追加的方式写入磁盘文件。此时文件系统缓存在写入性能中也会扮演关键角色。在创建数据库时可以利用 wal 参数来选择性能优先或者可靠性优先。 +- 1: 写 WAL 但不执行 fsync ，新写入 WAL 的数据保存在文件系统缓存中但并未写入磁盘，这种方式性能优先 +- 2: 写 WAL 且执行 fsync，新写入 WAL 的数据被立即同步到磁盘上，可靠性更高 + +## 客户端缓存 + +为了进一步提升整个系统的处理效率，除了以上提到的服务端缓存技术之外，在 TDengine 的所有客户端都要调用的核心库 libtaos.so （也称为 taosc ）中也充分利用了缓存技术。在 taosc 中会缓存所访问过的各个数据库、超级表以及子表的元数据，集群的拓扑结构等关键元数据。 + +当有多个客户端同时访问 TDengine 集群，且其中一个客户端对某些元数据进行了修改的情况下，有可能会出现其它客户端所缓存的元数据不同步或失效的情况，此时需要在客户端执行 "reset query cache" 以让整个缓存失效从而强制重新拉取最新的元数据重新建立缓存。 From 31aa48e240effffc1283fea57ed05058a4645e69 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 09:44:29 +0800 Subject: [PATCH 18/94] fix: fix windows compile issue --- source/libs/scheduler/test/schedulerTests.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index cbfa8215dc..ca2122ed8f 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -649,7 +649,7 @@ TEST(queryTest, normalCase) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - int32_t code = schedulerInit(NULL); + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); schtBuildQueryDag(&dag); @@ -756,7 +756,7 @@ TEST(queryTest, readyFirstCase) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - int32_t code = schedulerInit(NULL); + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); schtBuildQueryDag(&dag); @@ -866,7 +866,7 @@ TEST(queryTest, flowCtrlCase) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - int32_t code = schedulerInit(NULL); + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); schtBuildQueryFlowCtrlDag(&dag); @@ -975,7 +975,7 @@ TEST(insertTest, normalCase) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - int32_t code = schedulerInit(NULL); + int32_t code = schedulerInit(); ASSERT_EQ(code, 0); schtBuildInsertDag(&dag); From 697ab1c716fd577afae0cf454a22506f428f35bb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 09:48:11 +0800 Subject: [PATCH 19/94] test: restore some cases --- tests/script/jenkins/basic.txt | 4 +- tests/script/tsim/compute/block_dist.sim | 9 +++-- tests/script/tsim/qnode/basic1.sim | 49 ++++++++++++------------ tests/script/tsim/snode/basic1.sim | 48 +++++++++++------------ 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index dd164f7640..9d16f5b2d3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -205,7 +205,7 @@ ./test.sh -f tsim/qnode/basic1.sim # ---- snode -# ./test.sh -f tsim/snode/basic1.sim +# unsupport ./test.sh -f tsim/snode/basic1.sim # ---- bnode ./test.sh -f tsim/bnode/basic1.sim @@ -382,7 +382,7 @@ # ---- compute ./test.sh -f tsim/compute/avg.sim -# jira ./test.sh -f tsim/compute/block_dist.sim +./test.sh -f tsim/compute/block_dist.sim ./test.sh -f tsim/compute/bottom.sim ./test.sh -f tsim/compute/count.sim ./test.sh -f tsim/compute/diff.sim diff --git a/tests/script/tsim/compute/block_dist.sim b/tests/script/tsim/compute/block_dist.sim index 73cbca84cf..37ad8d8cff 100644 --- a/tests/script/tsim/compute/block_dist.sim +++ b/tests/script/tsim/compute/block_dist.sim @@ -1,5 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start sql connect @@ -80,11 +81,11 @@ $nt = $ntPrefix . $i #sql select _block_dist() from $nt print show table distributed $nt -sql show table distributed $nt +sql_error show table distributed $nt -if $rows == 0 then - return -1 -endi +#if $rows == 0 then +# return -1 +#endi print ============== TD-5998 sql_error select _block_dist() from (select * from $nt) diff --git a/tests/script/tsim/qnode/basic1.sim b/tests/script/tsim/qnode/basic1.sim index 7108fcaf59..81d95c704d 100644 --- a/tests/script/tsim/qnode/basic1.sim +++ b/tests/script/tsim/qnode/basic1.sim @@ -6,32 +6,32 @@ system sh/exec.sh -n dnode2 -s start sql connect print =============== show dnodes -sql show dnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -sql show mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -print =============== create dnodes sql create dnode $hostname port 7200 -sleep 2000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== show dnodes sql show dnodes; if $rows != 2 then return -1 @@ -125,7 +125,6 @@ system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 2000 sql show qnodes if $rows != 2 then return -1 diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index a9d4867354..93bf04ef41 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -6,32 +6,32 @@ system sh/exec.sh -n dnode2 -s start sql connect print =============== show dnodes -sql show dnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -sql show mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -print =============== create dnodes sql create dnode $hostname port 7200 -sleep 2000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== show dnodes sql show dnodes; if $rows != 2 then return -1 From 4ff107812f59268bca979cc582b6175fa28a7ff7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 10:00:35 +0800 Subject: [PATCH 20/94] feat(query): add avg function scalar version TD-17344 --- source/libs/scalar/src/sclfunc.c | 46 ++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 2de893380c..c64b1d79ba 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1924,7 +1924,7 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * SColumnInfoData *pOutputData = pOutput->columnData; int32_t type = GET_PARAM_TYPE(pInput); - int64_t count = 0, sum = 0; + int64_t count = 0; bool hasNull = false; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -1935,78 +1935,90 @@ int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * switch(type) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *in = (int8_t *)pInputData->pData; - sum += in[i]; + int8_t *in = (int8_t *)pInputData->pData; + int64_t *out = (int64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t *in = (int16_t *)pInputData->pData; - sum += in[i]; + int64_t *out = (int64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_INT: { int32_t *in = (int32_t *)pInputData->pData; - sum += in[i]; + int64_t *out = (int64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_BIGINT: { int64_t *in = (int64_t *)pInputData->pData; - sum += in[i]; + int64_t *out = (int64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *in = (uint8_t *)pInputData->pData; - sum += in[i]; + uint8_t *in = (uint8_t *)pInputData->pData; + uint64_t *out = (uint64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_USMALLINT: { uint16_t *in = (uint16_t *)pInputData->pData; - sum += in[i]; + uint64_t *out = (uint64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_UINT: { uint32_t *in = (uint32_t *)pInputData->pData; - sum += in[i]; + uint64_t *out = (uint64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_UBIGINT: { uint64_t *in = (uint64_t *)pInputData->pData; - sum += in[i]; + uint64_t *out = (uint64_t *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_FLOAT: { float *in = (float *)pInputData->pData; - sum += in[i]; + float *out = (float *)pOutputData->pData; + *out += in[i]; count++; break; } case TSDB_DATA_TYPE_DOUBLE: { double *in = (double *)pInputData->pData; - sum += in[i]; + double *out = (double *)pOutputData->pData; + *out += in[i]; count++; break; } } } - double *out = (double *)pOutputData->pData; if (hasNull) { colDataAppendNULL(pOutputData, 0); } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { - *out = (int64_t)sum / (double)count; + int64_t *out = (int64_t *)pOutputData->pData; + *(double *)out = *out / (double)count; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - *out = (uint64_t)sum / (double)count; + uint64_t *out = (uint64_t *)pOutputData->pData; + *(double *)out = *out / (double)count; } else if (IS_FLOAT_TYPE(type)) { - *out = (double)sum / (double)count; + double *out = (double *)pOutputData->pData; + *(double *)out = *out / (double)count; } } From b728833b9438189d1282e0afdd758284945e650d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 10:09:21 +0800 Subject: [PATCH 21/94] test: restore some 2.0 cases --- tests/script/jenkins/basic.txt | 24 +++++++++++----------- tests/script/tsim/table/back_insert.sim | 1 + tests/script/tsim/table/delete_reuse1.sim | 24 ---------------------- tests/script/tsim/table/delete_reuse2.sim | 24 ---------------------- tests/script/tsim/table/delete_writing.sim | 24 ++++------------------ 5 files changed, 17 insertions(+), 80 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9d16f5b2d3..de148390a6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -235,9 +235,9 @@ ./test.sh -f tsim/table/createmulti.sim ./test.sh -f tsim/table/date.sim ./test.sh -f tsim/table/db.table.sim -# ./test.sh -f tsim/table/delete_reuse1.sim -# ./test.sh -f tsim/table/delete_reuse2.sim -# ./test.sh -f tsim/table/delete_writing.sim +./test.sh -f tsim/table/delete_reuse1.sim +./test.sh -f tsim/table/delete_reuse2.sim +./test.sh -f tsim/table/delete_writing.sim ./test.sh -f tsim/table/describe.sim ./test.sh -f tsim/table/double.sim ./test.sh -f tsim/table/float.sim @@ -314,12 +314,12 @@ ./test.sh -f tsim/db/basic3.sim -m ./test.sh -f tsim/db/error1.sim -m ./test.sh -f tsim/insert/backquote.sim -m -# ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m +# nojira ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/query/interval-offset.sim -m ./test.sh -f tsim/tmq/basic3.sim -m ./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/qnode/basic1.sim -m -#./test.sh -f tsim/mnode/basic1.sim -m +# nojira ./test.sh -f tsim/mnode/basic1.sim -m # --- sma ./test.sh -f tsim/sma/drop_sma.sim @@ -333,13 +333,13 @@ ./test.sh -f tsim/valgrind/checkError3.sim # --- vnode -# ./test.sh -f tsim/vnode/replica3_basic.sim -# ./test.sh -f tsim/vnode/replica3_repeat.sim -# ./test.sh -f tsim/vnode/replica3_vgroup.sim -# ./test.sh -f tsim/vnode/replica3_many.sim -# ./test.sh -f tsim/vnode/replica3_import.sim -# ./test.sh -f tsim/vnode/stable_balance_replica1.sim -# ./test.sh -f tsim/vnode/stable_dnode2_stop.sim +# unsupport ./test.sh -f tsim/vnode/replica3_basic.sim +# unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim +# unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim +# unsupport ./test.sh -f tsim/vnode/replica3_many.sim +# unsupport ./test.sh -f tsim/vnode/replica3_import.sim +# unsupport ./test.sh -f tsim/vnode/stable_balance_replica1.sim +# unsupport ./test.sh -f tsim/vnode/stable_dnode2_stop.sim ./test.sh -f tsim/vnode/stable_dnode2.sim ./test.sh -f tsim/vnode/stable_dnode3.sim ./test.sh -f tsim/vnode/stable_replica3_dnode6.sim diff --git a/tests/script/tsim/table/back_insert.sim b/tests/script/tsim/table/back_insert.sim index 43831cca95..06c9a91029 100644 --- a/tests/script/tsim/table/back_insert.sim +++ b/tests/script/tsim/table/back_insert.sim @@ -3,5 +3,6 @@ $x = 1 begin: sql insert into db.tb values(now, $x ) -x begin #print ===> insert successed $x + sleep 100 $x = $x + 1 goto begin \ No newline at end of file diff --git a/tests/script/tsim/table/delete_reuse1.sim b/tests/script/tsim/table/delete_reuse1.sim index db59425487..b9f08851ac 100644 --- a/tests/script/tsim/table/delete_reuse1.sim +++ b/tests/script/tsim/table/delete_reuse1.sim @@ -1,28 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode2 -c walLevel -v 1 -system sh/cfg.sh -n dnode3 -c walLevel -v 1 -system sh/cfg.sh -n dnode4 -c walLevel -v 1 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect print ======== step1 @@ -37,7 +15,6 @@ endi print ======== step2 sql drop table d1.t1 -sleep 1000 sql insert into d1.t1 values(now, 2) -x step2 return -1 step2: @@ -58,7 +35,6 @@ $x = 0 while $x < 20 sql drop table d1.t1 - sleep 1000 sql insert into d1.t1 values(now, -1) -x step4 return -1 step4: diff --git a/tests/script/tsim/table/delete_reuse2.sim b/tests/script/tsim/table/delete_reuse2.sim index f2784a3f5f..46a76027e5 100644 --- a/tests/script/tsim/table/delete_reuse2.sim +++ b/tests/script/tsim/table/delete_reuse2.sim @@ -1,28 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode2 -c walLevel -v 1 -system sh/cfg.sh -n dnode3 -c walLevel -v 1 -system sh/cfg.sh -n dnode4 -c walLevel -v 1 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect print ======== step1 @@ -37,7 +15,6 @@ endi print ======== step2 sql drop table db1.t1 -sleep 1000 sql_error insert into db1.t1 values(now, 2) print ========= step3 @@ -59,7 +36,6 @@ while $x < 20 $tb = tb . $x sql drop table $tb - sleep 1000 sql_error insert into $tb values(now, -1) step4: diff --git a/tests/script/tsim/table/delete_writing.sim b/tests/script/tsim/table/delete_writing.sim index 5351d13d80..df7c7f47e8 100644 --- a/tests/script/tsim/table/delete_writing.sim +++ b/tests/script/tsim/table/delete_writing.sim @@ -1,23 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode2 -c walLevel -v 1 -system sh/cfg.sh -n dnode3 -c walLevel -v 1 -system sh/cfg.sh -n dnode4 -c walLevel -v 1 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/exec.sh -n dnode1 -s start +sql connect print ========= start dnodes system sh/exec.sh -n dnode1 -s start @@ -28,12 +12,12 @@ sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1) print ======== start back -run_back general/table/back_insert.sim +run_back tsim/table/back_insert.sim sleep 1000 print ======== step1 $x = 1 -while $x < 15 +while $x < 10 print drop table times $x sql drop table db.tb -x step1 From b73d08c78f3fa17d666d97f5219455384870d8c4 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 15 Jul 2022 10:09:28 +0800 Subject: [PATCH 22/94] update --- tests/system-test/0-others/sysinfo.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 7027e1ef75..f0db151165 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -24,6 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db' + self.delaytime = 10 def get_database_info(self): tdSql.query('select database()') tdSql.checkData(0,0,None) @@ -44,14 +45,14 @@ class TDTestCase: tdSql.query('select server_status()') tdSql.checkData(0,0,1) #!for bug - tdDnodes.stoptaosd(1) - sleep(10) - tdSql.error('select server_status()') + # tdDnodes.stoptaosd(1) + # sleep(self.delaytime) + # tdSql.error('select server_status()') def run(self): self.get_database_info() self.check_version() - # self.get_server_status() + self.get_server_status() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 742fb1e594b3291c2e3249a93c695eab05095504 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 10:27:37 +0800 Subject: [PATCH 23/94] test: adjust some cases --- tests/script/jenkins/basic.txt | 21 +- tests/script/tsim/issue/TD-2677.sim | 111 ---- tests/script/tsim/issue/TD-2680.sim | 202 ------- tests/script/tsim/issue/TD-2713.sim | 145 ----- tests/script/tsim/issue/TD-3300.sim | 556 ------------------ tests/script/tsim/sync/3Replica1VgElect.sim | 63 +- tests/script/tsim/sync/3Replica5VgElect.sim | 63 +- .../tsim/sync/3Replica5VgElect3mnode.sim | 68 +-- .../tsim/sync/3Replica5VgElect3mnodedrop.sim | 65 +- tests/script/tsim/sync/electTest.sim | 63 +- tests/script/tsim/sync/oneReplica1VgElect.sim | 63 +- .../sync/oneReplica1VgElectWithInsert.sim | 63 +- tests/script/tsim/sync/oneReplica5VgElect.sim | 63 +- .../script/tsim/sync/threeReplica1VgElect.sim | 65 +- .../sync/threeReplica1VgElectWihtInsert.sim | 63 +- tests/script/tsim/sync/vnodesnapshot-test.sim | 63 +- tests/script/tsim/sync/vnodesnapshot.sim | 63 +- 17 files changed, 284 insertions(+), 1516 deletions(-) delete mode 100644 tests/script/tsim/issue/TD-2677.sim delete mode 100644 tests/script/tsim/issue/TD-2680.sim delete mode 100644 tests/script/tsim/issue/TD-2713.sim delete mode 100644 tests/script/tsim/issue/TD-3300.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index de148390a6..bffed86d75 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -350,7 +350,6 @@ ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim -# ./test.sh -f tsim/sync/3Replica5VgElect3mnode.sim # --- catalog ./test.sh -f tsim/catalog/alterInCurrent.sim @@ -433,12 +432,6 @@ # ---- wal ./test.sh -f tsim/wal/kill.sim -# ---- issue -#./test.sh -f tsim/issue/TD-2677.sim -#./test.sh -f tsim/issue/TD-2680.sim -#./test.sh -f tsim/issue/TD-2713.sim -#./test.sh -f tsim/issue/TD-3300.sim - # ---- tag ./test.sh -f tsim/tag/3.sim ./test.sh -f tsim/tag/4.sim @@ -451,18 +444,18 @@ ./test.sh -f tsim/tag/bool_binary.sim ./test.sh -f tsim/tag/bool_int.sim ./test.sh -f tsim/tag/bool.sim -#./test.sh -f tsim/tag/change.sim -#./test.sh -f tsim/tag/column.sim -#./test.sh -f tsim/tag/commit.sim -#./test.sh -f tsim/tag/create.sim -#./test.sh -f tsim/tag/delete.sim +# ./test.sh -f tsim/tag/change.sim +# ./test.sh -f tsim/tag/column.sim +# ./test.sh -f tsim/tag/commit.sim +# ./test.sh -f tsim/tag/create.sim +# ./test.sh -f tsim/tag/delete.sim # jira ./test.sh -f tsim/tag/double.sim -#./test.sh -f tsim/tag/filter.sim +# ./test.sh -f tsim/tag/filter.sim # jira ./test.sh -f tsim/tag/float.sim ./test.sh -f tsim/tag/int_binary.sim ./test.sh -f tsim/tag/int_float.sim ./test.sh -f tsim/tag/int.sim -#./test.sh -f tsim/tag/set.sim +# ./test.sh -f tsim/tag/set.sim ./test.sh -f tsim/tag/smallint.sim ./test.sh -f tsim/tag/tinyint.sim diff --git a/tests/script/tsim/issue/TD-2677.sim b/tests/script/tsim/issue/TD-2677.sim deleted file mode 100644 index 8d2058a385..0000000000 --- a/tests/script/tsim/issue/TD-2677.sim +++ /dev/null @@ -1,111 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 - -print ============== deploy - -system sh/exec.sh -n dnode1 -s start -sql connect - -sql create dnode $hostname2 -sql create dnode $hostname3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -print =============== step1 -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step1 -endi -if $data4_2 != ready then - goto step1 -endi -if $data4_3 != ready then - goto step1 -endi - -sql show mnodes -$mnode1Role = $data2_1 -print mnode1Role $mnode1Role -$mnode2Role = $data2_2 -print mnode2Role $mnode2Role -$mnode3Role = $data2_3 -print mnode3Role $mnode3Role - -if $mnode1Role != master then - goto step1 -endi -if $mnode2Role != slave then - goto step1 -endi -if $mnode3Role != slave then - goto step1 -endi - -$x = 1 -show2: - -print =============== step1 -sql create database d1 replica 2 quorum 2 -sql create table d1.t1 (ts timestamp, i int) -sql_error create table d1.t1 (ts timestamp, i int) -sql insert into d1.t1 values(now, 1) -sql select * from d1.t1; -if $rows != 1 then - return -1 -endi - -print =============== step2 -sql create database d2 replica 3 quorum 2 -sql create table d2.t1 (ts timestamp, i int) -sql_error create table d2.t1 (ts timestamp, i int) -sql insert into d2.t1 values(now, 1) -sql select * from d2.t1; -if $rows != 1 then - return -1 -endi - -print =============== step3 -sql create database d4 replica 1 quorum 1 -sql_error create database d5 replica 1 quorum 2 -sql_error create database d6 replica 1 quorum 3 -sql_error create database d7 replica 1 quorum 4 -sql_error create database d8 replica 1 quorum 0 -sql create database d9 replica 2 quorum 1 -sql create database d10 replica 2 quorum 2 -sql_error create database d11 replica 2 quorum 3 -sql_error create database d12 replica 2 quorum 4 -sql_error create database d12 replica 2 quorum 0 -sql create database d13 replica 3 quorum 1 -sql create database d14 replica 3 quorum 2 -sql_error create database d15 replica 3 quorum 3 -sql_error create database d16 replica 3 quorum 4 -sql_error create database d17 replica 3 quorum 0 - - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/issue/TD-2680.sim b/tests/script/tsim/issue/TD-2680.sim deleted file mode 100644 index 631332160f..0000000000 --- a/tests/script/tsim/issue/TD-2680.sim +++ /dev/null @@ -1,202 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 - -system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 - -system sh/cfg.sh -n dnode1 -c role -v 1 -system sh/cfg.sh -n dnode2 -c role -v 2 -system sh/cfg.sh -n dnode3 -c role -v 2 -system sh/cfg.sh -n dnode4 -c role -v 2 - -system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator - -print ============== step0 -system sh/exec_tarbitrator.sh -s start - -print ============== step1 -system sh/exec.sh -n dnode1 -s start -sql connect -sql create dnode $hostname2 -sql create dnode $hostname3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step1 -endi -if $data4_2 != ready then - goto step1 -endi -if $data4_3 != ready then - goto step1 -endi - -sql show mnodes -print mnode1 $data2_1 -print mnode1 $data2_2 -print mnode1 $data2_3 -if $data2_1 != master then - goto step1 -endi - -print ============== step2 -sql show dnodes -if $rows != 4 then - return -1 -endi - -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 -print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 - -if $data30 != 0 then - return -1 -endi - -if $data32 != 0 then - return -1 -endi - -if $data33 != 0 then - return -1 -endi - -if $data34 != ready then - return -1 -endi - -if $data35 != arb then - return -1 -endi - -if $data37 != - then - return -1 -endi - -print ============== step4 -system sh/exec_tarbitrator.sh -s stop - -$x = 0 -step4: - $x = $x + 1 - sleep 1000 - if $x == 20 then - return -1 - endi - -sql show dnodes -if $rows != 4 then - return -1 -endi - -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 -print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 - -if $data30 != 0 then - return -1 -endi - -if $data32 != 0 then - return -1 -endi - -if $data33 != 0 then - return -1 -endi - -if $data34 != offline then - goto step4 -endi - -if $data35 != arb then - return -1 -endi - -if $data37 != - then - return -1 -endi - -print ============== step5 -system sh/exec_tarbitrator.sh -s start - -$x = 0 -step5: - $x = $x + 1 - sleep 1000 - if $x == 20 then - return -1 - endi - -sql show dnodes -if $rows != 4 then - return -1 -endi - -print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 -print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 -print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 -print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 - -if $data30 != 0 then - return -1 -endi - -if $data32 != 0 then - return -1 -endi - -if $data33 != 0 then - return -1 -endi - -if $data34 != ready then - goto step5 -endi - -if $data35 != arb then - return -1 -endi - -if $data37 != - then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/issue/TD-2713.sim b/tests/script/tsim/issue/TD-2713.sim deleted file mode 100644 index b66c55b9b9..0000000000 --- a/tests/script/tsim/issue/TD-2713.sim +++ /dev/null @@ -1,145 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode2 -c wallevel -v 2 -system sh/cfg.sh -n dnode3 -c wallevel -v 2 -system sh/cfg.sh -n dnode4 -c wallevel -v 2 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 - -system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c slaveQuery -v 1 -system sh/cfg.sh -n dnode2 -c slaveQuery -v 1 -system sh/cfg.sh -n dnode3 -c slaveQuery -v 1 -system sh/cfg.sh -n dnode4 -c slaveQuery -v 1 - -print ========= step1 -system sh/exec.sh -n dnode1 -s start -sql connect -sql create dnode $hostname2 -sql create dnode $hostname3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step1 -endi -if $data4_2 != ready then - goto step1 -endi -if $data4_3 != ready then - goto step1 -endi - -sql show mnodes -print mnode1 $data2_1 -print mnode1 $data2_2 -print mnode1 $data2_3 -if $data2_1 != master then - goto step1 -endi -if $data2_2 != slave then - goto step1 -endi -if $data2_3 != slave then - goto step1 -endi - -print ========= step2 -sql create database d1 replica 3 -sql create table d1.t1 (ts timestamp, i int) -sql insert into d1.t1 values(now, 1) - -$x = 0 -step2: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show d1.vgroups -print online vgroups: $data03 -if $data03 != 3 then - goto step2 -endi -sleep 1000 - -print ========= step3 -$i = 0 -while $i < 100 - $i = $i + 1 - sql select * from d1.t1 - print d1.t1 rows: $rows - if $rows != 1 then - return -1 - endi -endw - -print ========= step4 -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT - -system rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/* -system rm -rf ../../../sim/dnode3/data/vnode/vnode2/version.json - -system sh/exec.sh -n dnode1 -s start -x SIGINT -system sh/exec.sh -n dnode2 -s start -x SIGINT -system sh/exec.sh -n dnode3 -s start -x SIGINT - -$x = 0 -step4: - $x = $x + 1 - sleep 1000 - if $x == 30 then - return -1 - endi - -sql show d1.vgroups -print online vgroups: $data03 -if $data03 != 3 then - goto step4 -endi -sleep 1000 - -print ========= step5 -$i = 0 -while $i < 100 - $i = $i + 1 - sql select * from d1.t1 - if $rows != 1 then - return -1 - endi - print d1.t1 rows: $rows -endw - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/issue/TD-3300.sim b/tests/script/tsim/issue/TD-3300.sim deleted file mode 100644 index 0745ceb849..0000000000 --- a/tests/script/tsim/issue/TD-3300.sim +++ /dev/null @@ -1,556 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 -system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 - -system sh/cfg.sh -n dnode1 -c role -v 1 -system sh/cfg.sh -n dnode2 -c role -v 2 -system sh/cfg.sh -n dnode3 -c role -v 2 -system sh/cfg.sh -n dnode4 -c role -v 2 - -system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator -system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator - -print ============== step0: start tarbitrator -system sh/exec_tarbitrator.sh -s start - -print ============== step1: start dnode1, only deploy mnode -system sh/exec.sh -n dnode1 -s start -sql connect - -print ============== step2: start dnode2/dnode3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -sql create dnode $hostname2 -sql create dnode $hostname3 - -$x = 0 -step2: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step2 -endi -if $data4_2 != ready then - goto step2 -endi -if $data4_3 != ready then - goto step2 -endi - -sleep 1000 - -print ============== step3 -sql create database db replica 2 -sql use db - -sql create table stb (ts timestamp, c1 int, c2 int) tags(t1 int) -sql create table t1 using stb tags(1) -sql insert into t1 values(1577980800000, 1, 5) -sql insert into t1 values(1577980800001, 2, 4) -sql insert into t1 values(1577980800002, 3, 3) -sql insert into t1 values(1577980800003, 4, 2) -sql insert into t1 values(1577980800004, 5, 1) - -sql show db.vgroups -if $data04 != 3 then - return -1 -endi -if $data06 != 2 then - return -1 -endi -if $data05 != master then - return -1 -endi -if $data07 != slave then - return -1 -endi - -sql select * from t1 -if $rows != 5 then - return -1 -endi - -system sh/exec.sh -n dnode2 -s stop -x SIGKILL -system sh/exec.sh -n dnode3 -s stop -x SIGKILL - -print ============== step4 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start - -$x = 0 -step4: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step4 -endi -if $data4_2 != ready then - goto step4 -endi -if $data4_3 != ready then - goto step4 -endi - -sql show db.vgroups -if $data04 != 3 then - goto step4 -endi -if $data06 != 2 then - goto step4 -endi -if $data05 != master then - goto step4 -endi -if $data07 != slave then - goto step4 -endi - -sql create table t2 using stb tags(1) -sql insert into t2 values(1577980800000, 1, 5) -sql insert into t2 values(1577980800001, 2, 4) -sql insert into t2 values(1577980800002, 3, 3) -sql insert into t2 values(1577980800003, 4, 2) -sql insert into t2 values(1577980800004, 5, 1) - -sql select * from t2 -if $rows != 5 then - return -1 -endi - -print ============== step5 -system sh/exec.sh -n dnode3 -s stop -x SIGKILL - -$x = 0 -step5: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step5 -endi -if $data4_2 != ready then - goto step5 -endi -if $data4_3 != offline then - goto step5 -endi - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi - -sql show db.vgroups -if $data04 != 3 then - goto step5 -endi -if $data06 != 2 then - goto step5 -endi -if $data05 != offline then - goto step5 -endi -if $data07 != master then - goto step5 -endi - -print ============== step6 -sql create table t3 using stb tags(1) -sql insert into t3 values(1577980800000, 1, 5) -sql insert into t3 values(1577980800001, 2, 4) -sql insert into t3 values(1577980800002, 3, 3) -sql insert into t3 values(1577980800003, 4, 2) -sql insert into t3 values(1577980800004, 5, 1) -sql insert into t3 values(1577980800010, 11, 5) -sql insert into t3 values(1577980800011, 12, 4) -sql insert into t3 values(1577980800012, 13, 3) -sql insert into t3 values(1577980800013, 14, 2) -sql insert into t3 values(1577980800014, 15, 1) - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi - -system sh/exec.sh -n dnode3 -s start - -$x = 0 -step6: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step6 -endi -if $data4_2 != ready then - goto step6 -endi -if $data4_3 != ready then - goto step6 -endi - -sql show db.vgroups -if $data04 != 3 then - goto step6 -endi -if $data06 != 2 then - goto step6 -endi -if $data05 != slave then - goto step6 -endi -if $data07 != master then - goto step6 -endi - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi - -print ============== step7 -sql create table t4 using stb tags(1) -sql insert into t4 values(1577980800000, 1, 5) -sql insert into t4 values(1577980800001, 2, 4) -sql insert into t4 values(1577980800002, 3, 3) -sql insert into t4 values(1577980800003, 4, 2) -sql insert into t4 values(1577980800004, 5, 1) -sql insert into t4 values(1577980800010, 11, 5) -sql insert into t4 values(1577980800011, 12, 4) -sql insert into t4 values(1577980800012, 13, 3) -sql insert into t4 values(1577980800013, 14, 2) -sql insert into t4 values(1577980800014, 15, 1) -sql insert into t4 values(1577980800020, 21, 5) -sql insert into t4 values(1577980800021, 22, 4) -sql insert into t4 values(1577980800022, 23, 3) -sql insert into t4 values(1577980800023, 24, 2) -sql insert into t4 values(1577980800024, 25, 1) - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi - -system sh/exec.sh -n dnode2 -s stop -x SIGKILL -$x = 0 -step7: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step7 -endi -if $data4_2 != offline then - goto step7 -endi -if $data4_3 != ready then - goto step7 -endi - -sql show db.vgroups -if $data04 != 3 then - goto step7 -endi -if $data06 != 2 then - goto step7 -endi -if $data05 != master then - goto step7 -endi -if $data07 != offline then - goto step7 -endi - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi - -print ============== step8 -sql create table t5 using stb tags(1) -sql insert into t5 values(1577980800000, 1, 5) -sql insert into t5 values(1577980800001, 2, 4) -sql insert into t5 values(1577980800002, 3, 3) -sql insert into t5 values(1577980800003, 4, 2) -sql insert into t5 values(1577980800004, 5, 1) -sql insert into t5 values(1577980800010, 11, 5) - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi -sql select * from t5 -if $rows != 6 then - return -1 -endi - -system sh/exec.sh -n dnode2 -s start -$x = 0 -step8: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step8 -endi -if $data4_2 != ready then - goto step8 -endi -if $data4_3 != ready then - goto step8 -endi - -sql show db.vgroups -if $data04 != 3 then - goto step8 -endi -if $data06 != 2 then - goto step8 -endi -if $data05 != master then - goto step8 -endi -if $data07 != slave then - goto step8 -endi - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi -sql select * from t5 -if $rows != 6 then - return -1 -endi - -print ============== step9 -sql create table t6 using stb tags(1) -sql insert into t6 values(1577980800000, 1, 5) -sql insert into t6 values(1577980800001, 2, 4) -sql insert into t6 values(1577980800002, 3, 3) -sql insert into t6 values(1577980800003, 4, 2) -sql insert into t6 values(1577980800004, 5, 1) -sql insert into t6 values(1577980800010, 11, 5) -sql insert into t6 values(1577980800011, 12, 4) - -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi -sql select * from t5 -if $rows != 6 then - return -1 -endi -sql select * from t6 -if $rows != 7 then - return -1 -endi - -system sh/exec.sh -n dnode3 -s stop -x SIGKILL -$x = 0 -step9: - $x = $x + 1 - sleep 1000 - if $x == 10 then - return -1 - endi - -sql show dnodes -print dnode1 $data4_1 -print dnode2 $data4_2 -print dnode3 $data4_3 - -if $data4_1 != ready then - goto step9 -endi -if $data4_2 != ready then - goto step9 -endi -if $data4_3 != offline then - goto step9 -endi - -print ============== 2 -sql show db.vgroups - -if $data04 != 3 then - goto step7 -endi -if $data06 != 2 then - goto step7 -endi -if $data05 != offline then - goto step7 -endi -if $data07 != master then - goto step7 -endi - -print ============== 3 -sql select * from t1 -if $rows != 5 then - return -1 -endi -sql select * from t2 -if $rows != 5 then - return -1 -endi -sql select * from t3 -if $rows != 10 then - return -1 -endi -sql select * from t4 -if $rows != 15 then - return -1 -endi -sql select * from t5 -if $rows != 6 then - return -1 -endi -sql select * from t6 -if $rows != 7 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -system sh/exec.sh -n dnode2 -s stop -system sh/exec.sh -n dnode3 -s stop diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index 6649b9c335..a451b1cba2 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index d96ad1655d..aec5666e6c 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnode.sim b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim index 8df2c2007d..8a69d5ca07 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnode.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim @@ -3,66 +3,44 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 - system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -### create clusters using four dnodes; - - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> 1-dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim index ae02a23c9b..d84679add1 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim @@ -11,57 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -print ===> create clusters using four dnodes; - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> 1-dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/electTest.sim b/tests/script/tsim/sync/electTest.sim index 5433434014..750ab9d63a 100644 --- a/tests/script/tsim/sync/electTest.sim +++ b/tests/script/tsim/sync/electTest.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim index cf8912e654..829cf029b4 100644 --- a/tests/script/tsim/sync/oneReplica1VgElect.sim +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $vgroups = 1 diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim index b816226cff..aff458c02a 100644 --- a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $vgroups = 1 diff --git a/tests/script/tsim/sync/oneReplica5VgElect.sim b/tests/script/tsim/sync/oneReplica5VgElect.sim index 71c26e17c7..2cf9b7f1f5 100644 --- a/tests/script/tsim/sync/oneReplica5VgElect.sim +++ b/tests/script/tsim/sync/oneReplica5VgElect.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 1 diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index c3e9c13793..bf2cb05570 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - -#sql connect +sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $vgroups = 1 diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index 3c21dff1b6..a6a0dda8a7 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $vgroups = 1 diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index 3cf8cb4d93..8589078b50 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index 347255b489..b9452d2147 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -11,55 +11,38 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][0] != 1 then - return -1 -endi -if $data[0][4] != ready then - goto check_dnode_ready -endi - sql connect sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 -check_dnode_ready_1: -$loop_cnt = $loop_cnt + 1 -sleep 200 -if $loop_cnt == 10 then - print ====> dnodes not ready! - return -1 -endi +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi sql show dnodes -print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] -print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] -print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] -if $data[0][4] != ready then - goto check_dnode_ready_1 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 endi -if $data[1][4] != ready then - goto check_dnode_ready_1 +if $data(1)[4] != ready then + goto step1 endi -if $data[2][4] != ready then - goto check_dnode_ready_1 +if $data(2)[4] != ready then + goto step1 endi -if $data[3][4] != ready then - goto check_dnode_ready_1 +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 endi $replica = 3 From 873ff96bc14bc0e7af5a66e3aee4f5909217c5a9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 11:01:19 +0800 Subject: [PATCH 24/94] fix: a problem of last_row function plan --- source/libs/parser/test/mockCatalog.cpp | 28 ++++++++++--------- .../libs/parser/test/mockCatalogService.cpp | 7 +++-- source/libs/parser/test/mockCatalogService.h | 2 +- source/libs/planner/src/planOptimizer.c | 12 ++++---- source/libs/planner/src/planSpliter.c | 3 +- source/libs/planner/test/planBasicTest.cpp | 18 ++++++++++++ tests/system-test/2-query/tsbsQuery.py | 2 +- 7 files changed, 48 insertions(+), 24 deletions(-) diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index d054a3434e..6c6a1f24f0 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -159,8 +159,8 @@ void generatePerformanceSchema(MockCatalogService* mcs) { * c4 | column | DOUBLE | 8 | * c5 | column | DOUBLE | 8 | */ -void generateTestTables(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) +void generateTestTables(MockCatalogService* mcs, const std::string& db) { + ITableBuilder& builder = mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) .setVgid(1) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) @@ -193,9 +193,9 @@ void generateTestTables(MockCatalogService* mcs) { * jtag | tag | json | -- | * Child Table: st2s1, st2s2 */ -void generateTestStables(MockCatalogService* mcs) { +void generateTestStables(MockCatalogService* mcs, const std::string& db) { { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 3) + ITableBuilder& builder = mcs->createTableBuilder(db, "st1", TSDB_SUPER_TABLE, 3, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT) @@ -204,20 +204,20 @@ void generateTestStables(MockCatalogService* mcs) { .addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) .addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP); builder.done(); - mcs->createSubTable("test", "st1", "st1s1", 1); - mcs->createSubTable("test", "st1", "st1s2", 2); - mcs->createSubTable("test", "st1", "st1s3", 1); + mcs->createSubTable(db, "st1", "st1s1", 1); + mcs->createSubTable(db, "st1", "st1s2", 2); + mcs->createSubTable(db, "st1", "st1s3", 1); } { - ITableBuilder& builder = mcs->createTableBuilder("test", "st2", TSDB_SUPER_TABLE, 3, 1) + ITableBuilder& builder = mcs->createTableBuilder(db, "st2", TSDB_SUPER_TABLE, 3, 1) .setPrecision(TSDB_TIME_PRECISION_MILLI) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT) .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) .addTag("jtag", TSDB_DATA_TYPE_JSON); builder.done(); - mcs->createSubTable("test", "st2", "st2s1", 1); - mcs->createSubTable("test", "st2", "st2s2", 2); + mcs->createSubTable(db, "st2", "st2s1", 1); + mcs->createSubTable(db, "st2", "st2s2", 2); } } @@ -237,6 +237,11 @@ void generateDatabases(MockCatalogService* mcs) { mcs->createDatabase(TSDB_INFORMATION_SCHEMA_DB); mcs->createDatabase(TSDB_PERFORMANCE_SCHEMA_DB); mcs->createDatabase("test"); + generateTestTables(g_mockCatalogService.get(), "test"); + generateTestStables(g_mockCatalogService.get(), "test"); + mcs->createDatabase("cache_db", false, 1); + generateTestTables(g_mockCatalogService.get(), "cache_db"); + generateTestStables(g_mockCatalogService.get(), "cache_db"); mcs->createDatabase("rollup_db", true); } @@ -369,11 +374,8 @@ void generateMetaData() { generateDatabases(g_mockCatalogService.get()); generateInformationSchema(g_mockCatalogService.get()); generatePerformanceSchema(g_mockCatalogService.get()); - generateTestTables(g_mockCatalogService.get()); - generateTestStables(g_mockCatalogService.get()); generateFunctions(g_mockCatalogService.get()); generateDnodes(g_mockCatalogService.get()); - g_mockCatalogService->showTables(); } void destroyMetaDataEnv() { g_mockCatalogService.reset(); } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 0f759018d9..5322e34c60 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -334,11 +334,12 @@ class MockCatalogServiceImpl { dnode_.insert(std::make_pair(dnodeId, epSet)); } - void createDatabase(const std::string& db, bool rollup) { + void createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { SDbCfgInfo cfg = {0}; if (rollup) { cfg.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention)); } + cfg.cacheLast = cacheLast; dbCfg_.insert(std::make_pair(db, cfg)); } @@ -627,7 +628,9 @@ void MockCatalogService::createDnode(int32_t dnodeId, const std::string& host, i impl_->createDnode(dnodeId, host, port); } -void MockCatalogService::createDatabase(const std::string& db, bool rollup) { impl_->createDatabase(db, rollup); } +void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { + impl_->createDatabase(db, rollup, cacheLast); +} int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { return impl_->catalogGetTableMeta(pTableName, pTableMeta); diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 5c8a8acad1..c1e926b08c 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -63,7 +63,7 @@ class MockCatalogService { void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize); void createSmaIndex(const SMCreateSmaReq* pReq); void createDnode(int32_t dnodeId, const std::string& host, int16_t port); - void createDatabase(const std::string& db, bool rollup = false); + void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0); int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 1eb807dfbe..97858f83fb 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2011,11 +2011,13 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic SNode* pNode = NULL; FOREACH(pNode, pAgg->pAggFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pNode; - int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row"); - pFunc->functionName[len] = '\0'; - int32_t code = fmGetFuncInfo(pFunc, NULL, 0); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { + int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row"); + pFunc->functionName[len] = '\0'; + int32_t code = fmGetFuncInfo(pFunc, NULL, 0); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } } pAgg->hasLastRow = false; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 871bcf015b..ae0ccb1c51 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -198,8 +198,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || - (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 0cff136349..2fe731a553 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -98,6 +98,24 @@ TEST_F(PlanBasicTest, interpFunc) { } TEST_F(PlanBasicTest, lastRowFunc) { + useDb("root", "cache_db"); + + run("SELECT LAST_ROW(c1) FROM t1"); + + run("SELECT LAST_ROW(*) FROM t1"); + + run("SELECT LAST_ROW(c1, c2) FROM t1"); + + run("SELECT LAST_ROW(c1), c2 FROM t1"); + + run("SELECT LAST_ROW(c1) FROM st1"); + + run("SELECT LAST_ROW(c1) FROM st1 PARTITION BY TBNAME"); + + run("SELECT LAST_ROW(c1), SUM(c3) FROM t1"); +} + +TEST_F(PlanBasicTest, lastRowFuncWithoutCache) { useDb("root", "test"); run("SELECT LAST_ROW(c1) FROM t1"); diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 8180f511e2..1e017bb39b 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -85,7 +85,7 @@ class TDTestCase: # tdSql.checkRows(10) # test partition interval Pseudo time-column - tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") # test From fd2dec280bf79e844fdb8aa90d3b1cf65adf0911 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 11:18:26 +0800 Subject: [PATCH 25/94] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 1 - source/dnode/vnode/src/tsdb/tsdbRead.c | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 5d3cfee592..66843d9a28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -177,7 +177,6 @@ int32_t tsdbRetrieveLastRow(void* pReader, SSDataBlock* pResBlock, const int32_t saveOneRow(pRow, pResBlock, pr, slotIds); taosArrayPush(pTableUidList, &pKeyInfo->uid); - // taosMemoryFree(pRow); tsdbCacheRelease(lruCache, h); pr->tableIndex += 1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1a9e12c9ca..f0aea0cefb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -830,9 +830,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - uint8_t *pb = NULL, *pb1 = NULL; int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols, - pBlockData, &pb, &pb1); + pBlockData, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3007,11 +3006,14 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { + tBlockDataClear(&pStatus->fileBlockData); + terrno = code; return NULL; } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); + tBlockDataClear(&pStatus->fileBlockData); return pReader->pResBlock->pDataBlock; } From b9a471624d49fb7289c0f00e9e11c7e4b62f7f68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 11:19:00 +0800 Subject: [PATCH 26/94] fix(query): handle multi group last_row query. --- source/libs/executor/inc/executorimpl.h | 12 +- source/libs/executor/src/cachescanoperator.c | 140 ++++++++++++++----- source/libs/executor/src/executorimpl.c | 85 +++++------ source/libs/function/src/builtins.c | 2 +- 4 files changed, 160 insertions(+), 79 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 84d503b972..1bf6ca8dab 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -317,12 +317,16 @@ typedef struct STagScanInfo { typedef struct SLastrowScanInfo { SSDataBlock *pRes; - SArray *pTableList; SReadHandle readHandle; void *pLastrowReader; SArray *pColMatchInfo; int32_t *pSlotIds; SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock *pBufferredRes; + SArray *pUidList; + int32_t indexOfBufferedRes; } SLastrowScanInfo; typedef enum EStreamScanMode { @@ -826,8 +830,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SArray* pTableList, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -945,8 +948,9 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, +int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 9034397d0f..78a7e58ee1 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -30,15 +30,13 @@ static SSDataBlock* doScanLastrow(SOperatorInfo* pOperator); static void destroyLastrowScanOperator(void* param, int32_t numOfOutput); static int32_t extractTargetSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); -SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SArray* pTableList, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pTableList = pTableList; pInfo->readHandle = *readHandle; pInfo->pRes = createResDataBlock(pScanNode->scan.node.pOutputDataBlockDesc); @@ -50,8 +48,22 @@ SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SRead goto _error; } - tsdbLastRowReaderOpen(readHandle->vnode, LASTROW_RETRIEVE_TYPE_SINGLE, pTableList, taosArrayGetSize(pInfo->pColMatchInfo), - &pInfo->pLastrowReader); + STableListInfo* pTableList = &pTaskInfo->tableqinfoList; + + initResultSizeInfo(pOperator, 1024); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); + + // partition by tbname + if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) { + pInfo->retrieveType = LASTROW_RETRIEVE_TYPE_ALL; + tsdbLastRowReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList, + taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader); + pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false); + blockDataEnsureCapacity(pInfo->pBufferredRes, pOperator->resultInfo.capacity); + } else { // by tags + pInfo->retrieveType = LASTROW_RETRIEVE_TYPE_SINGLE; + } if (pScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* pPseudoExpr = &pInfo->pseudoExprSup; @@ -60,19 +72,17 @@ SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pScanNode, SRead pPseudoExpr->pCtx = createSqlFunctionCtx(pPseudoExpr->pExprInfo, pPseudoExpr->numOfExprs, &pPseudoExpr->rowEntryInfoOffset); } - pOperator->name = "LastrowScanOperator"; + pOperator->name = "LastrowScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - initResultSizeInfo(pOperator, 1024); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doScanLastrow, NULL, NULL, destroyLastrowScanOperator, NULL, NULL, NULL); + pOperator->cost.openCost = 0; return pOperator; @@ -90,43 +100,105 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { SLastrowScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t size = taosArrayGetSize(pInfo->pTableList); + STableListInfo* pTableList = &pTaskInfo->tableqinfoList; + int32_t size = taosArrayGetSize(pTableList->pTableList); if (size == 0) { - setTaskStatus(pTaskInfo, TASK_COMPLETED); + doSetOperatorCompleted(pOperator); return NULL; } + blockDataCleanup(pInfo->pRes); + // check if it is a group by tbname - if (size == taosArrayGetSize(pInfo->pTableList)) { - blockDataCleanup(pInfo->pRes); - SArray* pUidList = taosArrayInit(1, sizeof(tb_uid_t)); - int32_t code = tsdbRetrieveLastRow(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pUidList); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); + if (pInfo->retrieveType == LASTROW_RETRIEVE_TYPE_ALL) { + if (pInfo->indexOfBufferedRes >= pInfo->pBufferredRes->info.rows) { + blockDataCleanup(pInfo->pBufferredRes); + taosArrayClear(pInfo->pUidList); + + int32_t code = tsdbRetrieveLastRow(pInfo->pLastrowReader, pInfo->pBufferredRes, pInfo->pSlotIds, pInfo->pUidList); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + // check for tag values + int32_t resultRows = pInfo->pBufferredRes->info.rows; + ASSERT(resultRows == taosArrayGetSize(pInfo->pUidList)); + pInfo->indexOfBufferedRes = 0; } - // check for tag values - if (pInfo->pRes->info.rows > 0 && pInfo->pseudoExprSup.numOfExprs > 0) { - SExprSupp* pSup = &pInfo->pseudoExprSup; - pInfo->pRes->info.uid = *(tb_uid_t*) taosArrayGet(pUidList, 0); - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) { + for(int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { + SColMatchInfo* pMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + int32_t slotId = pMatchInfo->targetSlotId; + + SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId); + SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, slotId); + + char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); + bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes); + colDataAppend(pDst, 0, p, isNull); + } + + if (pInfo->pseudoExprSup.numOfExprs > 0) { + SExprSupp* pSup = &pInfo->pseudoExprSup; + addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + GET_TASKID(pTaskInfo)); + } + + pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); + pInfo->pRes->info.groupId = *groupId; + + pInfo->indexOfBufferedRes += 1; + pInfo->pRes->info.rows = 1; + return pInfo->pRes; + } else { + doSetOperatorCompleted(pOperator); + return NULL; + } + } else { + size_t totalGroups = taosArrayGetSize(pTableList->pGroupList); + + while (pInfo->currentGroupIndex < totalGroups) { + SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex); + + tsdbLastRowReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList, + taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader); + taosArrayClear(pInfo->pUidList); + + int32_t code = tsdbRetrieveLastRow(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + pInfo->currentGroupIndex += 1; + + // check for tag values + if (pInfo->pRes->info.rows > 0) { + if (pInfo->pseudoExprSup.numOfExprs > 0) { + SExprSupp* pSup = &pInfo->pseudoExprSup; + pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); + + STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0); + pInfo->pRes->info.groupId = pKeyInfo->groupId; + + addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + GET_TASKID(pTaskInfo)); + } + + tsdbLastrowReaderClose(pInfo->pLastrowReader); + return pInfo->pRes; + } } doSetOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - // todo fetch the result for each group + return NULL; } - - return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } void destroyLastrowScanOperator(void* param, int32_t numOfOutput) { SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; blockDataDestroy(pInfo->pRes); - tsdbLastrowReaderClose(pInfo->pLastrowReader); - taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 89542571ea..cecd80123b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -516,6 +516,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt // NOTE: the last parameter is the primary timestamp column if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { pInput->pPTS = pInput->pData[j]; + ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { @@ -4291,6 +4292,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); @@ -4309,6 +4311,30 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, return TDB_CODE_SUCCESS; } +static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { + memset(pCond, 0, sizeof(SQueryTableDataCond)); + + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + if (pCond->colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return terrno; + } + + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); + + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) { @@ -4318,7 +4344,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, + pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); if (code) { pTaskInfo->code = code; return NULL; @@ -4337,7 +4364,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, + pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); if (code) { pTaskInfo->code = code; return NULL; @@ -4366,7 +4394,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .maxTs = INT64_MIN, }; if (pHandle) { - int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, + pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); if (code) { pTaskInfo->code = code; return NULL; @@ -4406,25 +4435,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SQueryTableDataCond cond = {0}; - - { - cond.order = TSDB_ORDER_ASC; - cond.numOfCols = 1; - cond.colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - cond.colList->colId = 1; - cond.colList->type = TSDB_DATA_TYPE_TIMESTAMP; - cond.colList->bytes = sizeof(TSKEY); - - cond.twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - cond.suid = pBlockNode->suid; - cond.type = BLOCK_LOAD_OFFSET_ORDER; - cond.startVersion = -1; - cond.endVersion = -1; + int32_t code = initTableblockDistQueryCond(pBlockNode->suid, &cond); + if (code != TSDB_CODE_SUCCESS) { + return NULL; } STsdbReader* pReader = NULL; @@ -4435,31 +4448,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; - // int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - // if (code) { - // pTaskInfo->code = code; - // return NULL; - // } - - int32_t code = extractTableSchemaInfo(pHandle, pScanNode->scan.uid, pTaskInfo); + int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, + queryId, taskId); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; } - pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); - if (pScanNode->scan.tableType == TSDB_SUPER_TABLE) { - code = vnodeGetAllTableList(pHandle->vnode, pScanNode->scan.uid, pTableListInfo->pTableList); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; - return NULL; - } - } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pScanNode->scan.uid, .groupId = 0}; - taosArrayPush(pTableListInfo->pTableList, &info); + code = extractTableSchemaInfo(pHandle, pScanNode->scan.uid, pTaskInfo); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; } - return createLastrowScanOperator(pScanNode, pHandle, pTableListInfo->pTableList, pTaskInfo); + return createLastrowScanOperator(pScanNode, pHandle, pTaskInfo); } else { ASSERT(0); } @@ -4928,6 +4930,9 @@ static void doDestroyTableList(STableListInfo* pTableqinfoList) { if (pTableqinfoList->needSortTableByGroupId) { for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) { SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); + if (tmp == pTableqinfoList->pTableList) { + continue; + } taosArrayDestroy(tmp); } } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7b784e9f32..f6352f421a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2222,7 +2222,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_cache_last_row", .type = FUNCTION_TYPE_CACHE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, From b4644963af3344051b60e5b6f17350b0885fe08e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 11:21:58 +0800 Subject: [PATCH 27/94] test: restore some 2.0 cases --- tests/script/jenkins/basic.txt | 10 ++-- tests/script/tsim/parser/alter.sim | 47 +++++++++---------- tests/script/tsim/parser/alter1.sim | 7 +-- .../parser/alter__for_community_version.sim | 37 +++++---------- tests/script/tsim/parser/alter_column.sim | 42 ++--------------- tests/script/tsim/parser/alter_stable.sim | 19 ++++---- 6 files changed, 52 insertions(+), 110 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bffed86d75..7ccfe93c11 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -82,11 +82,11 @@ ./test.sh -f tsim/insert/update0.sim # ---- parser -# ./test.sh -f tsim/parser/alter.sim -# ./test.sh -f tsim/parser/alter1.sim -## ./test.sh -f tsim/parser/alter__for_community_version.sim -## ./test.sh -f tsim/parser/alter_column.sim -# ./test.sh -f tsim/parser/alter_stable.sim +./test.sh -f tsim/parser/alter.sim +# nojira ./test.sh -f tsim/parser/alter1.sim +./test.sh -f tsim/parser/alter__for_community_version.sim +./test.sh -f tsim/parser/alter_column.sim +./test.sh -f tsim/parser/alter_stable.sim # ./test.sh -f tsim/parser/auto_create_tb.sim # ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim # ./test.sh -f tsim/parser/between_and.sim diff --git a/tests/script/tsim/parser/alter.sim b/tests/script/tsim/parser/alter.sim index 78c1a3029a..499d4d302b 100644 --- a/tests/script/tsim/parser/alter.sim +++ b/tests/script/tsim/parser/alter.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_alt_db @@ -40,62 +37,61 @@ sql_error alter database $db keep 20,20,20,20 sql_error alter database $db keep 365001,365001,365001 sql alter database $db keep 21 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 21,21,21 then +if $data27 != 30240m,30240m,30240m then return -1 endi sql alter database $db keep 11,12 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 11,12,12 then +if $data27 != 15840m,17280m,17280m then return -1 endi sql alter database $db keep 20,20,20 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 20,20,20 then +if $data27 != 28800m,28800m,28800m then return -1 endi sql alter database $db keep 10,10,10 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 10,10,10 then +if $data27 != 14400m,14400m,14400m then return -1 endi sql alter database $db keep 10,10,11 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 10,10,11 then +if $data27 != 14400m,14400m,15840m then return -1 endi sql alter database $db keep 11,12,13 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 11,12,13 then +if $data27 != 15840m,17280m,18720m then return -1 endi sql alter database $db keep 365000,365000,365000 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 365000,365000,365000 then +if $data27 != 525600000m,525600000m,525600000m then return -1 endi - ##### alter table test, simeplest case sql create table tb (ts timestamp, c1 int, c2 int, c3 int) sql insert into tb values (now, 1, 1, 1) @@ -187,7 +183,6 @@ endi sql drop table tb sql drop table mt -sleep 100 ### ALTER TABLE WHILE STREAMING [TBASE271] #sql create table tb1 (ts timestamp, c1 int, c2 nchar(5), c3 int) #sql create table strm as select count(*), avg(c1), first(c2), sum(c3) from tb1 interval(2s) @@ -195,9 +190,9 @@ sleep 100 #if $rows != 0 then # return -1 #endi -##sleep 12000 + #sql insert into tb1 values (now, 1, 'taos', 1) -#sleep 20000 + #sql select * from strm #print rows = $rows #if $rows != 1 then @@ -207,9 +202,9 @@ sleep 100 # return -1 #endi #sql alter table tb1 drop column c3 -#sleep 500 + #sql insert into tb1 values (now, 2, 'taos') -#sleep 30000 + #sql select * from strm #if $rows != 2 then # return -1 @@ -218,9 +213,9 @@ sleep 100 # return -1 #endi #sql alter table tb1 add column c3 int -#sleep 500 + #sql insert into tb1 values (now, 3, 'taos', 3); -#sleep 100 + #sql select * from strm #if $rows != 3 then # return -1 @@ -259,7 +254,7 @@ sql create database $db sql use $db sql create table mt (ts timestamp, c1 int, c2 nchar(7), c3 int) tags (t1 int) sql create table tb using mt tags(1) -sleep 100 + sql insert into tb values ('2018-11-01 16:30:00.000', 1, 'insert', 1) sql alter table mt drop column c3 diff --git a/tests/script/tsim/parser/alter1.sim b/tests/script/tsim/parser/alter1.sim index 3b6b0d9465..b01e98a834 100644 --- a/tests/script/tsim/parser/alter1.sim +++ b/tests/script/tsim/parser/alter1.sim @@ -1,11 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect -sql reset query cache $dbPrefix = alt1_db @@ -87,9 +83,8 @@ if $data13 != NULL then return -1 endi -sleep 100 print ================== insert values into table -sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2,) car2 values (now, 1,3,3) +sql insert into car1 values (now, 1, 1,1 ) (now +1s, 2,2,2) car2 values (now, 1,3,3) sql select c1+speed from stb where c1 > 0 if $rows != 3 then diff --git a/tests/script/tsim/parser/alter__for_community_version.sim b/tests/script/tsim/parser/alter__for_community_version.sim index 7a9a970822..9a960e21c2 100644 --- a/tests/script/tsim/parser/alter__for_community_version.sim +++ b/tests/script/tsim/parser/alter__for_community_version.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_alt_db @@ -23,10 +20,10 @@ sql drop database if exists $db sql create database $db duration 10 keep 20 sql use $db sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 20 then +if $data27 != 28800m,28800m,28800m then return -1 endi @@ -47,44 +44,44 @@ sql_error alter database $db keep 20,19,18 sql_error alter database $db keep 20,20,20,20 sql_error alter database $db keep 365001,365001,365001 sql_error alter database $db keep 365001 -sql alter database $db keep 20 +sql_error alter database $db keep 20 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 20 then +if $data27 != 28800m,28800m,28800m then return -1 endi sql alter database $db keep 10 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 10 then +if $data27 != 14400m,14400m,14400m then return -1 endi sql alter database $db keep 11 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 11 then +if $data27 != 15840m,15840m,15840m then return -1 endi sql alter database $db keep 13 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 13 then +if $data27 != 18720m,18720m,18720m then return -1 endi sql alter database $db keep 365000 sql show databases -if $rows != 1 then +if $rows != 3 then return -1 endi -if $data07 != 365000 then +if $data27 != 525600000m,525600000m,525600000m then return -1 endi @@ -180,7 +177,6 @@ endi sql drop table tb sql drop table mt -sleep 100 ### ALTER TABLE WHILE STREAMING [TBASE271] #sql create table tb1 (ts timestamp, c1 int, c2 nchar(5), c3 int) #sql create table strm as select count(*), avg(c1), first(c2), sum(c3) from tb1 interval(2s) @@ -188,9 +184,7 @@ sleep 100 #if $rows != 0 then # return -1 #endi -##sleep 12000 #sql insert into tb1 values (now, 1, 'taos', 1) -#sleep 20000 #sql select * from strm #print rows = $rows #if $rows != 1 then @@ -200,9 +194,7 @@ sleep 100 # return -1 #endi #sql alter table tb1 drop column c3 -#sleep 500 #sql insert into tb1 values (now, 2, 'taos') -#sleep 30000 #sql select * from strm #if $rows != 2 then # return -1 @@ -211,9 +203,7 @@ sleep 100 # return -1 #endi #sql alter table tb1 add column c3 int -#sleep 500 #sql insert into tb1 values (now, 3, 'taos', 3); -#sleep 100 #sql select * from strm #if $rows != 3 then # return -1 @@ -252,7 +242,6 @@ sql create database $db sql use $db sql create table mt (ts timestamp, c1 int, c2 nchar(7), c3 int) tags (t1 int) sql create table tb using mt tags(1) -sleep 100 sql insert into tb values ('2018-11-01 16:30:00.000', 1, 'insert', 1) sql alter table mt drop column c3 diff --git a/tests/script/tsim/parser/alter_column.sim b/tests/script/tsim/parser/alter_column.sim index fe109352d1..3b6f0e4da7 100644 --- a/tests/script/tsim/parser/alter_column.sim +++ b/tests/script/tsim/parser/alter_column.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_alt_db @@ -26,51 +23,20 @@ sql use $db sql create table tb (ts timestamp, c1 int, c2 binary(10), c3 nchar(10)) sql insert into tb values (now, 1, "1", "1") sql alter table tb modify column c2 binary(20); -if $rows != 0 then - return -1 -endi sql alter table tb modify column c3 nchar(20); -if $rows != 0 then - return -1 -endi sql create stable stb (ts timestamp, c1 int, c2 binary(10), c3 nchar(10)) tags(id1 int, id2 binary(10), id3 nchar(10)) sql create table tb1 using stb tags(1, "a", "b") sql insert into tb1 values (now, 1, "1", "1") sql alter stable stb modify column c2 binary(20); -if $rows != 0 then - return -1 -endi sql alter table stb modify column c2 binary(30); -if $rows != 0 then - return -1 -endi sql alter stable stb modify column c3 nchar(20); -if $rows != 0 then - return -1 -endi sql alter table stb modify column c3 nchar(30); -if $rows != 0 then - return -1 -endi - sql alter table stb modify tag id2 binary(11); -if $rows != 0 then - return -1 -endi -sql alter stable stb modify tag id2 binary(11); -if $rows != 0 then - return -1 -endi +sql_error alter stable stb modify tag id2 binary(11); sql alter table stb modify tag id3 nchar(11); -if $rows != 0 then - return -1 -endi -sql alter stable stb modify tag id3 nchar(11); -if $rows != 0 then - return -1 -endi +sql_error alter stable stb modify tag id3 nchar(11); ##### ILLEGAL OPERATIONS @@ -82,14 +48,14 @@ sql_error alter table tb modify column c2 binary(10); sql_error alter table tb modify column c2 binary(9); sql_error alter table tb modify column c2 binary(-9); sql_error alter table tb modify column c2 binary(0); -sql_error alter table tb modify column c2 binary(17000); +sql alter table tb modify column c2 binary(17000); sql_error alter table tb modify column c2 nchar(30); sql_error alter table tb modify column c3 double; sql_error alter table tb modify column c3 nchar(10); sql_error alter table tb modify column c3 nchar(0); sql_error alter table tb modify column c3 nchar(-1); sql_error alter table tb modify column c3 binary(80); -sql_error alter table tb modify column c3 nchar(17000); +sql alter table tb modify column c3 nchar(17000); sql_error alter table tb modify column c3 nchar(100), c2 binary(30); sql_error alter table tb modify column c1 nchar(100), c2 binary(30); sql_error alter stable tb modify column c2 binary(30); diff --git a/tests/script/tsim/parser/alter_stable.sim b/tests/script/tsim/parser/alter_stable.sim index 1406af6087..8659d92f7e 100644 --- a/tests/script/tsim/parser/alter_stable.sim +++ b/tests/script/tsim/parser/alter_stable.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect print ========== alter_stable.sim @@ -13,19 +10,19 @@ sql drop database if exists $db sql create database $db sql use $db -##### alter stable test : change tag name -# case-1 change tag name: new name inclue old name +##### alter stable test : rename tag name +# case-1 rename tag name: new name inclue old name sql create table mt1 (ts timestamp, c1 int) tags (a int) -sql alter table mt1 change tag a abcd -sql alter table mt1 change tag abcd a -sql_error alter table mt1 change tag a 1 +sql alter table mt1 rename tag a abcd +sql alter table mt1 rename tag abcd a +sql_error alter table mt1 rename tag a 1 sql_error create table mtx1 (ts timestamp, c1 int) tags (123 int) sql_error create table mt2 (ts timestamp, c1 int) tags (abc012345678901234567890123456789012345678901234567890123456789def int) sql create table mt3 (ts timestamp, c1 int) tags (abc012345678901234567890123456789012345678901234567890123456789 int) -sql_error alter table mt3 change tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg012345678901234567890123456789012345678901234567890123456789 -sql alter table mt3 change tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg0123456789012345678901234567890123456789 +sql_error alter table mt3 rename tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg012345678901234567890123456789012345678901234567890123456789 +sql alter table mt3 rename tag abc012345678901234567890123456789012345678901234567890123456789 abcdefg0123456789012345678901234567890123456789 # case-2 set tag value sql create table mt4 (ts timestamp, c1 int) tags (name binary(16), len int) @@ -37,7 +34,7 @@ sql alter table tb1 set tag len = 379 # case TD-5594 sql create stable st5520(ts timestamp, f int) tags(t0 bool, t1 nchar(4093), t2 nchar(1)) -sql_error alter stable st5520 modify tag t2 nchar(2); +sql alter stable st5520 modify tag t2 nchar(2); # test end sql drop database $db From 72f4a4afbb8eab30f571f479a08e4733019348e5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 11:27:33 +0800 Subject: [PATCH 28/94] test: comment out case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 7ccfe93c11..b5cb7ddb79 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -381,7 +381,7 @@ # ---- compute ./test.sh -f tsim/compute/avg.sim -./test.sh -f tsim/compute/block_dist.sim +# jira ./test.sh -f tsim/compute/block_dist.sim ./test.sh -f tsim/compute/bottom.sim ./test.sh -f tsim/compute/count.sim ./test.sh -f tsim/compute/diff.sim From 6c13a8e2e8ed00e3bc9a48f873448b1b09aab591 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 11:43:06 +0800 Subject: [PATCH 29/94] fix(query):fix syntax error. --- source/libs/executor/src/scanoperator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7b274e5af8..127bc61b65 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2379,9 +2379,9 @@ typedef struct STableMergeScanInfo { SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; -int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, +int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { - int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2391,8 +2391,8 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle return TSDB_CODE_SUCCESS; } - pTableListInfo->needSortTableByGroupId = pTableScanNode->groupSort; - code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags); + pTableListInfo->needSortTableByGroupId = groupSort; + code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags); if (code != TSDB_CODE_SUCCESS) { return code; } From 67afe73a1172d5555d6f1a4cd9865e0576c4d381 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 15 Jul 2022 11:47:31 +0800 Subject: [PATCH 30/94] test:add test case for last_row --- tests/system-test/2-query/csum.py | 32 +- tests/system-test/2-query/last_row.py | 803 +++++++++++++++++++++ tests/system-test/2-query/max_partition.py | 24 +- tests/system-test/2-query/sample.py | 66 +- 4 files changed, 861 insertions(+), 64 deletions(-) create mode 100644 tests/system-test/2-query/last_row.py diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 708aa35183..425597a919 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -425,8 +425,8 @@ class TDTestCase: tdSql.checkRows(70) tdSql.query("select csum(c1) from stb1 partition by tbname ") tdSql.checkRows(40) - # tdSql.query("select csum(st1) from stb1 partition by tbname") - # tdSql.checkRows(70) + tdSql.query("select csum(st1) from stb1 partition by tbname") + tdSql.checkRows(70) tdSql.query("select csum(st1+c1) from stb1 partition by tbname") tdSql.checkRows(40) tdSql.query("select csum(st1+c1) from stb1 partition by tbname") @@ -445,22 +445,22 @@ class TDTestCase: tdSql.checkRows(40) # bug need fix - # tdSql.query("select tbname , csum(c1) from stb1 partition by tbname") - # tdSql.checkRows(40) - # tdSql.query("select tbname , csum(st1) from stb1 partition by tbname") - # tdSql.checkRows(70) - # tdSql.query("select tbname , csum(st1) from stb1 partition by tbname slimit 1") - # tdSql.checkRows(7) + tdSql.query("select tbname , csum(c1) from stb1 partition by tbname") + tdSql.checkRows(40) + tdSql.query("select tbname , csum(st1) from stb1 partition by tbname") + tdSql.checkRows(70) + tdSql.query("select tbname , csum(st1) from stb1 partition by tbname slimit 1") + tdSql.checkRows(7) # partition by tags - # tdSql.query("select st1 , csum(c1) from stb1 partition by st1") - # tdSql.checkRows(40) - # tdSql.query("select csum(c1) from stb1 partition by st1") - # tdSql.checkRows(40) - # tdSql.query("select st1 , csum(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(4) - # tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1") - # tdSql.checkRows(4) + tdSql.query("select st1 , csum(c1) from stb1 partition by st1") + tdSql.checkRows(40) + tdSql.query("select csum(c1) from stb1 partition by st1") + tdSql.checkRows(40) + tdSql.query("select st1 , csum(c1) from stb1 partition by st1 slimit 1") + tdSql.checkRows(4) + tdSql.query("select csum(c1) from stb1 partition by st1 slimit 1") + tdSql.checkRows(4) # partition by col # tdSql.query("select c1 , csum(c1) from stb1 partition by c1") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py new file mode 100644 index 0000000000..449f5c7806 --- /dev/null +++ b/tests/system-test/2-query/last_row.py @@ -0,0 +1,803 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":0} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + self.tb_nums = 10 + self.row_nums = 20 + self.ts = 1434938400000 + self.time_step = 1000 + + def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): + tdLog.info(" prepare datas for auto check abs function ") + + tdSql.execute(" create database test cachelast 1 ") + tdSql.execute(" use test ") + tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") + for tbnum in range(tbnums): + tbname = "sub_tb_%d"%tbnum + tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) + + ts = self.ts + for row in range(rownums): + ts = self.ts + time_step*row + c1 = random.randint(0,10000) + c2 = random.randint(0,100000) + c3 = random.randint(0,125) + c4 = random.randint(0,125) + c5 = random.random()/1.0 + c6 = random.random()/1.0 + c7 = "'true'" + c8 = "'binary_val'" + c9 = "'nchar_val'" + c10 = ts + tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") + + tdSql.execute("use test") + tbnames = ["stb", "sub_tb_1"] + support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname) + origin_sql = "select {} from {} order by tbname".format(colname, tbname) + if coltype[1] in support_types: + self.check_result_auto(origin_sql , abs_sql) + + + def prepare_datas(self): + tdSql.execute("create database if not exists db keep 3650 duration 1000 cachelast 1") + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def prepare_tag_datas(self): + # prepare datas + tdSql.execute( + "create database if not exists testdb keep 3650 duration 1000 cachelast 1") + tdSql.execute(" use testdb ") + + tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ + uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags( t1 int , t2 bigint , t3 smallint , t4 tinyint , t5 float , t6 double , t7 bool , t8 binary(36)\ + , t9 nchar(36) , t10 int unsigned , t11 bigint unsigned ,t12 smallint unsigned , t13 tinyint unsigned ,t14 timestamp ) ") + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute( + f'create table ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" + ) + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") + + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto(self, origin_query, abs_query): + abs_result = tdSql.getResult(abs_query) + origin_result = tdSql.getResult(origin_query) + + auto_result = [] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >= 0: + elem = elem + else: + elem = -elem + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index, row in enumerate(abs_result): + for col_index, elem in enumerate(row): + if auto_result[row_index][col_index] != elem: + check_status = False + if not check_status: + tdLog.notice( + "abs function value has not as expected , sql is \"%s\" " % abs_query) + sys.exit(1) + else: + tdLog.info( + "abs value check pass , it work as expected ,sql is \"%s\" " % abs_query) + + def test_errors(self): + tdSql.execute("use testdb") + + # bug need fix + # tdSql.query("select last_row(c1 ,NULL) from t1") + + error_sql_lists = [ + "select last_row from t1", + "select last_row(-+--+c1) from t1", + # "select +-last_row(c1) from t1", + # "select ++-last_row(c1) from t1", + # "select ++--last_row(c1) from t1", + # "select - -last_row(c1)*0 from t1", + # "select last_row(tbname+1) from t1 ", + "select last_row(123--123)==1 from t1", + "select last_row(c1) as 'd1' from t1", + "select last_row(c1 ,NULL) from t1", + "select last_row(,) from t1;", + "select last_row(abs(c1) ab from t1)", + "select last_row(c1) as int from t1", + "select last_row from stb1", + # "select last_row(-+--+c1) from stb1", + # "select +-last_row(c1) from stb1", + # "select ++-last_row(c1) from stb1", + # "select ++--last_row(c1) from stb1", + # "select - -last_row(c1)*0 from stb1", + # "select last_row(tbname+1) from stb1 ", + "select last_row(123--123)==1 from stb1", + "select last_row(c1) as 'd1' from stb1", + # "select last_row(c1 ,c2 ) from stb1", + "select last_row(c1 ,NULL) from stb1", + "select last_row(,) from stb1;", + "select last_row(abs(c1) ab from stb1)", + "select last_row(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + tdSql.execute("use testdb") + tbnames = ["stb1", "t1", "ct1", "ct2"] + + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + col_note = coltype[-1] + if col_note != "TAG": + abs_sql = "select last_row({}) from {}".format(colname, tbname) + tdSql.query(abs_sql) + + + def basic_abs_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select last_row(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select last_row(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select last_row(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select last_row(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select last_row(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select last_row(c6) from ct3") + + # used for regular table + + # bug need fix + tdSql.query("select last_row(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.query("select last_row(c1) from ct4") + tdSql.checkData(0, 0, None) + tdSql.query("select last_row(c1) from stb1") + tdSql.checkData(0, 0, None) + + # # bug need fix + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + + # # bug need fix + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from ct1") + tdSql.checkData(0, 0, 9) + tdSql.checkData(0, 1, -99999) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4,-9.99000) + + # bug need fix + # tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") + # tdSql.checkData(0, 0, 9) + # tdSql.checkData(0, 1, -99999) + # tdSql.checkData(0, 2, -999) + # tdSql.checkData(0, 3, None) + # tdSql.checkData(0, 4,-9.99000) + + # bug fix + tdSql.query("select last_row(abs(c1)) from ct1") + tdSql.checkData(0,0,9) + + # # bug fix + tdSql.query("select last_row(c1+1) from ct1") + tdSql.query("select last_row(c1+1) from stb1") + tdSql.query("select last_row(c1+1) from t1") + + # used for stable table + tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from ct1") + tdSql.checkData(0,0,9) + tdSql.checkData(0,1,-99999) + tdSql.checkData(0,2,-999) + tdSql.checkData(0,3,None) + + # bug need fix + tdSql.query("select last_row(c1 ,c2 ,c3) from stb1 ") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,None) + + # tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from stb1 where ts 5 ") + tdSql.checkData(0, 0, 6) + tdSql.checkData(0, 1, 6.000000000) + tdSql.checkData(0, 2, 6.000000000) + tdSql.checkData(0, 3, 5.900000000) + tdSql.checkData(0, 4, 2.084962501) + + tdSql.query( + "select last_row(c1,c2,c1+5) from ct4 where c1=5 ") + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 55555) + tdSql.checkData(0, 2, 10.000000000) + + tdSql.query( + "select last(c1,c2,c1+5) from ct4 where c1=5 ") + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 55555) + tdSql.checkData(0, 2, 10.000000000) + + tdSql.query( + "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 1, 88888) + tdSql.checkData(0, 2, 8.000000000) + tdSql.checkData(0, 3, 8.000000000) + tdSql.checkData(0, 4, 7.900000000) + tdSql.checkData(0, 5, 3.000000000) + + def abs_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test cachelast 2") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + # check basic elem for table per row + tdSql.query( + "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from sub1_bound ") + tdSql.checkData(0, 0, -2147483646) + tdSql.checkData(0, 1, -9223372036854775806) + tdSql.checkData(0, 2, -32765.000000000) + tdSql.checkData(0, 3, -125.000000000) + # check + - * / in functions + tdSql.query( + "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from sub1_bound ") + + def test_tag_compute_for_scalar_function(self): + + tdSql.execute("use testdb") + + # bug need fix + + tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") + tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") + tdSql.checkRows(13) + tdSql.query("select last_row(c1,t1) from stb1 ") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,3) + # tdSql.query("select last_row(c1),t1 from stb1 ") + # tdSql.checkData(0,0,None) + # tdSql.checkData(0,1,3) + tdSql.query("select last_row(c1,t1),last(t1) from stb1 ") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,3) + tdSql.checkData(0,2,3) + + tdSql.query("select last_row(t1) from stb1 where t1 >0 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,3) + tdSql.query("select last_row(t1) from stb1 where t1 =3 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,3) + + tdSql.query("select last_row(t1) from stb1 where t1 =2") + tdSql.checkRows(0) + + # nest query for last_row + # tdSql.query("select last_row(t1) from (select c1 ,t1 from stb1)") + # tdSql.checkData(0,0,61) + # tdSql.query("select distinct(c1) ,t1 from stb1") + # tdSql.checkRows(20) + tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") + tdSql.checkData(0,0,None) + + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkData(0,0,None) + + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") + tdSql.checkData(0,1,None,None) + + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts now -1h and ts now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') + # tdSql.checkRows(40) + # tdSql.checkData(0,0,None) + tdSql.query('select max(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.checkRows(5) + + + + def support_super_table_test(self): + tdSql.execute(" use testdb ") + self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + + self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + pass + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + # tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + self.prepare_tag_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: abs basic query ============") + + self.basic_abs_function() + + tdLog.printNoPrefix("==========step5: abs boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step6: abs filter query ============") + + self.abs_func_filter() + + tdLog.printNoPrefix("==========step6: tag coumpute query ============") + + self.test_tag_compute_for_scalar_function() + + tdLog.printNoPrefix("==========step7: check result of query ============") + + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) + + tdLog.printNoPrefix("==========step8: check abs result of stable query ============") + + self.support_super_table_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 90b8d25cb1..cf0d6639c2 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -45,8 +45,8 @@ class TDTestCase: tdSql.query(" select max(c1) from stb group by tbname order by tbname ") tdSql.checkRows(self.tb_nums) # bug need fix - # tdSql.query(" select max(t1) from stb group by t2 order by t2 ") - # tdSql.checkRows(self.tb_nums) + tdSql.query(" select max(t2) from stb group by t2 order by t2 ") + tdSql.checkRows(self.tb_nums) tdSql.query(" select max(c1) from stb group by c1 order by c1 ") tdSql.checkRows(self.row_nums+1) @@ -90,8 +90,8 @@ class TDTestCase: tdSql.query("select tbname , max(t2) from stb partition by t2 order by t2") # # bug need fix - # tdSql.query("select t2 , max(t2) from stb partition by t2 order by t2") - # tdSql.checkRows(self.tb_nums) + tdSql.query("select t2 , max(t2) from stb partition by t2 order by t2") + tdSql.checkRows(self.tb_nums) tdSql.query("select tbname , max(c1) from stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) @@ -126,8 +126,8 @@ class TDTestCase: tdSql.checkData(0,0,self.row_nums) # bug need fix - # tdSql.query("select count(c1) , max(t1) ,abs(c1) from stb partition by abs(c1) order by abs(c1)") - # tdSql.checkRows(self.row_nums+1) + tdSql.query("select count(c1) , max(t2) ,abs(c1) from stb partition by abs(c1) order by abs(c1)") + tdSql.checkRows(self.row_nums+1) tdSql.query("select max(ceil(c2)) , max(floor(t2)) ,max(floor(c2)) from stb partition by abs(c2) order by abs(c2)") @@ -136,6 +136,18 @@ class TDTestCase: tdSql.query("select max(ceil(c1-2)) , max(floor(t2+1)) ,max(c2-c1) from stb partition by abs(floor(c1)) order by abs(floor(c1))") tdSql.checkRows(self.row_nums+1) + + tdSql.query("select tbname , max(c1) ,c1 from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,'sub_stb_0') + tdSql.checkData(0,1,9) + tdSql.checkData(0,2,9) + + tdSql.query("select tbname ,top(c1,1) ,c1 from stb partition by tbname order by tbname") + tdSql.checkRows(self.tb_nums) + + tdSql.query(" select c1 , sample(c1,2) from stb partition by tbname order by tbname ") + tdSql.checkRows(self.tb_nums*2) # interval diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index f583b7dd78..84ff5a0056 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -31,6 +31,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.ts = 1537146000000 def sample_query_form(self, sel="select", func="sample(", col="c1", m_comm =",", k=1,r_comm=")", alias="", fr="from",table_expr="t1", condition=""): ''' @@ -624,10 +625,11 @@ class TDTestCase: tdLog.info(" sample data is in datas groups ,successed sql is : %s" % sample_query ) else: tdLog.exit(" sample data is not in datas groups ,failed sql is : %s" % sample_query ) - + + def basic_sample_query(self): tdSql.execute(" drop database if exists db ") - tdSql.execute(" create database if not exists db duration 300 ") + tdSql.execute(" create database if not exists db duration 300d ") tdSql.execute(" use db ") tdSql.execute( '''create table stb1 @@ -759,14 +761,6 @@ class TDTestCase: self.check_sample("select sample( c1 ,3 ) from t1 where c1 between 1 and 10" ,"select c1 from t1 where c1 between 1 and 10") - tdSql.query("select sample(c1,2) ,c2,c3 ,c5 from stb1") - tdSql.checkRows(2) - tdSql.checkCols(4) - - self.check_sample("select sample( c1 ,3 ),c2,c3,c4,c5 from t1 where c1 between 1 and 10" ,"select c1,c2,c3,c4,c5 from t1 where c1 between 1 and 10") - self.check_sample("select sample( c1 ,3 ),c2,c3,c4,c5 from stb1 where c1 between 1 and 10" ,"select c1,c2,c3,c4,c5 from stb1 where c1 between 1 and 10") - self.check_sample("select sample( c1 ,3 ),t1 from stb1 where c1 between 1 and 10" ,"select c1,t1 from stb1 where c1 between 1 and 10") - # join tdSql.query("select sample( ct4.c1 , 1 ) from ct1, ct4 where ct4.ts=ct1.ts") @@ -779,8 +773,8 @@ class TDTestCase: self.check_sample("select sample(c1,2) from stb1 partition by tbname" , "select c1 from stb1 partition by tbname") # nest query - tdSql.query("select sample(c1,2) from (select c1 from t1); ") - tdSql.checkRows(2) + # tdSql.query("select sample(c1,2) from (select c1 from t1); ") + # tdSql.checkRows(2) # union all tdSql.query("select sample(c1,2) from t1 union all select sample(c1,3) from t1") @@ -798,36 +792,6 @@ class TDTestCase: tdSql.query("select sample(c1,100)+2 from ct1") tdSql.query("select abs(sample(c1,100)) from ct1") - # support stable and tbname - tdSql.query("select tbname ,sample(c1,2) from stb1 partition by tbname order by tbname") - tdSql.checkRows(4) - tdSql.checkData(0,0,'ct1') - tdSql.checkData(3,0,'ct4') - - # # bug need fix - # tdSql.query(" select tbname ,c1 ,t1, sample(c1,2) from stb1 partition by tbname order by tbname ") - # tdSql.checkRows(4) - # tdSql.checkData(0,0,'ct1') - # tdSql.checkData(3,0,'ct4') - # tdSql.checkData(0,2,1) - # tdSql.checkData(3,2,4) - - tdSql.query(" select tbname ,c1 ,t1, sample(c1,2) from stb1 partition by t1 order by t1 ") - tdSql.checkRows(4) - tdSql.checkData(0,0,'ct1') - tdSql.checkData(3,0,'ct4') - tdSql.checkData(0,2,1) - tdSql.checkData(3,2,4) - - # bug need fix - # tdSql.query(" select tbname ,c1 ,t1, sample(c1,2) from stb1 partition by c1 order by c1 ") - # tdSql.checkRows(21) - - # bug need fix - # tdSql.query(" select sample(c1,2) from stb1 partition by c1 ") - # tdSql.checkRows(21) - - def sample_test_run(self) : tdLog.printNoPrefix("==========support sample function==========") tbnum = 10 @@ -891,11 +855,29 @@ class TDTestCase: self.basic_sample_query() + def sample_big_data(self): + tdSql.execute("create database sample_db") + tdSql.execute("use sample_db") + tdSql.execute("create stable st (ts timestamp ,c1 int ) tags(ind int)" ) + tdSql.execute("create table sub_tb using st tags(1)") + + for i in range(2000): + ts = self.ts+i*10 + tdSql.execute(f"insert into sub_tb values({ts} ,{i})") + + tdSql.query("select count(*) from st") + tdSql.checkData(0,0,2000) + tdSql.query("select sample(c1 ,1000) from st") + tdSql.checkRows(1000) + + + def run(self): import traceback try: # run in develop branch self.sample_test_run() + self.sample_big_data() pass except Exception as e: traceback.print_exc() From 013f5031bccdf647f0538e6d64c5b2c78b5ff613 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 11:53:32 +0800 Subject: [PATCH 31/94] fix(query):update the time type check. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cecd80123b..79d94e89b4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -514,7 +514,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->startRowIndex = 0; // NOTE: the last parameter is the primary timestamp column - if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1) && pOneExpr->base.numOfParams > 1) { pInput->pPTS = pInput->pData[j]; ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); } From 1342fbe913a2cf562bef18981d551f5d5a3cdeee Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 11:59:42 +0800 Subject: [PATCH 32/94] fix: sql command 'show table distributed' --- source/libs/parser/src/parAstParser.c | 38 ++++++++++-------------- source/libs/parser/test/parShowToUse.cpp | 6 ++++ source/libs/planner/src/planSpliter.c | 3 +- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 3fba404eaa..d24da2f83c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -118,36 +118,33 @@ static bool needGetTableIndex(SNode* pStmt) { return false; } -static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTableNode* pRealTable, +static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, AUTH_TYPE authType) { - int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, - pCxt->pMetaCache); + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { - code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, - pCxt->pMetaCache); + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pRealTable->table.dbName, authType, - pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, authType, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code && needGetTableIndex(pCxt->pStmt)) { - code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, - pCxt->pMetaCache); + code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); } - if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_DNODE_VARIABLES))) { + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); + code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache); } return code; } static EDealRes collectMetaKeyFromRealTable(SCollectMetaKeyFromExprCxt* pCxt, SRealTableNode* pRealTable) { - pCxt->errCode = collectMetaKeyFromRealTableImpl(pCxt->pComCxt, pRealTable, AUTH_TYPE_READ); + pCxt->errCode = collectMetaKeyFromRealTableImpl(pCxt->pComCxt, pRealTable->table.dbName, pRealTable->table.tableName, + AUTH_TYPE_READ); return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -454,11 +451,13 @@ static int32_t collectMetaKeyFromShowTransactions(SCollectMetaKeyCxt* pCxt, SSho } static int32_t collectMetaKeyFromDelete(SCollectMetaKeyCxt* pCxt, SDeleteStmt* pStmt) { - return collectMetaKeyFromRealTableImpl(pCxt, (SRealTableNode*)pStmt->pFromTable, AUTH_TYPE_WRITE); + STableNode* pTable = (STableNode*)pStmt->pFromTable; + return collectMetaKeyFromRealTableImpl(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE); } static int32_t collectMetaKeyFromInsert(SCollectMetaKeyCxt* pCxt, SInsertStmt* pStmt) { - int32_t code = collectMetaKeyFromRealTableImpl(pCxt, (SRealTableNode*)pStmt->pTable, AUTH_TYPE_WRITE); + STableNode* pTable = (STableNode*)pStmt->pTable; + int32_t code = collectMetaKeyFromRealTableImpl(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE); if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } @@ -471,14 +470,7 @@ static int32_t collectMetaKeyFromShowBlockDist(SCollectMetaKeyCxt* pCxt, SShowTa strcpy(name.tname, pStmt->tableName); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); if (TSDB_CODE_SUCCESS == code) { - code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); - } - - if (TSDB_CODE_SUCCESS == code) { - code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); - } - if (TSDB_CODE_SUCCESS == code) { - code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); + code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ); } return code; } diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp index eecb291554..57f07b2285 100644 --- a/source/libs/parser/test/parShowToUse.cpp +++ b/source/libs/parser/test/parShowToUse.cpp @@ -179,6 +179,12 @@ TEST_F(ParserShowToUseTest, showTables) { run("SHOW test.tables like 'c%'"); } +TEST_F(ParserShowToUseTest, showTableDistributed) { + useDb("root", "test"); + + run("SHOW TABLE DISTRIBUTED st1"); +} + // todo SHOW topics TEST_F(ParserShowToUseTest, showUsers) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index ae0ccb1c51..871bcf015b 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -198,7 +198,8 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || + (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { From dd6b50f7c25b91079ed30c92e76cf8c29c1c887f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 Jul 2022 12:57:27 +0800 Subject: [PATCH 33/94] feat: change upx download location for taosadapter3.0 (#14934) * feat(taosAdapter): taosAdapter for 3.0 * chore: change upx download location Co-authored-by: t_max <1172915550@qq.com> --- tools/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 5d463e1de9..e6cd47b91d 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -114,7 +114,7 @@ ELSE () COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND curl -sL https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -o upx.tar.xz && tar -xvJf upx.tar.xz -C ${CMAKE_BINARY_DIR} --strip-components 1 > /dev/null && ${CMAKE_BINARY_DIR}/upx taosadapter || : + COMMAND wget -c https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O ${CMAKE_CURRENT_SOURCE_DIR}/upx.tar.xz && tar -xvJf ${CMAKE_CURRENT_SOURCE_DIR}/upx.tar.xz -C ${CMAKE_CURRENT_SOURCE_DIR} --strip-components 1 > /dev/null && ${CMAKE_CURRENT_SOURCE_DIR}/upx taosadapter || : COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ From cd958f41aed38fb4d45e19a634cc2ca8bcb5cf69 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 13:08:49 +0800 Subject: [PATCH 34/94] doc: move cache section back to develop guide --- docs/zh/07-develop/08-cache.md | 48 +++++++++++++++++++++++------- docs/zh/21-tdinternal/03-cache.md | 49 ------------------------------- 2 files changed, 38 insertions(+), 59 deletions(-) delete mode 100644 docs/zh/21-tdinternal/03-cache.md diff --git a/docs/zh/07-develop/08-cache.md b/docs/zh/07-develop/08-cache.md index cc59c0353c..bd9da6062d 100644 --- a/docs/zh/07-develop/08-cache.md +++ b/docs/zh/07-develop/08-cache.md @@ -1,21 +1,49 @@ --- sidebar_label: 缓存 title: 缓存 -description: "提供写驱动的缓存管理机制，将每个表最近写入的一条记录持续保存在缓存中，可以提供高性能的最近状态查询。" +description: "TDengine 内部的缓存设计" --- +为了实现高效的写入和查询，TDengine 充分利用了各种缓存技术，本节将对 TDengine 中对缓存的使用做详细的说明。 + +## 写缓存 + TDengine 采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine 充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 -TDengine 通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，可通过设置合适的配置参数将 TDengine 作为数据缓存来使用，而不需要再部署额外的缓存系统，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine 重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的 key-value 缓存系统再将之前缓存的数据重新加载到缓存中。 - -TDengine 分配固定大小的内存空间作为缓存空间，缓存空间可根据应用的需求和硬件资源配置。通过适当的设置缓存空间，TDengine 可以提供极高性能的写入和查询的支持。TDengine 中每个虚拟节点（virtual node）创建时分配独立的缓存池。每个虚拟节点管理自己的缓存池，不同虚拟节点间不共享缓存池。每个虚拟节点内部所属的全部表共享该虚拟节点的缓存池。 - -TDengine 将内存池按块划分进行管理，数据在内存块里是以行（row）的形式存储。一个 vnode 的内存池是在 vnode 创建时按块分配好，而且每个内存块按照先进先出的原则进行管理。在创建内存池时，块的大小由系统配置参数 cache 决定；每个 vnode 中内存块的数目则由配置参数 blocks 决定。因此对于一个 vnode，总的内存大小为：`cache * blocks`。一个 cache block 需要保证每张表能存储至少几十条以上记录，才会有效率。 - -你可以通过函数 last_row() 快速获取一张表或一张超级表的最后一条记录，这样很便于在大屏显示各设备的实时状态或采集值。例如： +每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。 ```sql -select last_row(voltage) from meters where location='California.SanFrancisco'; +create database db0 vgroups 100 buffer 16MB ``` -该 SQL 语句将获取所有位于加利福尼亚州旧金山市的电表最后记录的电压值。 +理论上缓存越大越好，但超过一定阈值后再增加缓存对写入性能提升并无帮助，一般情况下使用默认值即可。 + +## 读缓存 + +在创建数据库时可以选择是否缓存该数据库中每个子表的最新数据。由参数 cachelast 设置，分为三种情况： +- 0: 不缓存 +- 1: 缓存子表最近一行数据，这将显著改善 last_row 函数的性能 +- 2: 缓存子表每一列最近的非 NULL 值，这将显著改善无特殊影响（比如 WHERE, ORDER BY, GROUP BY, INTERVAL）时的 last 函数的性能 +- 3: 同时缓存行和列，即等同于上述 cachelast 值为 1 或 2 时的行为同时生效 + +## 元数据缓存 + +为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。 + +```sql +create database db0 pages 128 pagesize 16kb +``` + +上述语句会为数据库 db0 的每个 vnode 创建 128 个 page，每个 page 16kb 的元数据缓存。 + +## 文件系统缓存 + +TDengine 利用 WAL 技术来提供基本的数据可靠性。写入 WAL 本质上是以顺序追加的方式写入磁盘文件。此时文件系统缓存在写入性能中也会扮演关键角色。在创建数据库时可以利用 wal 参数来选择性能优先或者可靠性优先。 +- 1: 写 WAL 但不执行 fsync ，新写入 WAL 的数据保存在文件系统缓存中但并未写入磁盘，这种方式性能优先 +- 2: 写 WAL 且执行 fsync，新写入 WAL 的数据被立即同步到磁盘上，可靠性更高 + +## 客户端缓存 + +为了进一步提升整个系统的处理效率，除了以上提到的服务端缓存技术之外，在 TDengine 的所有客户端都要调用的核心库 libtaos.so （也称为 taosc ）中也充分利用了缓存技术。在 taosc 中会缓存所访问过的各个数据库、超级表以及子表的元数据，集群的拓扑结构等关键元数据。 + +当有多个客户端同时访问 TDengine 集群，且其中一个客户端对某些元数据进行了修改的情况下，有可能会出现其它客户端所缓存的元数据不同步或失效的情况，此时需要在客户端执行 "reset query cache" 以让整个缓存失效从而强制重新拉取最新的元数据重新建立缓存。 diff --git a/docs/zh/21-tdinternal/03-cache.md b/docs/zh/21-tdinternal/03-cache.md deleted file mode 100644 index 8009c3d712..0000000000 --- a/docs/zh/21-tdinternal/03-cache.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -sidebar_label: 缓存 -title: 缓存 -description: "TDengine 内部的缓存设计" ---- - -为了实现高效的写入和查询，TDengine 充分利用了各种缓存技术，本节将对 TDengine 中对缓存的使用做详细的说明。 - -## 写缓存 - -TDengine 采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine 充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 - -每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。 - -```sql -create database db0 vgroups 100 buffer 16 -``` - -理论上缓存越大越好，但超过一定阈值后再增加缓存对写入性能提升并无帮助，一般情况下使用默认值即可。 - -## 读缓存 - -在创建数据库时可以选择是否缓存该数据库中每个子表的最新数据。由参数 cachelast 设置，分为三种情况： -- 0: 不缓存 -- 1: 缓存子表最近一行数据，这将显著改善 last_row 函数的性能 -- 2: 缓存子表每一列最近的非 NULL 值，这将显著改善无特殊影响（比如 WHERE, ORDER BY, GROUP BY, INTERVAL）时的 last 函数的性能 -- 3: 同时缓存行和列，即等同于上述 cachelast 值为 1 或 2 时的行为同时生效 - -## 元数据缓存 - -为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。 - -```sql -create database db0 pages 128 pagesize 16kb -``` - -TBD: 元数据缓存中的替换策略需要补充 - -## 文件系统缓存 - -TDengine 利用 WAL 技术来提供基本的数据可靠性。写入 WAL 本质上是以顺序追加的方式写入磁盘文件。此时文件系统缓存在写入性能中也会扮演关键角色。在创建数据库时可以利用 wal 参数来选择性能优先或者可靠性优先。 -- 1: 写 WAL 但不执行 fsync ，新写入 WAL 的数据保存在文件系统缓存中但并未写入磁盘，这种方式性能优先 -- 2: 写 WAL 且执行 fsync，新写入 WAL 的数据被立即同步到磁盘上，可靠性更高 - -## 客户端缓存 - -为了进一步提升整个系统的处理效率，除了以上提到的服务端缓存技术之外，在 TDengine 的所有客户端都要调用的核心库 libtaos.so （也称为 taosc ）中也充分利用了缓存技术。在 taosc 中会缓存所访问过的各个数据库、超级表以及子表的元数据，集群的拓扑结构等关键元数据。 - -当有多个客户端同时访问 TDengine 集群，且其中一个客户端对某些元数据进行了修改的情况下，有可能会出现其它客户端所缓存的元数据不同步或失效的情况，此时需要在客户端执行 "reset query cache" 以让整个缓存失效从而强制重新拉取最新的元数据重新建立缓存。 From 283b3eb3d9455562991a97e22aef2a2e4fb8d0d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 13:25:15 +0800 Subject: [PATCH 35/94] fix(query): remove invalid check. --- source/libs/executor/src/executorimpl.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 79d94e89b4..c8f2083456 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -514,9 +514,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->startRowIndex = 0; // NOTE: the last parameter is the primary timestamp column - if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1) && pOneExpr->base.numOfParams > 1) { - pInput->pPTS = pInput->pData[j]; - ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); + // todo: refactor this + if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + pInput->pPTS = pInput->pData[j]; // in case of merge function, this is not always the ts column data. +// ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP); } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { From 5b55a8044ed262c0ec95e60a255dc83a87640ef5 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 13:34:47 +0800 Subject: [PATCH 36/94] doc: english version of cache --- docs/en/07-develop/08-cache.md | 51 ++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/docs/en/07-develop/08-cache.md b/docs/en/07-develop/08-cache.md index 743452faff..8e86eff741 100644 --- a/docs/en/07-develop/08-cache.md +++ b/docs/en/07-develop/08-cache.md @@ -1,19 +1,52 @@ --- sidebar_label: Cache title: Cache -description: "The latest row of each table is kept in cache to provide high performance query of latest state." +description: "Caching System inside TDengine" --- +To achieve the purpose of high performance data writing and querying, TDengine employs a lot of caching technologies in both server side and client side. + +## Write Cache + The cache management policy in TDengine is First-In-First-Out (FIFO). FIFO is also known as insert driven cache management policy and it is different from read driven cache management, which is more commonly known as Least-Recently-Used (LRU). FIFO simply stores the latest data in cache and flushes the oldest data in cache to disk, when the cache usage reaches a threshold. In IoT use cases, it is the current state i.e. the latest or most recent data that is important. The cache policy in TDengine, like much of the design and architecture of TDengine, is based on the nature of IoT data. -Caching the latest data provides the capability of retrieving data in milliseconds. With this capability, TDengine can be configured properly to be used as a caching system without deploying another separate caching system. This simplifies the system architecture and minimizes operational costs. The cache is emptied after TDengine is restarted. TDengine does not reload data from disk into cache, like a key-value caching system. - -The memory space used by the TDengine cache is fixed in size and configurable. It should be allocated based on application requirements and system resources. An independent memory pool is allocated for and managed by each vnode (virtual node) in TDengine. There is no sharing of memory pools between vnodes. All the tables belonging to a vnode share all the cache memory of the vnode. - -The memory pool is divided into blocks and data is stored in row format in memory and each block follows FIFO policy. The size of each block is determined by configuration parameter `cache` and the number of blocks for each vnode is determined by the parameter `blocks`. For each vnode, the total cache size is `cache * blocks`. A cache block needs to ensure that each table can store at least dozens of records, to be efficient. - -`last_row` function can be used to retrieve the last row of a table or a STable to quickly show the current state of devices on monitoring screen. For example the below SQL statement retrieves the latest voltage of all meters in San Francisco, California. +The memory space used by each vnode as write cache is determined when creating a database. Parameter `vgroups` and `buffer` can be used to specify the number of vnode and the size of write cache for each vnode when creating the database. Then, the total size of write cache for this database is `vgroups * buffer`. ```sql -select last_row(voltage) from meters where location='California.SanFrancisco'; +create database db0 vgroups 100 buffer 16MB ``` + +The above statement creates a database of 100 vnodes while each vnode has a write cache of 16MB. + +Even though in theory it's always better to have a larger cache, the extra effect would be very minor once the size of cache grows beyond a threshold. So normally it's enough to use the default value of `buffer` parameter. + +## Read Cache + +When creating a database, it's also possible to specify whether to cache the latest data of each sub table, using parameter `cachelast`. There are 3 cases: +- 0: No cache for latest data +- 1: The last row of each table is cached, `last_row` function can benefit significantly from it +- 2: The latest non-NULL value of each column for each table is cached, `last` function can benefit very much when there is no `where`, `group by`, `order by` or `interval` clause +- 3: Bot hthe last row and the latest non-NULL value of each column for each table are cached, identical to the behavior of both 1 and 2 are set together + + +## Meta Cache + +To process data writing and querying efficiently, each vnode caches the metadata that's already retrieved. Parameters `pages` and `pagesize` are used to specify the size of metadata cache for each vnode. + +```sql +create database db0 pages 128 pagesize 16kb +``` + +The above statement will create a database db0 each of whose vnode is allocated a meta cache of `128 * 16 KB = 2 MB` . + +## File System Cache + +TDengine utilizes WAL to provide basic reliability. The essential of WAL is to append data in a disk file, so the file system cache also plays an important role in the writing performance. Parameter `wal` can be used to specify the policy of writing WAL, there are 2 cases: +- 1: Write data to WAL without calling fsync, the data is actually written to the file system cache without flushing immediately, in this way you can get better write performance +- 2: Write data to WAL and invoke fsync, the data is immediately flushed to disk, in this way you can get higher reliability + +## Client Cache + +To improve the overall efficiency of processing data, besides the above caches, the core library `libtaos.so` (also referred to as `taosc`) which all client programs depend on also has its own cache. `taosc` caches the metadata of the databases, super tables, tables that the invoking client has accessed, plus other critical metadata such as the cluster topology. + +When multiple client programs are accessing a TDengine cluster, if one of the clients modifies some metadata, the cache may become invalid in other clients. If this case happens, the client programs need to "reset query cache" to invalidate the whole cache so that `taosc` is enforced to repull the metadata it needs to rebuild the cache. From 9237014e7c876ba70fd2ab06026b2c6d2e99c3d6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 13:35:00 +0800 Subject: [PATCH 37/94] test: restore some 2.0 cases --- tests/script/tsim/parser/auto_create_tb.sim | 60 ++++++++++----------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/tests/script/tsim/parser/auto_create_tb.sim b/tests/script/tsim/parser/auto_create_tb.sim index a902469fde..673a472cf7 100644 --- a/tests/script/tsim/parser/auto_create_tb.sim +++ b/tests/script/tsim/parser/auto_create_tb.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = ac_db @@ -153,36 +150,37 @@ print $rows $data00 $data10 $data20 if $rows != 3 then return -1 endi -if $data00 != tb1 then +if $data(tb1)[0] != tb1 then return -1 endi -if $data10 != tb2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data20 != tb3 then +if $data(tb3)[0] != tb3 then return -1 endi -sql select ts,c1,c2,c3,c4,c5,c7,c8,c9 from $stb +sql select c1,c1,c2,c3,c4,c5,c7,c8,c9 from $stb +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + if $rows != 3 then return -1 endi -#if $data00 != @18-09-17 09:00:00.000@ then -# return -1 -#endi -if $data01 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data08 != 涛思数据1 then +if $data(1)[8] != 涛思数据1 then return -1 endi -if $data14 != 2.000000000 then +if $data(2)[4] != 2.000000000 then return -1 endi -if $data18 != 涛思数据2 then +if $data(2)[8] != 涛思数据2 then return -1 endi -if $data28 != 涛思数据3 then +if $data(3)[8] != 涛思数据3 then return -1 endi @@ -208,12 +206,7 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start -print ================== server restart completed -sql connect -sleep 100 -sql use $db #### auto create multiple tables sql insert into tb1 using $stb tags(1) values ( $ts0 , 1, 1, 1, 1, 'bin1', 1, 1, 1, '涛思数据1') tb2 using $stb tags(2) values ( $ts0 , 2, 2, 2, 2, 'bin2', 2, 2, 2, '涛思数据2') tb3 using $stb tags(3) values ( $ts0 , 3, 3, 3, 3, 'bin3', 3, 3, 3, '涛思数据3') @@ -221,36 +214,37 @@ sql show tables if $rows != 3 then return -1 endi -if $data00 != tb1 then +if $data(tb1)[0] != tb1 then return -1 endi -if $data10 != tb2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data20 != tb3 then +if $data(tb3)[0] != tb3 then return -1 endi -sql select ts,c1,c2,c3,c4,c5,c7,c8,c9 from $stb +sql select c1,c1,c2,c3,c4,c5,c7,c8,c9 from $stb +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + if $rows != 3 then return -1 endi -#if $data00 != @18-09-17 09:00:00.000@ then -# return -1 -#endi -if $data01 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data08 != 涛思数据1 then +if $data(1)[8] != 涛思数据1 then return -1 endi -if $data14 != 2.000000000 then +if $data(2)[4] != 2.000000000 then return -1 endi -if $data18 != 涛思数据2 then +if $data(2)[8] != 涛思数据2 then return -1 endi -if $data28 != 涛思数据3 then +if $data(3)[8] != 涛思数据3 then return -1 endi From 7620185dc6510d3b6edc2f9becddce5ae2a2a8c1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 13:40:28 +0800 Subject: [PATCH 38/94] test: restore case --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b5cb7ddb79..f37d6c6796 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -87,7 +87,7 @@ ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim -# ./test.sh -f tsim/parser/auto_create_tb.sim +# jira ./test.sh -f tsim/parser/auto_create_tb.sim # ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim # ./test.sh -f tsim/parser/between_and.sim # ./test.sh -f tsim/parser/binary_escapeCharacter.sim @@ -381,7 +381,7 @@ # ---- compute ./test.sh -f tsim/compute/avg.sim -# jira ./test.sh -f tsim/compute/block_dist.sim +./test.sh -f tsim/compute/block_dist.sim ./test.sh -f tsim/compute/bottom.sim ./test.sh -f tsim/compute/count.sim ./test.sh -f tsim/compute/diff.sim From 5b6c7bc38df52ccda414e977f31361576c50e82b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:26:18 +0800 Subject: [PATCH 39/94] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/inc/mndInt.h | 25 +++++------ source/dnode/mnode/impl/src/mndMain.c | 9 ++++ source/dnode/mnode/impl/src/mndSync.c | 7 ++++ source/dnode/vnode/src/vnd/vnodeSync.c | 5 +++ .../script/tsim/sync/mnodeLeaderTransfer.sim | 41 +++++++++++++++++++ 5 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 tests/script/tsim/sync/mnodeLeaderTransfer.sim diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 9f168e2c83..8ad1ac56e4 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -50,8 +50,8 @@ extern "C" { // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); @@ -61,7 +61,7 @@ typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SQWorker SQHandle; typedef struct { - const char * name; + const char *name; MndInitFp initFp; MndCleanupFp cleanupFp; } SMnodeStep; @@ -70,7 +70,7 @@ typedef struct { int64_t showId; ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; - SCacheObj * cache; + SCacheObj *cache; } SShowMgmt; typedef struct { @@ -84,12 +84,13 @@ typedef struct { } STelemMgmt; typedef struct { - tsem_t syncSem; + tsem_t syncSem; int64_t sync; bool standby; SReplica replica; int32_t errCode; int32_t transId; + int8_t leaderTransferFinish; } SSyncMgmt; typedef struct { @@ -107,14 +108,14 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - char * path; + char *path; int64_t checkTime; - SSdb * pSdb; - SArray * pSteps; - SQHandle * pQuery; - SHashObj * infosMeta; - SHashObj * perfsMeta; - SWal * pWal; + SSdb *pSdb; + SArray *pSteps; + SQHandle *pQuery; + SHashObj *infosMeta; + SHashObj *perfsMeta; + SWal *pWal; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 29e68ce4e8..66ec4b28f7 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -367,8 +367,17 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { } void mndPreClose(SMnode *pMnode) { + mDebug("vgId:1, mnode pre-close"); if (pMnode != NULL) { + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); + + mDebug("vgId:1, wait for mnode leader transfer"); + // wait for leader transfer finish + while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { + taosMsleep(10); + } + mDebug("vgId:1, mnode leader transfer finish"); } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c3864b620..1414bea880 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -153,6 +153,12 @@ int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int return sdbDoWrite(pMnode->pSdb, pWriter, pBuf, len); } +void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SMnode *pMnode = pFsm->data; + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 1); + mDebug("vgId:1, mnd leader transfer finish"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -160,6 +166,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpPreCommitCb = NULL; pFsm->FpRollBackCb = NULL; pFsm->FpRestoreFinishCb = mndRestoreFinish; + pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 87148a8450..38b9df52b8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -536,6 +536,10 @@ static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void * #endif } +static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SVnode *pVnode = pFsm->data; +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -544,6 +548,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = NULL; + pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim new file mode 100644 index 0000000000..6c67f639f6 --- /dev/null +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -0,0 +1,41 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql connect + +print =============== show dnodes +sql show mnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data02 != leader then + return -1 +endi + +print =============== create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sleep 3000 + + +print =============== create mnode 2 +sql create mnode on dnode 2 +sql create mnode on dnode 3 +sleep 3000 + +print =============== create user +sql create user user1 PASS 'user1' +sql show users +if $rows != 2 then + return -1 +endi + From b8a4878b0005c6aee9baa61724abfe038d9a1daf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 14:34:40 +0800 Subject: [PATCH 40/94] fix(query): add leastsquares function scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 168 ++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 7c23184d93..d5e9a2e625 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -103,6 +103,7 @@ int32_t minScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t maxScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 747b1c8d9f..6fb808a2e8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1981,6 +1981,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getLeastSQRFuncEnv, .initFunc = leastSQRFunctionSetup, .processFunc = leastSQRFunction, + .sprocessFunc = leastSQRScalarFunction, .finalizeFunc = leastSQRFinalize, .invertFunc = NULL, .combineFunc = leastSQRCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c64b1d79ba..ba35356a9c 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2139,3 +2139,171 @@ int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } +#define LEASTSQR_CAL(p, x, y, index, step) \ + do { \ + (p)[0][0] += (double)(x) * (x); \ + (p)[0][1] += (double)(x); \ + (p)[0][2] += (double)(x) * (y)[index]; \ + (p)[1][2] += (y)[index]; \ + (x) += step; \ + } while (0) + +int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + double startVal, stepVal; + double matrix[2][3] = {0}; + GET_TYPED_DATA(startVal, double, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(stepVal, double, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + + int32_t type = GET_PARAM_TYPE(pInput); + int64_t count = 0; + + switch(type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *in = (int8_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *in = (int16_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *in = (int32_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *in = (int64_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *in = (uint8_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *in = (uint16_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *in = (uint32_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *in = (uint64_t *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *in = (float *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *in = (double *)pInputData->pData; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + count++; + LEASTSQR_CAL(matrix, startVal, in, i, stepVal); + } + break; + } + } + + if (count == 0) { + colDataAppendNULL(pOutputData, 0); + } else { + matrix[1][1] = (double)count; + matrix[1][0] = matrix[0][1]; + + double matrix00 = matrix[0][0] - matrix[1][0] * (matrix[0][1] / matrix[1][1]); + double matrix02 = matrix[0][2] - matrix[1][2] * (matrix[0][1] / matrix[1][1]); + double matrix12 = matrix[1][2] - matrix02 * (matrix[1][0] / matrix00); + matrix02 /= matrix00; + + matrix12 /= matrix[1][1]; + + char buf[64] = {0}; + size_t len = + snprintf(varDataVal(buf), sizeof(buf) - VARSTR_HEADER_SIZE, "{slop:%.6lf, intercept:%.6lf}", matrix02, matrix12); + varDataSetLen(buf, len); + colDataAppend(pOutputData, 0, buf, false); + + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} From 95023fd56ea2132537db921038aebcd7179365e9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:34:58 +0800 Subject: [PATCH 41/94] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/src/mndMain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 66ec4b28f7..40ac4514c5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -367,17 +367,17 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { } void mndPreClose(SMnode *pMnode) { - mDebug("vgId:1, mnode pre-close"); if (pMnode != NULL) { atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); - mDebug("vgId:1, wait for mnode leader transfer"); + mDebug("vgId:1, mnode start leader transfer"); // wait for leader transfer finish while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { taosMsleep(10); + mDebug("vgId:1, mnode waiting for leader transfer"); } - mDebug("vgId:1, mnode leader transfer finish"); + mDebug("vgId:1, mnode finish leader transfer"); } } From 35e8ad28117a9b1630381586540a3a201c54c0fb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 14:51:20 +0800 Subject: [PATCH 42/94] feat: add time unsynced check --- include/common/tmsg.h | 2 + include/util/taoserror.h | 1 + source/client/inc/clientInt.h | 2 +- source/client/src/clientEnv.c | 4 +- source/client/src/clientHb.c | 82 ++++++++++++------------ source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 12 ++++ source/common/src/tmsg.c | 4 ++ source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/libs/function/src/udfd.c | 9 +++ source/libs/index/src/indexCache.c | 5 +- source/libs/transport/src/transCli.c | 12 ++-- source/libs/transport/src/transSvr.c | 4 +- source/util/src/terror.c | 1 + 14 files changed, 89 insertions(+), 55 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0ab65e7978..cb8e1171e4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -525,6 +525,7 @@ typedef struct { int8_t superUser; int8_t connType; SEpSet epSet; + int32_t svrTimestamp; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; @@ -2224,6 +2225,7 @@ typedef struct { typedef struct { int64_t reqId; int64_t rspId; + int32_t svrTimestamp; SArray* rsps; // SArray } SClientHbBatchRsp; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce434612c3..24837aa35f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -73,6 +73,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) #define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0032) #define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0033) +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0034) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 700a4d9daf..3999eaccb2 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -285,7 +285,7 @@ static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; -extern void* tscQhandle; +extern int32_t timestampDeltaLimit; __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 207ac01a2c..67e6b69233 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -35,6 +35,8 @@ SAppInfo appInfo; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; +int32_t timestampDeltaLimit = 900; // s + static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; @@ -181,7 +183,7 @@ void destroyTscObj(void *pObj) { destroyAllRequests(pTscObj->pRequests); taosHashCleanup(pTscObj->pRequests); - + schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2a9d113108..b12806ee0f 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -70,7 +70,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - + vgInfo->vgVersion = rsp->vgVersion; vgInfo->hashMethod = rsp->hashMethod; vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -156,18 +156,18 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { STscObj *pTscObj = (STscObj *)acquireTscObj(pRsp->connKey.tscRid); if (NULL == pTscObj) { tscDebug("tscObj rid %" PRIx64 " not exist", pRsp->connKey.tscRid); - } else { + } else { if (pRsp->query->totalDnodes > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pRsp->query->epSet)) { - SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; - SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; - SEp* pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; - tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", - pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, - pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, pNewEp->port); - + SEpSet *pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; + SEp *pOrigEp = &pOrig->eps[pOrig->inUse]; + SEp *pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; + tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", pOrig->inUse, pOrig->numOfEps, + pOrigEp->fqdn, pOrigEp->port, pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, + pNewEp->port); + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pRsp->query->epSet); } - + pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; @@ -263,13 +263,20 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { - static int32_t emptyRspNum = 0; + static int32_t emptyRspNum = 0; char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; if (TSDB_CODE_SUCCESS == code) { tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); } - + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - pRsp.svrTimestamp); + if (delta > timestampDeltaLimit) { + code = TSDB_CODE_TIME_UNSYNCED; + tscError("time diff: %ds is too big", delta); + } + int32_t rspNum = taosArrayGetSize(pRsp.rsps); taosThreadMutexLock(&appInfo.mutex); @@ -373,7 +380,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { releaseTscObj(connKey->tscRid); return TSDB_CODE_QRY_OUT_OF_MEMORY; } - + hbBasic->connId = pTscObj->connId; int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; @@ -392,7 +399,6 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); if (code) { releaseTscObj(connKey->tscRid); @@ -436,13 +442,12 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); } - + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; } - int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -483,8 +488,8 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SSTableVersion *stbs = NULL; - uint32_t stbNum = 0; - int32_t code = 0; + uint32_t stbNum = 0; + int32_t code = 0; code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); if (TSDB_CODE_SUCCESS != code) { @@ -521,20 +526,19 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { memcpy(&req->app, pApp, sizeof(*pApp)); } else { memset(&req->app.summary, 0, sizeof(req->app.summary)); req->app.pid = taosGetPId(); req->app.appId = clientHbMgr.appId; - taosGetAppName(req->app.name, NULL); + taosGetAppName(req->app.name, NULL); } return TSDB_CODE_SUCCESS; } - int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -602,7 +606,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { continue; } - //hbClearClientHbReq(pOneReq); + // hbClearClientHbReq(pOneReq); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } @@ -615,11 +619,9 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } -void hbThreadFuncUnexpectedStopped(void) { - atomic_store_8(&clientHbMgr.threadStop, 2); -} +void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } -void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { +void hbMergeSummary(SAppClusterSummary *dst, SAppClusterSummary *src) { dst->numOfInsertsReq += src->numOfInsertsReq; dst->numOfInsertRows += src->numOfInsertRows; dst->insertElapsedTime += src->insertElapsedTime; @@ -633,7 +635,7 @@ void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { int32_t hbGatherAppInfo(void) { SAppHbReq req = {0}; - int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); if (sz > 0) { req.pid = taosGetPId(); req.appId = clientHbMgr.appId; @@ -641,11 +643,11 @@ int32_t hbGatherAppInfo(void) { } taosHashClear(clientHbMgr.appSummary); - + for (int32_t i = 0; i < sz; ++i) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); - uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL == pApp) { memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary)); req.startTime = pAppHbMgr->startTime; @@ -654,7 +656,7 @@ int32_t hbGatherAppInfo(void) { if (pAppHbMgr->startTime < pApp->startTime) { pApp->startTime = pAppHbMgr->startTime; } - + hbMergeSummary(&pApp->summary, &pAppHbMgr->pAppInstInfo->summary); } } @@ -662,7 +664,6 @@ int32_t hbGatherAppInfo(void) { return TSDB_CODE_SUCCESS; } - static void *hbThreadFunc(void *param) { setThreadName("hb"); #ifdef WINDOWS @@ -681,7 +682,7 @@ static void *hbThreadFunc(void *param) { if (sz > 0) { hbGatherAppInfo(); } - + for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -698,7 +699,7 @@ static void *hbThreadFunc(void *param) { if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); break; } @@ -708,7 +709,7 @@ static void *hbThreadFunc(void *param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); break; } @@ -725,7 +726,7 @@ static void *hbThreadFunc(void *param) { SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -759,7 +760,7 @@ static void hbStopThread() { return; } - taosThreadJoin(clientHbMgr.thread, NULL); + taosThreadJoin(clientHbMgr.thread, NULL); tscDebug("hb thread stopped"); } @@ -808,7 +809,7 @@ void hbFreeAppHbMgr(SAppHbMgr *pTarget) { } taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - + taosMemoryFree(pTarget->key); taosMemoryFree(pTarget); } @@ -843,7 +844,7 @@ int hbMgrInit() { clientHbMgr.appId = tGenIdPI64(); tscDebug("app %" PRIx64 " initialized", clientHbMgr.appId); - + clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); taosThreadMutexInit(&clientHbMgr.lock, NULL); @@ -881,7 +882,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clust SClientHbReq hbReq = {0}; hbReq.connKey = connKey; hbReq.clusterId = clusterId; - //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + // hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); @@ -920,4 +921,3 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } - diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 32c11983c2..940e6e8ebc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1478,7 +1478,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU tsem_wait(&pParam->sem); } - if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { return NULL; } else { if (setupOneRowPtr) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 94bd5dd787..331ca86d78 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -52,6 +52,18 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - connectRsp.svrTimestamp); + if (delta > timestampDeltaLimit) { + code = TSDB_CODE_TIME_UNSYNCED; + tscError("time diff: %" PRId64 "ms is too big", delta); + taosMemoryFree(pMsg->pData); + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { taosMemoryFree(pMsg->pData); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eca8d49b6a..058ef2e81d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -453,6 +453,7 @@ int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBa if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pBatchRsp->reqId) < 0) return -1; if (tEncodeI64(&encoder, pBatchRsp->rspId) < 0) return -1; + if (tEncodeI32(&encoder, pBatchRsp->svrTimestamp) < 0) return -1; int32_t rspNum = taosArrayGetSize(pBatchRsp->rsps); if (tEncodeI32(&encoder, rspNum) < 0) return -1; @@ -474,6 +475,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pBatchRsp->reqId) < 0) return -1; if (tDecodeI64(&decoder, &pBatchRsp->rspId) < 0) return -1; + if (tDecodeI32(&decoder, &pBatchRsp->svrTimestamp) < 0) return -1; int32_t rspNum = 0; if (tDecodeI32(&decoder, &rspNum) < 0) return -1; @@ -3613,6 +3615,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; if (tEncodeI8(&encoder, pRsp->connType) < 0) return -1; if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->svrTimestamp) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; tEndEncode(&encoder); @@ -3634,6 +3637,7 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->connType) < 0) return -1; if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->svrTimestamp) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 38368b4ece..81bd887ad9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndQnode.h" #include "mndShow.h" #include "mndStb.h" @@ -274,6 +274,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); + connectRsp.svrTimestamp = taosGetTimestampSec(); strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, @@ -623,6 +624,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { } SClientHbBatchRsp batchRsp = {0}; + batchRsp.svrTimestamp = taosGetTimestampMs(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); int32_t sz = taosArrayGetSize(batchReq.reqs); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a412b589a9..e33c785917 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -382,6 +382,15 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - connectRsp.svrTimestamp); + if (delta > 900) { + msgInfo->code = TSDB_CODE_TIME_UNSYNCED; + goto _return; + } + + if (connectRsp.epSet.numOfEps == 0) { msgInfo->code = TSDB_CODE_MND_APP_ERROR; goto _return; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 05ce418037..bfc3047719 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -516,13 +516,14 @@ static void idxCacheMakeRoomForWrite(IndexCache* cache) { idxCacheRef(cache); cache->imm = cache->mem; cache->mem = idxInternalCacheCreate(cache->type); + cache->mem->pCache = cache; cache->occupiedMem = 0; if (quit == false) { atomic_store_32(&cache->merging, 1); } - // sched to merge - // unref cache in bgwork + // 1. sched to merge + // 2. unref cache in bgwork idxCacheSchedToMerge(cache, quit); } } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 40227f02cc..ff984e464b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1041,7 +1041,7 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, + tGDebug("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); @@ -1133,11 +1133,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (hasEpSet) { char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); } if (pCtx->pSem != NULL) { - tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { @@ -1146,7 +1146,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (retry == false && hasEpSet == true) { pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } else { @@ -1256,7 +1256,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1296,7 +1296,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 68e12a1963..9a511adf9b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1020,7 +1020,7 @@ void transRefSrvHandle(void* handle) { return; } int ref = T_REF_INC((SSvrConn*)handle); - tDebug("conn %p ref count:%d", handle, ref); + tTrace("conn %p ref count:%d", handle, ref); } void transUnrefSrvHandle(void* handle) { @@ -1028,7 +1028,7 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSvrConn*)handle); - tDebug("conn %p ref count:%d", handle, ref); + tTrace("conn %p ref count:%d", handle, ref); if (ref == 0) { destroyConn((SSvrConn*)handle, true); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ef6697b3b5..8600c61587 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -78,6 +78,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp for TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") +TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Unsynced time") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") From 681993884f12727f777d59879ce8abb078e40c79 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:55:33 +0800 Subject: [PATCH 43/94] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/src/mndMain.c | 16 +++++++++------- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncAppendEntries.c | 7 +++++++ source/libs/sync/src/syncMain.c | 5 ++++- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 40ac4514c5..041fc2a2d1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -371,13 +371,15 @@ void mndPreClose(SMnode *pMnode) { atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); - mDebug("vgId:1, mnode start leader transfer"); - // wait for leader transfer finish - while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { - taosMsleep(10); - mDebug("vgId:1, mnode waiting for leader transfer"); - } - mDebug("vgId:1, mnode finish leader transfer"); + /* + mDebug("vgId:1, mnode start leader transfer"); + // wait for leader transfer finish + while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { + taosMsleep(10); + mDebug("vgId:1, mnode waiting for leader transfer"); + } + mDebug("vgId:1, mnode finish leader transfer"); + */ } } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e361c8021c..1f26033f63 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -253,6 +253,7 @@ bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg); int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 8409e1c711..dca80f6826 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -477,6 +477,13 @@ static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); + + // leader transfer + if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { + int32_t code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); + ASSERT(code == 0); + } + if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta = {0}; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 80b2032ab7..387400c25e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2598,7 +2598,7 @@ const char* syncStr(ESyncState state) { } } -static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); syncNodeEventLog(ths, "do leader transfer"); @@ -2811,11 +2811,14 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ASSERT(code == 0); } +#if 0 + // execute in pre-commit // leader transfer if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); ASSERT(code == 0); } +#endif // restore finish // if only snapshot, a noop entry will be append, so syncLogLastIndex is always ok From 8006ee1d8e0d03f73f6ca22112d838b97018add4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 14:56:23 +0800 Subject: [PATCH 44/94] feat: add time unsynced check --- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 331ca86d78..520a566e2b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -57,7 +57,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t delta = abs(now - connectRsp.svrTimestamp); if (delta > timestampDeltaLimit) { code = TSDB_CODE_TIME_UNSYNCED; - tscError("time diff: %" PRId64 "ms is too big", delta); + tscError("time diff:%ds is too big", delta); taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 81bd887ad9..a1da68606b 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -624,7 +624,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { } SClientHbBatchRsp batchRsp = {0}; - batchRsp.svrTimestamp = taosGetTimestampMs(); + batchRsp.svrTimestamp = taosGetTimestampSec(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); int32_t sz = taosArrayGetSize(batchReq.reqs); From af57edccb817226106c7cfff5bedfa138e003f65 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 14:39:37 +0800 Subject: [PATCH 45/94] test: restore some 2.0 cases --- tests/script/jenkins/basic.txt | 32 +----- .../tsim/parser/auto_create_tb_drop_tb.sim | 12 +-- tests/script/tsim/parser/between_and.sim | 98 ++++++------------- .../tsim/parser/binary_escapeCharacter.sim | 4 - .../tsim/parser/col_arithmetic_operation.sim | 10 +- .../tsim/parser/col_arithmetic_query.sim | 24 ++--- tests/script/tsim/parser/columnValue.sim | 20 ++-- tests/script/tsim/parser/commit.sim | 4 - tests/script/tsim/parser/condition.sim | 11 +-- tests/script/tsim/parser/constCol.sim | 21 ---- tests/script/tsim/parser/create_db.sim | 6 +- .../create_db__for_community_version.sim | 6 +- tests/script/tsim/parser/create_mt.sim | 6 +- tests/script/tsim/parser/create_tb.sim | 6 +- .../tsim/parser/create_tb_with_tag_name.sim | 5 +- tests/script/tsim/parser/dbtbnameValidate.sim | 3 - tests/script/tsim/parser/distinct.sim | 4 - tests/script/tsim/parser/fill.sim | 3 - tests/script/tsim/parser/fill_stb.sim | 3 - tests/script/tsim/parser/fill_us.sim | 3 - tests/script/tsim/parser/first_last.sim | 8 +- .../tsim/parser/fourArithmetic-basic.sim | 19 ---- tests/script/tsim/parser/function.sim | 3 - tests/script/tsim/parser/groupby-basic.sim | 4 +- tests/script/tsim/parser/groupby.sim | 4 - tests/script/tsim/parser/having.sim | 5 +- tests/script/tsim/parser/having_child.sim | 5 +- tests/script/tsim/parser/import.sim | 3 - tests/script/tsim/parser/import_commit1.sim | 4 - tests/script/tsim/parser/import_commit2.sim | 4 - tests/script/tsim/parser/import_commit3.sim | 4 - tests/script/tsim/parser/import_file.sim | 6 +- tests/script/tsim/parser/insert_multiTbl.sim | 6 +- tests/script/tsim/parser/insert_tb.sim | 6 +- tests/script/tsim/parser/interp.sim | 7 +- tests/script/tsim/parser/join.sim | 7 -- tests/script/tsim/parser/join_manyblocks.sim | 5 - tests/script/tsim/parser/join_multitables.sim | 5 +- tests/script/tsim/parser/join_multivnode.sim | 5 - tests/script/tsim/parser/last_cache.sim | 9 +- tests/script/tsim/parser/last_groupby.sim | 5 +- tests/script/tsim/parser/lastrow.sim | 8 +- tests/script/tsim/parser/like.sim | 5 +- tests/script/tsim/parser/limit.sim | 12 +-- tests/script/tsim/parser/limit1.sim | 12 +-- .../script/tsim/parser/limit1_tblocks100.sim | 12 +-- tests/script/tsim/parser/limit2.sim | 4 +- .../script/tsim/parser/limit2_tblocks100.sim | 4 +- tests/script/tsim/parser/line_insert.sim | 3 - tests/script/tsim/parser/mixed_blocks.sim | 5 - tests/script/tsim/parser/nchar.sim | 4 - tests/script/tsim/parser/nestquery.sim | 4 - tests/script/tsim/parser/null_char.sim | 5 - tests/script/tsim/parser/precision_ns.sim | 3 - .../tsim/parser/projection_limit_offset.sim | 4 - tests/script/tsim/parser/regex.sim | 4 - tests/script/tsim/parser/repeatAlter.sim | 2 +- tests/script/tsim/parser/selectResNum.sim | 4 - .../tsim/parser/select_across_vnodes.sim | 4 - .../tsim/parser/select_distinct_tag.sim | 4 - .../tsim/parser/select_from_cache_disk.sim | 4 - tests/script/tsim/parser/select_with_tags.sim | 4 - tests/script/tsim/parser/set_tag_vals.sim | 4 - tests/script/tsim/parser/single_row_in_tb.sim | 8 +- tests/script/tsim/parser/sliding.sim | 6 -- tests/script/tsim/parser/slimit.sim | 8 +- tests/script/tsim/parser/slimit1.sim | 8 +- tests/script/tsim/parser/stableOp.sim | 5 +- .../tsim/parser/tags_dynamically_specifiy.sim | 4 - tests/script/tsim/parser/tags_filter.sim | 3 - tests/script/tsim/parser/tbnameIn.sim | 7 +- tests/script/tsim/parser/timestamp.sim | 9 +- tests/script/tsim/parser/top_groupby.sim | 5 +- tests/script/tsim/parser/topbot.sim | 5 - tests/script/tsim/parser/union.sim | 7 -- tests/script/tsim/parser/where.sim | 8 +- 76 files changed, 117 insertions(+), 486 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f37d6c6796..6c1938d9fc 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -88,23 +88,13 @@ ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim # jira ./test.sh -f tsim/parser/auto_create_tb.sim -# ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -# ./test.sh -f tsim/parser/between_and.sim -# ./test.sh -f tsim/parser/binary_escapeCharacter.sim -# ./test.sh -f tsim/parser/col_arithmetic_operation.sim -## ./test.sh -f tsim/parser/col_arithmetic_query.sim -## ./test.sh -f tsim/parser/columnValue.sim -## ./test.sh -f tsim/parser/columnValue_bigint.sim -## ./test.sh -f tsim/parser/columnValue_bool.sim -## ./test.sh -f tsim/parser/columnValue_double.sim -## ./test.sh -f tsim/parser/columnValue_float.sim -## ./test.sh -f tsim/parser/columnValue_int.sim -## ./test.sh -f tsim/parser/columnValue_smallint.sim -## ./test.sh -f tsim/parser/columnValue_tinyint.sim -## ./test.sh -f tsim/parser/columnValue_unsign.sim +./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +./test.sh -f tsim/parser/between_and.sim +./test.sh -f tsim/parser/binary_escapeCharacter.sim +# nojira ./test.sh -f tsim/parser/col_arithmetic_operation.sim +# nojira ./test.sh -f tsim/parser/columnValue.sim ## ./test.sh -f tsim/parser/commit.sim ## ./test.sh -f tsim/parser/condition.sim -## ./test.sh -f tsim/parser/condition_query.sim ## ./test.sh -f tsim/parser/constCol.sim # ./test.sh -f tsim/parser/create_db.sim ## ./test.sh -f tsim/parser/create_db__for_community_version.sim @@ -117,7 +107,6 @@ # ./test.sh -f tsim/parser/fill_stb.sim ## ./test.sh -f tsim/parser/fill_us.sim # ./test.sh -f tsim/parser/first_last.sim -## ./test.sh -f tsim/parser/first_last_query.sim ./test.sh -f tsim/parser/fourArithmetic-basic.sim ## ./test.sh -f tsim/parser/function.sim ./test.sh -f tsim/parser/groupby-basic.sim @@ -132,24 +121,18 @@ ## ./test.sh -f tsim/parser/insert_multiTbl.sim # ./test.sh -f tsim/parser/insert_tb.sim ## ./test.sh -f tsim/parser/interp.sim -## ./test.sh -f tsim/parser/interp_test.sim # ./test.sh -f tsim/parser/join.sim # ./test.sh -f tsim/parser/join_manyblocks.sim ## ./test.sh -f tsim/parser/join_multitables.sim # ./test.sh -f tsim/parser/join_multivnode.sim # ./test.sh -f tsim/parser/last_cache.sim -## ./test.sh -f tsim/parser/last_cache_query.sim ## ./test.sh -f tsim/parser/last_groupby.sim # ./test.sh -f tsim/parser/lastrow.sim -## ./test.sh -f tsim/parser/lastrow_query.sim ## ./test.sh -f tsim/parser/like.sim # ./test.sh -f tsim/parser/limit.sim # ./test.sh -f tsim/parser/limit1.sim -## ./test.sh -f tsim/parser/limit1_stb.sim -## ./test.sh -f tsim/parser/limit1_tb.sim # ./test.sh -f tsim/parser/limit1_tblocks100.sim ## ./test.sh -f tsim/parser/limit2.sim -## ./test.sh -f tsim/parser/limit2_query.sim ## ./test.sh -f tsim/parser/limit2_tblocks100.sim ## ./test.sh -f tsim/parser/limit_stb.sim ## ./test.sh -f tsim/parser/limit_tb.sim @@ -169,20 +152,15 @@ # ./test.sh -f tsim/parser/select_with_tags.sim # ./test.sh -f tsim/parser/set_tag_vals.sim # ./test.sh -f tsim/parser/single_row_in_tb.sim -## ./test.sh -f tsim/parser/single_row_in_tb_query.sim # ./test.sh -f tsim/parser/sliding.sim # ./test.sh -f tsim/parser/slimit.sim # ./test.sh -f tsim/parser/slimit1.sim -## ./test.sh -f tsim/parser/slimit1_query.sim # ./test.sh -f tsim/parser/slimit_alter_tags.sim -## ./test.sh -f tsim/parser/slimit_query.sim # ./test.sh -f tsim/parser/stableOp.sim # ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim # ./test.sh -f tsim/parser/tags_filter.sim # ./test.sh -f tsim/parser/tbnameIn.sim -## ./test.sh -f tsim/parser/tbnameIn_query.sim # ./test.sh -f tsim/parser/timestamp.sim -## ./test.sh -f tsim/parser/timestamp_query.sim ## ./test.sh -f tsim/parser/top_groupby.sim # ./test.sh -f tsim/parser/topbot.sim # ./test.sh -f tsim/parser/udf.sim diff --git a/tests/script/tsim/parser/auto_create_tb_drop_tb.sim b/tests/script/tsim/parser/auto_create_tb_drop_tb.sim index b04d024643..0cff016b5f 100644 --- a/tests/script/tsim/parser/auto_create_tb_drop_tb.sim +++ b/tests/script/tsim/parser/auto_create_tb_drop_tb.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 -system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = db @@ -20,9 +16,9 @@ $i = 0 $db = $dbPrefix . $i $stb = $stbPrefix . $i -sql drop database $db -x step1 -step1: -sql create database $db maxrows 200 cache 2 +sql drop database if exists $db +sql create database $db + print ====== create tables sql use $db @@ -49,8 +45,6 @@ while $t < $tbNum endw print ====== tables created -sleep 100 - sql drop table tb2 $x = 0 while $x < $rowNum diff --git a/tests/script/tsim/parser/between_and.sim b/tests/script/tsim/parser/between_and.sim index cdced47cb6..aa9944d9a4 100644 --- a/tests/script/tsim/parser/between_and.sim +++ b/tests/script/tsim/parser/between_and.sim @@ -1,20 +1,14 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start - $db = testdb - sql create database $db sql use $db sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) - sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); sql create table tb2 using st2 tags (2,2.0,"2",2.0,2,2,"2"); sql create table tb3 using st2 tags (3,3.0,"3",3.0,3,3,"3"); @@ -25,132 +19,102 @@ sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") -sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") -sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") -sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") - -sql select tbname,id1 from st2; +sql insert into tb2 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb3 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb4 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") +sql select distinct(tbname), id1 from st2; if $rows != 4 then return -1 endi - sql select * from st2; - if $rows != 8 then return -1 endi sql select * from st2 where ts between now-50s and now+450s - if $rows != 5 then return -1 endi -sql select tbname,id1 from st2 where id1 between 2 and 3; - +sql select tbname, id1 from st2 where id1 between 2 and 3; if $rows != 2 then return -1 endi -if $data00 != tb2 then +sql select tbname, id2 from st2 where id2 between 0.0 and 3.0; +if $rows != 7 then return -1 endi -if $data01 != 2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data10 != tb3 then +if $data(tb2)[1] != 2.00000 then return -1 endi -if $data11 != 3 then +if $data(tb3)[0] != tb3 then + return -1 +endi +if $data(tb3)[1] != 3.00000 then return -1 endi -sql select tbname,id2 from st2 where id2 between 2.0 and 3.0; - +sql select tbname, id4 from st2 where id2 between 2.0 and 3.0; if $rows != 2 then return -1 endi - -if $data00 != tb2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data01 != 2.00000 then +if $data(tb2)[1] != 2.000000000 then return -1 endi -if $data10 != tb3 then +if $data(tb3)[0] != tb3 then return -1 endi -if $data11 != 3.00000 then +if $data(tb3)[1] != 3.000000000 then return -1 endi - -sql select tbname,id4 from st2 where id4 between 2.0 and 3.0; - +sql select tbname, id5 from st2 where id5 between 2.0 and 3.0; if $rows != 2 then return -1 endi - -if $data00 != tb2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data01 != 2.000000000 then +if $data(tb2)[1] != 2 then return -1 endi -if $data10 != tb3 then +if $data(tb3)[0] != tb3 then return -1 endi -if $data11 != 3.000000000 then - return -1 -endi - - -sql select tbname,id5 from st2 where id5 between 2.0 and 3.0; - -if $rows != 2 then - return -1 -endi - -if $data00 != tb2 then - return -1 -endi -if $data01 != 2 then - return -1 -endi -if $data10 != tb3 then - return -1 -endi -if $data11 != 3 then +if $data(tb3)[1] != 3 then return -1 endi sql select tbname,id6 from st2 where id6 between 2.0 and 3.0; - if $rows != 2 then return -1 endi - -if $data00 != tb2 then +if $data(tb2)[0] != tb2 then return -1 endi -if $data01 != 2 then +if $data(tb2)[1] != 2 then return -1 endi -if $data10 != tb3 then +if $data(tb3)[0] != tb3 then return -1 endi -if $data11 != 3 then +if $data(tb3)[1] != 3 then return -1 endi sql select * from st2 where f1 between 2 and 3 and f2 between 2.0 and 3.0 and f3 between 2.0 and 3.0 and f4 between 2.0 and 3.0 and f5 between 2.0 and 3.0 and f6 between 2.0 and 3.0; - if $rows != 2 then return -1 endi - if $data01 != 2 then return -1 endi @@ -158,8 +122,8 @@ if $data11 != 3 then return -1 endi -sql_error select * from st2 where f7 between 2.0 and 3.0; -sql_error select * from st2 where f8 between 2.0 and 3.0; -sql_error select * from st2 where f9 between 2.0 and 3.0; +sql select * from st2 where f7 between 2.0 and 3.0; +sql select * from st2 where f8 between 2.0 and 3.0; +sql select * from st2 where f9 between 2.0 and 3.0; system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/binary_escapeCharacter.sim b/tests/script/tsim/parser/binary_escapeCharacter.sim index 176edd4e29..0b437d8b04 100644 --- a/tests/script/tsim/parser/binary_escapeCharacter.sim +++ b/tests/script/tsim/parser/binary_escapeCharacter.sim @@ -1,10 +1,6 @@ - system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect sql drop database if exists ecdb diff --git a/tests/script/tsim/parser/col_arithmetic_operation.sim b/tests/script/tsim/parser/col_arithmetic_operation.sim index 8bb692e3bb..add2945c66 100644 --- a/tests/script/tsim/parser/col_arithmetic_operation.sim +++ b/tests/script/tsim/parser/col_arithmetic_operation.sim @@ -1,10 +1,8 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect + #========================================= setup environment ================================ $dbPrefix = ca_db @@ -101,19 +99,17 @@ $halfTbNum = $tbNum / 2 #endw #=================================== above are setup test environment ============================= -run general/parser/col_arithmetic_query.sim +run tsim/parser/col_arithmetic_query.sim #======================================= all in files query ======================================= print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 5000 -run general/parser/col_arithmetic_query.sim +run tsim/parser/col_arithmetic_query.sim # ================================================================================================ diff --git a/tests/script/tsim/parser/col_arithmetic_query.sim b/tests/script/tsim/parser/col_arithmetic_query.sim index 17ae6cfd6b..8ee891660b 100644 --- a/tests/script/tsim/parser/col_arithmetic_query.sim +++ b/tests/script/tsim/parser/col_arithmetic_query.sim @@ -173,7 +173,7 @@ endi # multi row result aggregation [d.4] sql_error select top(c1, 1) - bottom(c1, 1) from $tb sql_error select top(c1, 99) - bottom(c1, 99) from $tb -sql_error select top(c1,1) - 88 from $tb +sql select top(c1,1) - 88 from $tb # all data types [d.6] ================================================================ sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb @@ -227,13 +227,13 @@ endi # error case, ts/bool/binary/nchar not support arithmetic expression sql_error select ts+ts from $tb -sql_error select ts+22 from $tb -sql_error select c7*12 from $tb -sql_error select c8/55 from $tb -sql_error select c9+c8 from $tb -sql_error select c7-c8, c9-c8 from $tb +sql select ts+22 from $tb +sql select c7*12 from $tb +sql select c8/55 from $tb +sql select c9+c8 from $tb +sql select c7-c8, c9-c8 from $tb sql_error select ts-c9 from $tb -sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb +sql select c8+c7, c9+c9+c8+c7/c6 from $tb # arithmetic expression in join [d.7]================================================== @@ -339,7 +339,7 @@ if $data20 != 0 then endi # tag filter(not support for normal table). [d.15]===================================== -sql_error select c2+99 from $tb where t1=12; +sql select c2+99 from $tb where t1=12; # multi-field output [d.16]============================================================ sql select c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2 from $tb @@ -391,7 +391,7 @@ if $data00 != 0.000000000 then return -1 endi -sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb order by ts desc; +sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb if $rows != 1 then return -1 endi @@ -408,7 +408,7 @@ if $data02 != 200000.000000000 then return -1 endi -sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb order by ts asc; +sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb if $rows != 1 then return -1 endi @@ -487,8 +487,8 @@ sql_error select top(c1, 99) - bottom(c1, 99) from $stb sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $stb # error case, ts/bool/binary/nchar not support arithmetic expression -sql_error select first(c7)*12 from $stb -sql_error select last(c8)/55 from $stb +sql select first(c7)*12 from $stb +sql select last(c8)/55 from $stb sql_error select last_row(c9) + last_row(c8) from $stb # arithmetic expression in join [d.7]=============================================================== diff --git a/tests/script/tsim/parser/columnValue.sim b/tests/script/tsim/parser/columnValue.sim index c98542fbf2..68336cdcc1 100644 --- a/tests/script/tsim/parser/columnValue.sim +++ b/tests/script/tsim/parser/columnValue.sim @@ -1,10 +1,6 @@ -#### system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect print ========== columnValues.sim @@ -13,14 +9,14 @@ sql drop database if exists db sql create database db sql use db -run general/parser/columnValue_bool.sim -run general/parser/columnValue_tinyint.sim -run general/parser/columnValue_smallint.sim -run general/parser/columnValue_int.sim -run general/parser/columnValue_bigint.sim -run general/parser/columnValue_float.sim -run general/parser/columnValue_double.sim -run general/parser/columnValue_unsign.sim +run tsim/parser/columnValue_bool.sim +run tsim/parser/columnValue_tinyint.sim +run tsim/parser/columnValue_smallint.sim +run tsim/parser/columnValue_int.sim +run tsim/parser/columnValue_bigint.sim +run tsim/parser/columnValue_float.sim +run tsim/parser/columnValue_double.sim +run tsim/parser/columnValue_unsign.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index 7c4c883fb1..83b457673b 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxTablesperVnode -v 100 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sc_db diff --git a/tests/script/tsim/parser/condition.sim b/tests/script/tsim/parser/condition.sim index c3aed7e2a3..8c1327baae 100644 --- a/tests/script/tsim/parser/condition.sim +++ b/tests/script/tsim/parser/condition.sim @@ -1,14 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 -system sh/cfg.sh -n dnode1 -c cache -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect - sql drop database if exists cdb sql create database if not exists cdb sql use cdb @@ -139,7 +132,7 @@ sleep 100 sql connect -run general/parser/condition_query.sim +run tsim/parser/condition_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -149,5 +142,5 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/condition_query.sim +run tsim/parser/condition_query.sim diff --git a/tests/script/tsim/parser/constCol.sim b/tests/script/tsim/parser/constCol.sim index 66523517be..5f50c950dd 100644 --- a/tests/script/tsim/parser/constCol.sim +++ b/tests/script/tsim/parser/constCol.sim @@ -1,26 +1,5 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 - -system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 -system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 -system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 - -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 - system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index 040331ec4f..c4c5b89bd2 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/create_db__for_community_version.sim b/tests/script/tsim/parser/create_db__for_community_version.sim index 5dc4263d5d..32a8f303c1 100644 --- a/tests/script/tsim/parser/create_db__for_community_version.sim +++ b/tests/script/tsim/parser/create_db__for_community_version.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/create_mt.sim b/tests/script/tsim/parser/create_mt.sim index c606ba99ec..fafee66c76 100644 --- a/tests/script/tsim/parser/create_mt.sim +++ b/tests/script/tsim/parser/create_mt.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/create_tb.sim b/tests/script/tsim/parser/create_tb.sim index ca57f401b9..5203f289dc 100644 --- a/tests/script/tsim/parser/create_tb.sim +++ b/tests/script/tsim/parser/create_tb.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/create_tb_with_tag_name.sim b/tests/script/tsim/parser/create_tb_with_tag_name.sim index 130f4097f6..b7b39b2f5f 100644 --- a/tests/script/tsim/parser/create_tb_with_tag_name.sim +++ b/tests/script/tsim/parser/create_tb_with_tag_name.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/dbtbnameValidate.sim b/tests/script/tsim/parser/dbtbnameValidate.sim index bc3bfefafb..86ffbe5c37 100644 --- a/tests/script/tsim/parser/dbtbnameValidate.sim +++ b/tests/script/tsim/parser/dbtbnameValidate.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect print ========== db name and table name check in create and drop, describe diff --git a/tests/script/tsim/parser/distinct.sim b/tests/script/tsim/parser/distinct.sim index e0eb74b5a5..b90ca593ba 100644 --- a/tests/script/tsim/parser/distinct.sim +++ b/tests/script/tsim/parser/distinct.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sav_db diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index 3413a0b596..642c7bd8d4 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_fl_db diff --git a/tests/script/tsim/parser/fill_stb.sim b/tests/script/tsim/parser/fill_stb.sim index ba8ddbdf6a..0aadcc5a9f 100644 --- a/tests/script/tsim/parser/fill_stb.sim +++ b/tests/script/tsim/parser/fill_stb.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = fl1_db diff --git a/tests/script/tsim/parser/fill_us.sim b/tests/script/tsim/parser/fill_us.sim index 762413d0a1..98c37c435d 100644 --- a/tests/script/tsim/parser/fill_us.sim +++ b/tests/script/tsim/parser/fill_us.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_fl_db diff --git a/tests/script/tsim/parser/first_last.sim b/tests/script/tsim/parser/first_last.sim index 09047b4528..27bf42ead3 100644 --- a/tests/script/tsim/parser/first_last.sim +++ b/tests/script/tsim/parser/first_last.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxTablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = first_db @@ -73,7 +69,7 @@ sql import into $tb (ts) values ( $ts ) print ====== test data created -run general/parser/first_last_query.sim +run tsim/parser/first_last_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -83,7 +79,7 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/first_last_query.sim +run tsim/parser/first_last_query.sim print =================> insert data regression test sql create database test keep 36500 diff --git a/tests/script/tsim/parser/fourArithmetic-basic.sim b/tests/script/tsim/parser/fourArithmetic-basic.sim index bd01813c61..bfda75e54d 100644 --- a/tests/script/tsim/parser/fourArithmetic-basic.sim +++ b/tests/script/tsim/parser/fourArithmetic-basic.sim @@ -1,25 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start - -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi - sql connect $dbNamme = d0 diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 556292b21b..451947e82a 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = m_func_db diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index 4d6b33612f..1edc99624a 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -1,9 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start +sql connect $loop_cnt = 0 check_dnode_ready: diff --git a/tests/script/tsim/parser/groupby.sim b/tests/script/tsim/parser/groupby.sim index 1fe19714bb..8d7fad8cbc 100644 --- a/tests/script/tsim/parser/groupby.sim +++ b/tests/script/tsim/parser/groupby.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = group_db diff --git a/tests/script/tsim/parser/having.sim b/tests/script/tsim/parser/having.sim index e063333853..b3d64a4e3d 100644 --- a/tests/script/tsim/parser/having.sim +++ b/tests/script/tsim/parser/having.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index 0fe5448869..1ee1481943 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/import.sim b/tests/script/tsim/parser/import.sim index 4468ab87a9..5946cff4e2 100644 --- a/tests/script/tsim/parser/import.sim +++ b/tests/script/tsim/parser/import.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = impt_db diff --git a/tests/script/tsim/parser/import_commit1.sim b/tests/script/tsim/parser/import_commit1.sim index f330fe4cd9..23259d8b01 100644 --- a/tests/script/tsim/parser/import_commit1.sim +++ b/tests/script/tsim/parser/import_commit1.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = ic_db diff --git a/tests/script/tsim/parser/import_commit2.sim b/tests/script/tsim/parser/import_commit2.sim index 47b30acb49..49fca0d477 100644 --- a/tests/script/tsim/parser/import_commit2.sim +++ b/tests/script/tsim/parser/import_commit2.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = ic_db diff --git a/tests/script/tsim/parser/import_commit3.sim b/tests/script/tsim/parser/import_commit3.sim index 1e041375de..d353c10387 100644 --- a/tests/script/tsim/parser/import_commit3.sim +++ b/tests/script/tsim/parser/import_commit3.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = ic_db diff --git a/tests/script/tsim/parser/import_file.sim b/tests/script/tsim/parser/import_file.sim index cf11194ba7..35b656eb87 100644 --- a/tests/script/tsim/parser/import_file.sim +++ b/tests/script/tsim/parser/import_file.sim @@ -1,11 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 sql connect -sleep 500 sql drop database if exists indb sql create database if not exists indb @@ -13,7 +9,7 @@ sql use indb $inFileName = '~/data.csv' $numOfRows = 10000 -system general/parser/gendata.sh +system tsim/parser/gendata.sh sql create table stbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16), imei BINARY(16), mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2)) tags(a int, b binary(12)); diff --git a/tests/script/tsim/parser/insert_multiTbl.sim b/tests/script/tsim/parser/insert_multiTbl.sim index b17323280e..85c58ef3d3 100644 --- a/tests/script/tsim/parser/insert_multiTbl.sim +++ b/tests/script/tsim/parser/insert_multiTbl.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 500 sql connect -sleep 100 + print ======================== dnode1 start sql create database mul_db diff --git a/tests/script/tsim/parser/insert_tb.sim b/tests/script/tsim/parser/insert_tb.sim index 1e431aef3d..4fa04e0625 100644 --- a/tests/script/tsim/parser/insert_tb.sim +++ b/tests/script/tsim/parser/insert_tb.sim @@ -1,12 +1,8 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/interp.sim b/tests/script/tsim/parser/interp.sim index f192837bb7..4bb273af46 100644 --- a/tests/script/tsim/parser/interp.sim +++ b/tests/script/tsim/parser/interp.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = intp_db @@ -58,7 +55,7 @@ print ====== tables created sql create table ap1 (ts timestamp, pav float); sql INSERT INTO ap1 VALUES ('2021-07-25 02:19:54.100',1) ('2021-07-25 02:19:54.200',2) ('2021-07-25 02:19:54.300',3) ('2021-07-25 02:19:56.500',4) ('2021-07-25 02:19:57.500',5) ('2021-07-25 02:19:57.600',6) ('2021-07-25 02:19:57.900',7) ('2021-07-25 02:19:58.100',8) ('2021-07-25 02:19:58.300',9) ('2021-07-25 02:19:59.100',10) ('2021-07-25 02:19:59.300',11) ('2021-07-25 02:19:59.500',12) ('2021-07-25 02:19:59.700',13) ('2021-07-25 02:19:59.900',14) ('2021-07-25 02:20:05.000', 20) ('2021-07-25 02:25:00.000', 10000); -run general/parser/interp_test.sim +run tsim/parser/interp_test.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -66,7 +63,7 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/interp_test.sim +run tsim/parser/interp_test.sim print ================= TD-5931 sql create stable st5931(ts timestamp, f int) tags(t int) diff --git a/tests/script/tsim/parser/join.sim b/tests/script/tsim/parser/join.sim index e2132589bd..55842d5c16 100644 --- a/tests/script/tsim/parser/join.sim +++ b/tests/script/tsim/parser/join.sim @@ -1,13 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c debugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 - system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = join_db diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index fddd59c0a1..eb5e34b079 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -1,12 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 - system sh/exec.sh -n dnode1 -s start sql connect -sleep 100 $dbPrefix = join_m_db $tbPrefix = join_tb diff --git a/tests/script/tsim/parser/join_multitables.sim b/tests/script/tsim/parser/join_multitables.sim index d675499640..7a1c77acff 100644 --- a/tests/script/tsim/parser/join_multitables.sim +++ b/tests/script/tsim/parser/join_multitables.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/join_multivnode.sim b/tests/script/tsim/parser/join_multivnode.sim index 61438241b0..a204b4fcea 100644 --- a/tests/script/tsim/parser/join_multivnode.sim +++ b/tests/script/tsim/parser/join_multivnode.sim @@ -1,12 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 - system sh/exec.sh -n dnode1 -s start sql connect -sleep 100 $dbPrefix = join_m_db $tbPrefix = join_tb diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 9c414263ec..5f45b64796 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb @@ -57,13 +54,13 @@ sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005) sql insert into tbc values ("2021-05-11 10:12:29",36, 37, NULL, -4005) sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) -run general/parser/last_cache_query.sim +run tsim/parser/last_cache_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -run general/parser/last_cache_query.sim +run tsim/parser/last_cache_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_groupby.sim b/tests/script/tsim/parser/last_groupby.sim index f993324cd1..8f9574412d 100644 --- a/tests/script/tsim/parser/last_groupby.sim +++ b/tests/script/tsim/parser/last_groupby.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/lastrow.sim b/tests/script/tsim/parser/lastrow.sim index 7cdd04e2cc..d6638f2e98 100644 --- a/tests/script/tsim/parser/lastrow.sim +++ b/tests/script/tsim/parser/lastrow.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = lr_db @@ -58,7 +54,7 @@ endw print ====== test data created -run general/parser/lastrow_query.sim +run tsim/parser/lastrow_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -68,7 +64,7 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/lastrow_query.sim +run tsim/parser/lastrow_query.sim print =================== last_row + nested query sql use $db diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index fce996ebee..3d1bff6abf 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start - -sleep 10 sql connect + print ======================== dnode1 start diff --git a/tests/script/tsim/parser/limit.sim b/tests/script/tsim/parser/limit.sim index 3af2cb3018..f4a23697cd 100644 --- a/tests/script/tsim/parser/limit.sim +++ b/tests/script/tsim/parser/limit.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = lm_db @@ -61,8 +57,8 @@ while $i < $halfNum endw print ====== tables created -run general/parser/limit_tb.sim -run general/parser/limit_stb.sim +run tsim/parser/limit_tb.sim +run tsim/parser/limit_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -72,8 +68,8 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/limit_tb.sim -run general/parser/limit_stb.sim +run tsim/parser/limit_tb.sim +run tsim/parser/limit_stb.sim print ========> TD-6017 sql use $db diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index e37bea9220..1f72999784 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = lm1_db @@ -56,8 +52,8 @@ while $i < $halfNum endw print ====== tables created -run general/parser/limit1_tb.sim -run general/parser/limit1_stb.sim +run tsim/parser/limit1_tb.sim +run tsim/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -65,7 +61,7 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/limit1_tb.sim -run general/parser/limit1_stb.sim +run tsim/parser/limit1_tb.sim +run tsim/parser/limit1_stb.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/limit1_tblocks100.sim b/tests/script/tsim/parser/limit1_tblocks100.sim index 4546ffdb79..f541ea7158 100644 --- a/tests/script/tsim/parser/limit1_tblocks100.sim +++ b/tests/script/tsim/parser/limit1_tblocks100.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = lm1_db @@ -56,8 +52,8 @@ while $i < $halfNum endw print ====== tables created -run general/parser/limit1_tb.sim -run general/parser/limit1_stb.sim +run tsim/parser/limit1_tb.sim +run tsim/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -65,7 +61,7 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/limit1_tb.sim -run general/parser/limit1_stb.sim +run tsim/parser/limit1_tb.sim +run tsim/parser/limit1_stb.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/limit2.sim b/tests/script/tsim/parser/limit2.sim index 336f9234c8..af03c6bb7f 100644 --- a/tests/script/tsim/parser/limit2.sim +++ b/tests/script/tsim/parser/limit2.sim @@ -65,7 +65,7 @@ while $i < $halfNum endw print ====== tables created -#run general/parser/limit2_query.sim +#run tsim/parser/limit2_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -73,6 +73,6 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/limit2_query.sim +run tsim/parser/limit2_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/limit2_tblocks100.sim b/tests/script/tsim/parser/limit2_tblocks100.sim index 11f7a15eb0..0d87a41838 100644 --- a/tests/script/tsim/parser/limit2_tblocks100.sim +++ b/tests/script/tsim/parser/limit2_tblocks100.sim @@ -65,7 +65,7 @@ while $i < $halfNum endw print ====== tables created -#run general/parser/limit2_query.sim +#run tsim/parser/limit2_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -73,4 +73,4 @@ sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/limit2_query.sim +run tsim/parser/limit2_query.sim diff --git a/tests/script/tsim/parser/line_insert.sim b/tests/script/tsim/parser/line_insert.sim index 85f2714ad3..cbd960bed6 100644 --- a/tests/script/tsim/parser/line_insert.sim +++ b/tests/script/tsim/parser/line_insert.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print =============== step1 diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index c20cf9a915..50229ab35a 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -1,11 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect $dbPrefix = mb_db diff --git a/tests/script/tsim/parser/nchar.sim b/tests/script/tsim/parser/nchar.sim index 84719efcab..52fc8b6864 100644 --- a/tests/script/tsim/parser/nchar.sim +++ b/tests/script/tsim/parser/nchar.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect print ======================== dnode1 start diff --git a/tests/script/tsim/parser/nestquery.sim b/tests/script/tsim/parser/nestquery.sim index 3c1ba03369..c82718c1cb 100644 --- a/tests/script/tsim/parser/nestquery.sim +++ b/tests/script/tsim/parser/nestquery.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect print ======================== dnode1 start diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index cb65290d25..2bdb960968 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -1,11 +1,6 @@ -#### TBASE-679 system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect print ========== NULL_char.sim diff --git a/tests/script/tsim/parser/precision_ns.sim b/tests/script/tsim/parser/precision_ns.sim index 3e9a2dd3ff..bb822cd2b1 100644 --- a/tests/script/tsim/parser/precision_ns.sim +++ b/tests/script/tsim/parser/precision_ns.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 1000 sql connect $dbPrefix = m_di_db_ns diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index ffbcb28ffd..37f2e79995 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = group_db diff --git a/tests/script/tsim/parser/regex.sim b/tests/script/tsim/parser/regex.sim index eed36018d4..41f52575d6 100644 --- a/tests/script/tsim/parser/regex.sim +++ b/tests/script/tsim/parser/regex.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect $db = testdb diff --git a/tests/script/tsim/parser/repeatAlter.sim b/tests/script/tsim/parser/repeatAlter.sim index 3695206f90..d28a03e193 100644 --- a/tests/script/tsim/parser/repeatAlter.sim +++ b/tests/script/tsim/parser/repeatAlter.sim @@ -2,7 +2,7 @@ $i = 1 $loops = 10 while $i <= $loops print ====== repeat: $i - run general/parser/alter.sim + run tsim/parser/alter.sim $i = $i + 1 endw diff --git a/tests/script/tsim/parser/selectResNum.sim b/tests/script/tsim/parser/selectResNum.sim index dfd204e152..ac5ccd6e07 100644 --- a/tests/script/tsim/parser/selectResNum.sim +++ b/tests/script/tsim/parser/selectResNum.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sc_db diff --git a/tests/script/tsim/parser/select_across_vnodes.sim b/tests/script/tsim/parser/select_across_vnodes.sim index 9bf61ee61d..0ee011cf8a 100644 --- a/tests/script/tsim/parser/select_across_vnodes.sim +++ b/tests/script/tsim/parser/select_across_vnodes.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sav_db diff --git a/tests/script/tsim/parser/select_distinct_tag.sim b/tests/script/tsim/parser/select_distinct_tag.sim index d8e92d4bc5..92303ce64e 100644 --- a/tests/script/tsim/parser/select_distinct_tag.sim +++ b/tests/script/tsim/parser/select_distinct_tag.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sav_db diff --git a/tests/script/tsim/parser/select_from_cache_disk.sim b/tests/script/tsim/parser/select_from_cache_disk.sim index 7f8af52c6b..2c9f359afe 100644 --- a/tests/script/tsim/parser/select_from_cache_disk.sim +++ b/tests/script/tsim/parser/select_from_cache_disk.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = scd_db diff --git a/tests/script/tsim/parser/select_with_tags.sim b/tests/script/tsim/parser/select_with_tags.sim index eb6cd75d21..b840a666f4 100644 --- a/tests/script/tsim/parser/select_with_tags.sim +++ b/tests/script/tsim/parser/select_with_tags.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = select_tags_db diff --git a/tests/script/tsim/parser/set_tag_vals.sim b/tests/script/tsim/parser/set_tag_vals.sim index 4a63f9c6f1..07b424ec6a 100644 --- a/tests/script/tsim/parser/set_tag_vals.sim +++ b/tests/script/tsim/parser/set_tag_vals.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = db diff --git a/tests/script/tsim/parser/single_row_in_tb.sim b/tests/script/tsim/parser/single_row_in_tb.sim index 5de2a51f0f..59a0552809 100644 --- a/tests/script/tsim/parser/single_row_in_tb.sim +++ b/tests/script/tsim/parser/single_row_in_tb.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sr_db @@ -28,7 +24,7 @@ sql create table $tb1 using $stb tags( 1 ) sql insert into $tb1 values ( $ts0 , 1, 2, 3, 4, true, 'binay10', '涛思nchar10' ) print ====== tables created -run general/parser/single_row_in_tb_query.sim +run tsim/parser/single_row_in_tb_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -36,6 +32,6 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/single_row_in_tb_query.sim +run tsim/parser/single_row_in_tb_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index b2695ff95f..18d7bda8a1 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -1,12 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c debugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = sliding_db diff --git a/tests/script/tsim/parser/slimit.sim b/tests/script/tsim/parser/slimit.sim index 0af31f9826..9ca5da678a 100644 --- a/tests/script/tsim/parser/slimit.sim +++ b/tests/script/tsim/parser/slimit.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = slm_db @@ -93,7 +89,7 @@ while $i < $tbNum endw print ====== $db tables created -run general/parser/slimit_query.sim +run tsim/parser/slimit_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -103,6 +99,6 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/slimit_query.sim +run tsim/parser/slimit_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/slimit1.sim b/tests/script/tsim/parser/slimit1.sim index 2dede439ec..bb12bc32f1 100644 --- a/tests/script/tsim/parser/slimit1.sim +++ b/tests/script/tsim/parser/slimit1.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = slm_alt_tg_db @@ -52,7 +48,7 @@ endw print ================== tables and data created -run general/parser/slimit1_query.sim +run tsim/parser/slimit1_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -62,6 +58,6 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/slimit1_query.sim +run tsim/parser/slimit1_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/stableOp.sim b/tests/script/tsim/parser/stableOp.sim index 8647657e7b..4fe0a6f38d 100644 --- a/tests/script/tsim/parser/stableOp.sim +++ b/tests/script/tsim/parser/stableOp.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $dbPrefix = fi_in_db diff --git a/tests/script/tsim/parser/tags_dynamically_specifiy.sim b/tests/script/tsim/parser/tags_dynamically_specifiy.sim index f6b3dabf15..d1f73c4f60 100644 --- a/tests/script/tsim/parser/tags_dynamically_specifiy.sim +++ b/tests/script/tsim/parser/tags_dynamically_specifiy.sim @@ -1,11 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect -sleep 100 $db = dytag_db $tbNum = 10 diff --git a/tests/script/tsim/parser/tags_filter.sim b/tests/script/tsim/parser/tags_filter.sim index 3d3e79b6f5..bf33febdae 100644 --- a/tests/script/tsim/parser/tags_filter.sim +++ b/tests/script/tsim/parser/tags_filter.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $db = tf_db diff --git a/tests/script/tsim/parser/tbnameIn.sim b/tests/script/tsim/parser/tbnameIn.sim index 003a86f90b..e9206b59e2 100644 --- a/tests/script/tsim/parser/tbnameIn.sim +++ b/tests/script/tsim/parser/tbnameIn.sim @@ -1,9 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = ti_db @@ -63,7 +60,7 @@ while $i < $halfNum endw print ====== tables created -run general/parser/tbnameIn_query.sim +run tsim/parser/tbnameIn_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -71,6 +68,6 @@ sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -run general/parser/tbnameIn_query.sim +run tsim/parser/tbnameIn_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/timestamp.sim b/tests/script/tsim/parser/timestamp.sim index 0a87bce51d..524f6d5de3 100644 --- a/tests/script/tsim/parser/timestamp.sim +++ b/tests/script/tsim/parser/timestamp.sim @@ -1,11 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 - system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = ts_db @@ -55,7 +50,7 @@ while $i < $tbNum endw print ====== $db tables created -run general/parser/timestamp_query.sim +run tsim/parser/timestamp_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -65,4 +60,4 @@ print ================== server restart completed sql connect sleep 100 -run general/parser/timestamp_query.sim +run tsim/parser/timestamp_query.sim diff --git a/tests/script/tsim/parser/top_groupby.sim b/tests/script/tsim/parser/top_groupby.sim index 5709f4d1d7..ff479a1494 100644 --- a/tests/script/tsim/parser/top_groupby.sim +++ b/tests/script/tsim/parser/top_groupby.sim @@ -1,11 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect + print ======================== dnode1 start $db = testdb diff --git a/tests/script/tsim/parser/topbot.sim b/tests/script/tsim/parser/topbot.sim index ddab1b3f83..61b2db2862 100644 --- a/tests/script/tsim/parser/topbot.sim +++ b/tests/script/tsim/parser/topbot.sim @@ -1,11 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 - system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = tb_db diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 0cd3cba84d..4d05d4ced7 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -1,13 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c debugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 - system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = union_db diff --git a/tests/script/tsim/parser/where.sim b/tests/script/tsim/parser/where.sim index 6b789de490..77eb3fd87e 100644 --- a/tests/script/tsim/parser/where.sim +++ b/tests/script/tsim/parser/where.sim @@ -1,11 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start - -sleep 100 sql connect $dbPrefix = wh_db @@ -62,7 +57,7 @@ $i = 1 $tb = $tbPrefix . $i ## -sql_error select * from $tb where c7 +sql select * from $tb where c7 # TBASE-654 : invalid filter expression cause server crashed sql select count(*) from $tb where c1<10 and c1<>2 @@ -73,7 +68,6 @@ if $data00 != 900 then return -1 endi - sql select * from $tb where c7 = false $val = $rowNum / 100 if $rows != $val then From f60f821c13a1508cf8a8182e11f31b071895e601 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 15:04:22 +0800 Subject: [PATCH 46/94] fix(query): fix partition by bug for last_row query. --- source/libs/executor/src/cachescanoperator.c | 27 +++++++++++++++----- source/libs/planner/src/planOptimizer.c | 3 ++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 78a7e58ee1..395b42aa86 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -139,18 +139,29 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { colDataAppend(pDst, 0, p, isNull); } + pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + pInfo->pRes->info.rows = 1; + if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } - pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); - int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); - pInfo->pRes->info.groupId = *groupId; + if (pTableList->map != NULL) { + int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); + pInfo->pRes->info.groupId = *groupId; + } else { + ASSERT(taosArrayGetSize(pTableList->pTableList) == 1); + STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0); + pInfo->pRes->info.groupId = pKeyInfo->groupId; + } pInfo->indexOfBufferedRes += 1; - pInfo->pRes->info.rows = 1; return pInfo->pRes; } else { doSetOperatorCompleted(pOperator); @@ -182,8 +193,12 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0); pInfo->pRes->info.groupId = pKeyInfo->groupId; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } tsdbLastrowReaderClose(pInfo->pLastrowReader); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 97858f83fb..2d05f6a16a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1993,7 +1993,8 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { SNode* pFunc = NULL; FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType && - FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType) { + FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType && + FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pFunc)->funcType) { return false; } } From 9e49a0012d603aebd9c3123a1fb947ad5eebeb7e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 15:52:32 +0800 Subject: [PATCH 47/94] feat(query): add percentile/apercentile scalar version TD-17344 --- include/libs/scalar/scalar.h | 2 ++ source/libs/function/src/builtins.c | 2 ++ source/libs/scalar/src/sclfunc.c | 31 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index d5e9a2e625..df3e27225f 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -104,6 +104,8 @@ int32_t maxScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6fb808a2e8..e274146d25 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2034,6 +2034,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, .processFunc = percentileFunction, + .sprocessFunc = percentileScalarFunction, .finalizeFunc = percentileFinalize, .invertFunc = NULL, .combineFunc = NULL, @@ -2046,6 +2047,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunction, + .sprocessFunc = apercentileScalarFunction, .finalizeFunc = apercentileFinalize, .invertFunc = NULL, .combineFunc = apercentileCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ba35356a9c..8808d17454 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2307,3 +2307,34 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + double val; + bool hasNull = false; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + char *in = pInputData->pData; + GET_TYPED_DATA(val, double, type, in); + } + + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + colDataAppend(pOutputData, 0, (char *)&val, false); + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + +int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return percentileScalarFunction(pInput, inputNum, pOutput); +} From a12f6a3e3d6d7c575b665b4af38e9bf22772e578 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:03:16 +0800 Subject: [PATCH 48/94] refactor: let transfer leader msg to be sent even when the dnode exiting --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 84491a82b0..49207225a5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -148,9 +148,9 @@ static int32_t mmStart(SMnodeMgmt *pMgmt) { static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); + mndPreClose(pMgmt->pMnode); taosThreadRwlockWrlock(&pMgmt->lock); pMgmt->stopped = 1; - mndPreClose(pMgmt->pMnode); taosThreadRwlockUnlock(&pMgmt->lock); mndStop(pMgmt->pMnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 9052fb20ca..7efa46c514 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -221,11 +221,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); - if (pDnode->status != DND_STAT_RUNNING) { + if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; terrno = TSDB_CODE_NODE_OFFLINE; - dError("failed to send rpc msg since %s, handle:%p", terrstr(), pMsg->info.handle); + dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); From 59c7348c948a3abf931ad1bc59dc58d5a3203444 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 16:07:45 +0800 Subject: [PATCH 49/94] refactor(sync): add leader transfer callback --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 387400c25e..b6c01f2923 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2601,6 +2601,10 @@ const char* syncStr(ESyncState state) { int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { + syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); + return 0; + } syncNodeEventLog(ths, "do leader transfer"); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); From f32d2bde1f6a11ff7be2dab98f6e255fa0621f32 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 16:07:54 +0800 Subject: [PATCH 50/94] feat: add time unsynced check --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0e21273305..6969e03e7c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -293,7 +293,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFreeClear(param); if (code != 0) { - (*pInst)->onlineDnodes = 0; + (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); } if (rspNum) { From ca7c1151490e7e73979edb04ba63c2de5ee02522 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:13:08 +0800 Subject: [PATCH 51/94] refactor: get retry epset from sync module only when mnode is not initialized --- source/dnode/mnode/impl/src/mndMnode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d7eaa72202..13655ac21f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -218,7 +218,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { -#if 0 SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); void *pIter = NULL; @@ -238,9 +237,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); sdbRelease(pSdb, pObj); } -#else - syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); -#endif + + if (pEpSet->numOfEps == 0) { + syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { From aec4c2313a8bcb4a08f42b02f89525d0a253fdfb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 16:17:00 +0800 Subject: [PATCH 52/94] fix: multi-rows function validity check --- include/libs/function/functionMgt.h | 3 +- source/libs/function/inc/functionMgtInt.h | 7 ++-- source/libs/function/src/builtins.c | 22 +++++----- source/libs/function/src/functionMgt.c | 6 +-- source/libs/parser/src/parTranslater.c | 49 ++++++++++------------- source/libs/parser/test/parSelectTest.cpp | 12 +++--- 6 files changed, 45 insertions(+), 54 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 299d808017..d55adcb63c 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -190,14 +190,13 @@ bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); bool fmIsForbidFillFunc(int32_t funcId); bool fmIsForbidStreamFunc(int32_t funcId); -bool fmIsForbidWindowFunc(int32_t funcId); -bool fmIsForbidGroupByFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); bool fmIsClientPseudoColumnFunc(int32_t funcId); +bool fmIsMultiRowsFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index da5dd0433d..0b4fdefec4 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -44,10 +44,9 @@ extern "C" { #define FUNC_MGT_FORBID_FILL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(15) #define FUNC_MGT_INTERVAL_INTERPO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(16) #define FUNC_MGT_FORBID_STREAM_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(17) -#define FUNC_MGT_FORBID_WINDOW_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) -#define FUNC_MGT_FORBID_GROUP_BY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) -#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) -#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21) +#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) +#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) +#define FUNC_MGT_MULTI_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 747b1c8d9f..e690a85d1c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2078,7 +2078,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2092,7 +2092,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2319,7 +2319,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2395,7 +2395,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -2405,7 +2405,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2415,7 +2415,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2425,7 +2425,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -2435,7 +2435,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -2445,7 +2445,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -2456,7 +2456,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "tail", .type = FUNCTION_TYPE_TAIL, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | - FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -2467,7 +2467,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f73a61fa60..ff74c880e3 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -175,16 +175,14 @@ bool fmIsIntervalInterpoFunc(int32_t funcId) { return isSpecificClassifyFunc(fun bool fmIsForbidStreamFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC); } -bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_WINDOW_FUNC); } - -bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); } - bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); } bool fmIsImplicitTsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IMPLICIT_TS_FUNC); } bool fmIsClientPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_CLIENT_PC_FUNC); } +bool fmIsMultiRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_ROWS_FUNC); } + bool fmIsInterpFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c88e38ddff..0d2959f21d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1096,6 +1096,24 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + if (NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow || NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s function is not supported in window query or group query", pFunc->functionName); + } + if (hasInvalidFuncNesting(pFunc->pParameterList)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsMultiRowsFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + } if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } @@ -1131,16 +1149,6 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio return TSDB_CODE_SUCCESS; } -static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidWindowFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsForbidStreamFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1151,16 +1159,6 @@ static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* return TSDB_CODE_SUCCESS; } -static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidGroupByFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1191,7 +1189,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (SQL_CLAUSE_SELECT != pCxt->currClause ) { + if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (isStar(pPara) || isTableStar(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, @@ -1322,21 +1320,18 @@ static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateWindowPseudoColumnFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidWindowFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidGroupByFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateRepeatScanFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { code = translateMultiResFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateMultiRowsFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { setFuncClassification(pCxt->pCurrStmt, pFunc); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index e682059793..b9a760d342 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -152,9 +152,9 @@ TEST_F(ParserSelectTest, IndefiniteRowsFuncSemanticCheck) { run("SELECT DIFF(c1), CSUM(c1) FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, useDefinedFunc) { @@ -178,9 +178,9 @@ TEST_F(ParserSelectTest, uniqueFunc) { TEST_F(ParserSelectTest, uniqueFuncSemanticCheck) { useDb("root", "test"); - run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, tailFunc) { @@ -194,9 +194,9 @@ TEST_F(ParserSelectTest, tailFunc) { TEST_F(ParserSelectTest, tailFuncSemanticCheck) { useDb("root", "test"); - run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, partitionBy) { From 4d53c09cb69ee4c0aefc4268ee03436dfbf04c9b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:24:46 +0800 Subject: [PATCH 53/94] test: valgrind case --- tests/script/tsim/valgrind/checkError3.sim | 38 ++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 10a8f01fb3..784b83d96b 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode1 -s start sql connect print =============== step1: create drop show dnodes @@ -56,28 +56,32 @@ print =============== step6: alter insert sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +print =============== restart +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start -v + print =============== stepa: query data sql select * from c1 -sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 -sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 +#sql select * from stb +#sql select * from stb_1 +#sql select ts, c1, c2, c3 from c1 +#sql select ts, c1, c2, c3 from stb +#sql select ts, c1 from stb_2 +#sql select ts, c1, t1 from c1 +#sql select ts, c1, t1 from stb +#sql select ts, c1, t1 from stb_2 print =============== stepb: count -sql select count(*) from c1; -sql select count(*) from stb; -sql select count(ts), count(c1), count(c2), count(c3) from c1 -sql select count(ts), count(c1), count(c2), count(c3) from stb +#sql select count(*) from c1; +#sql select count(*) from stb; +#sql select count(ts), count(c1), count(c2), count(c3) from c1 +#sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -sql select first(ts), first(c1), first(c2), first(c3) from c1 -sql select min(c2), min(c3), min(c4) from c1 -sql select max(c2), max(c3), max(c4) from c1 -sql select sum(c2), sum(c3), sum(c4) from c1 +#sql select first(ts), first(c1), first(c2), first(c3) from c1 +#sql select min(c2), min(c3), min(c4) from c1 +#sql select max(c2), max(c3), max(c4) from c1 +#sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 040f20be43458febcf81b20f12a5fcc161942bec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:28:31 +0800 Subject: [PATCH 54/94] test: valgrind case --- tests/script/tsim/valgrind/basic1.sim | 32 +++++++++++----------- tests/script/tsim/valgrind/checkError3.sim | 32 +++++++++++----------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 784b83d96b..86926509bb 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -62,26 +62,26 @@ system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data sql select * from c1 -#sql select * from stb -#sql select * from stb_1 -#sql select ts, c1, c2, c3 from c1 -#sql select ts, c1, c2, c3 from stb -#sql select ts, c1 from stb_2 -#sql select ts, c1, t1 from c1 -#sql select ts, c1, t1 from stb -#sql select ts, c1, t1 from stb_2 +sql select * from stb +sql select * from stb_1 +sql select ts, c1, c2, c3 from c1 +sql select ts, c1, c2, c3 from stb +sql select ts, c1 from stb_2 +sql select ts, c1, t1 from c1 +sql select ts, c1, t1 from stb +sql select ts, c1, t1 from stb_2 print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from c1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from c1 +sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c2), min(c3), min(c4) from c1 -#sql select max(c2), max(c3), max(c4) from c1 -#sql select sum(c2), sum(c3), sum(c4) from c1 +sql select first(ts), first(c1), first(c2), first(c3) from c1 +sql select min(c2), min(c3), min(c4) from c1 +sql select max(c2), max(c3), max(c4) from c1 +sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 784b83d96b..86926509bb 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,26 +62,26 @@ system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data sql select * from c1 -#sql select * from stb -#sql select * from stb_1 -#sql select ts, c1, c2, c3 from c1 -#sql select ts, c1, c2, c3 from stb -#sql select ts, c1 from stb_2 -#sql select ts, c1, t1 from c1 -#sql select ts, c1, t1 from stb -#sql select ts, c1, t1 from stb_2 +sql select * from stb +sql select * from stb_1 +sql select ts, c1, c2, c3 from c1 +sql select ts, c1, c2, c3 from stb +sql select ts, c1 from stb_2 +sql select ts, c1, t1 from c1 +sql select ts, c1, t1 from stb +sql select ts, c1, t1 from stb_2 print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from c1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from c1 +sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c2), min(c3), min(c4) from c1 -#sql select max(c2), max(c3), max(c4) from c1 -#sql select sum(c2), sum(c3), sum(c4) from c1 +sql select first(ts), first(c1), first(c2), first(c3) from c1 +sql select min(c2), min(c3), min(c4) from c1 +sql select max(c2), max(c3), max(c4) from c1 +sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 30e09c4b19dfa2941b3b7db93e21c96627b60872 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:41:19 +0800 Subject: [PATCH 55/94] fix(query): fix when select percentile() has no from table clause report error issue --- source/libs/parser/src/parTranslater.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c88e38ddff..8cb0e3b32d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1165,7 +1165,11 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (isSelectStmt(pCxt->pCurrStmt)) { + //select percentile() is also valid + if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + return TSDB_CODE_SUCCESS; + } SNode* pTable = ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; if (QUERY_NODE_REAL_TABLE == nodeType(pTable) && (TSDB_CHILD_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType || From 0377f865db2fd4a58280462091449c897bdb04af Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:45:30 +0800 Subject: [PATCH 56/94] fix conflict --- source/libs/scalar/src/scalar.c | 6 +++--- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index bbb7e07bad..b3bd989eb4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,9 +700,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - if (!fmIsScalarFunc(node->funcId)) { - return DEAL_RES_CONTINUE; - } + //if (!fmIsScalarFunc(node->funcId)) { + // return DEAL_RES_CONTINUE; + //} FOREACH(tnode, node->pParameterList) { if (!SCL_IS_CONST_NODE(tnode)) { diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..1163c0f60a 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit 1163c0f60aa65d6cc58283247c8bf8c56ba43b92 diff --git a/tools/taosadapter b/tools/taosadapter index df8678f070..c3815951fc 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit df8678f070e3f707faf59baebec90065f6e1268b +Subproject commit c3815951fc80617ecd171f3743b8b4a4d0bc712e From 7d5c496e103d611a45cb5b464b5b7d0a66255065 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:46:13 +0800 Subject: [PATCH 57/94] fix comment --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8cb0e3b32d..b0a012e20b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1166,7 +1166,7 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } if (isSelectStmt(pCxt->pCurrStmt)) { - //select percentile() is also valid + //select percentile() without from clause is also valid if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return TSDB_CODE_SUCCESS; } From 14ae310fea9f3d57c9d740aa55a14d7d2340d9c5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:47:52 +0800 Subject: [PATCH 58/94] fix issue --- source/libs/scalar/src/scalar.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3bd989eb4..bbb7e07bad 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,9 +700,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - //if (!fmIsScalarFunc(node->funcId)) { - // return DEAL_RES_CONTINUE; - //} + if (!fmIsScalarFunc(node->funcId)) { + return DEAL_RES_CONTINUE; + } FOREACH(tnode, node->pParameterList) { if (!SCL_IS_CONST_NODE(tnode)) { From a5a3b96e5860d7f379f7293269e92cfd44e514b2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:50:51 +0800 Subject: [PATCH 59/94] enable select agg function without from clause --- source/libs/scalar/src/scalar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index bbb7e07bad..484d95cb5a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,7 +700,7 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - if (!fmIsScalarFunc(node->funcId)) { + if (!fmIsScalarFunc(node->funcId) && (!ctx->dual)) { return DEAL_RES_CONTINUE; } From 68ecae1df5ae0462bdd5e458004f6a64e6f53651 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 16:57:45 +0800 Subject: [PATCH 60/94] refactor(sync): add case --- .../script/tsim/sync/mnodeLeaderTransfer.sim | 2 +- tests/script/tsim/sync/vnode-insert.sim | 181 ++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/script/tsim/sync/vnode-insert.sim diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim index 6c67f639f6..00329ced76 100644 --- a/tests/script/tsim/sync/mnodeLeaderTransfer.sim +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -27,7 +27,7 @@ sql create dnode $hostname port 7300 sleep 3000 -print =============== create mnode 2 +print =============== create mnode 2, 3 sql create mnode on dnode 2 sql create mnode on dnode 3 sleep 3000 diff --git a/tests/script/tsim/sync/vnode-insert.sim b/tests/script/tsim/sync/vnode-insert.sim new file mode 100644 index 0000000000..221e3dd2b2 --- /dev/null +++ b/tests/script/tsim/sync/vnode-insert.sim @@ -0,0 +1,181 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 1 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + +print ====> step4 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591201000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step5 sleep 20s, checking data +sleep 20000 + +print ====> step6 stop all +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ====> step7 start all +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print ====> step8 sleep 20s, checking data +sleep 20000 + +print ====> step9 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591202000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step10 sleep 20s, checking data +sleep 20000 + + From 33e4b353800647ec640a0e0f8d8b7de556d24f3b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 17:15:04 +0800 Subject: [PATCH 61/94] refactor(sync): add case --- .../script/tsim/sync/vnodeLeaderTransfer.sim | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/script/tsim/sync/vnodeLeaderTransfer.sim diff --git a/tests/script/tsim/sync/vnodeLeaderTransfer.sim b/tests/script/tsim/sync/vnodeLeaderTransfer.sim new file mode 100644 index 0000000000..cbeb7063ff --- /dev/null +++ b/tests/script/tsim/sync/vnodeLeaderTransfer.sim @@ -0,0 +1,141 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 6 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + From 3f11c849cdf0fbf1c24a64ad7d97995951d8303a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 17:34:54 +0800 Subject: [PATCH 62/94] refactor: do some internal refactor. --- source/libs/executor/src/executil.c | 2 +- source/libs/function/inc/builtinsimpl.h | 2 +- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 358 ++++++++---------------- 4 files changed, 127 insertions(+), 237 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 76d853ef3e..a8ff3188c8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -65,7 +65,7 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } rowSize += - (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData) + (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(doSaveTupleData) return rowSize; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 30fdbb245d..35669b3e42 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -106,7 +106,7 @@ bool irateFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFunction(SqlFunctionCtx *pCtx); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx); +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6795e4418c..ddff94e9e1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2232,7 +2232,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = cacheLastRowFunction, + .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e2288d9f70..a921e4a70b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -80,13 +80,14 @@ typedef struct STopBotRes { } STopBotRes; typedef struct SFirstLastRes { - bool hasResult; + bool hasResult; // used for last_row function only, isNullRes in SResultRowEntry can not be passed to downstream.So, // this attribute is required - bool isNull; - int32_t bytes; - int64_t ts; - char buf[]; + bool isNull; + int32_t bytes; + int64_t ts; + STuplePos pos; + char buf[]; } SFirstLastRes; typedef struct SStddevRes { @@ -1141,8 +1142,8 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); -static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { // the data is loaded, not only the block SMA value @@ -1195,7 +1196,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -1207,7 +1208,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1220,7 +1221,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -1232,7 +1233,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_FLOAT) { @@ -1246,7 +1247,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1271,7 +1272,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1283,7 +1284,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1302,7 +1303,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1314,7 +1315,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1333,7 +1334,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1345,7 +1346,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1364,7 +1365,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1376,7 +1377,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1397,7 +1398,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1409,7 +1410,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1428,7 +1429,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1440,7 +1441,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1459,7 +1460,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1471,7 +1472,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1490,7 +1491,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1502,7 +1503,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1522,7 +1523,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1534,7 +1535,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1553,7 +1554,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1565,7 +1566,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -2640,7 +2641,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); + return sizeof(SFirstLastRes) + resBytes; } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -2669,6 +2670,33 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } +static void saveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { + if (pCtx->subsidiaries.num <= 0) { + return; + } + + if (!pInfo->hasResult) { + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } else { + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } +} + +static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t currentTs, int32_t type, char* pData) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + if (IS_VAR_DATA_TYPE(type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + pInfo->ts = currentTs; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + // This ordinary first function does not care if current scan is ascending order or descending order scan // the OPTIMIZED version of first function will only handle the ascending order scan int32_t firstFunction(SqlFunctionCtx* pCtx) { @@ -2680,9 +2708,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; - int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; + pInfo->bytes = pInputCol->info.bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2700,8 +2726,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < startKey) { + if (pInfo->ts < startKey) { return TSDB_CODE_SUCCESS; } } @@ -2715,26 +2740,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2742,8 +2749,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < endKey) { + if (pInfo->ts < endKey) { return TSDB_CODE_SUCCESS; } } @@ -2758,24 +2764,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2821,26 +2811,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } + break; } } else { // descending order @@ -2853,24 +2827,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } break; } @@ -2885,8 +2843,9 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S int32_t start = pColInfo->startRowIndex; pOutput->bytes = pInput->bytes; - TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes); - TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + TSKEY* tsIn = &pInput->ts; + TSKEY* tsOut = &pOutput->ts; + if (pOutput->hasResult) { if (isFirst) { if (*tsIn > *tsOut) { @@ -2898,20 +2857,12 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S } } } + *tsOut = *tsIn; memcpy(pOutput->buf, pInput->buf, pOutput->bytes); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY)); - if (pCtx->subsidiaries.num > 0) { - if (!pOutput->hasResult) { - saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } - } - pOutput->hasResult = true; + saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); - return; + pOutput->hasResult = true; } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { @@ -2953,34 +2904,34 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); return pResInfo->numOfRes; } int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); - char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + // todo check for failure + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); colDataAppend(pCol, pBlock->info.rows, res, false); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); return 1; } +//todo rewrite: int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -2998,6 +2949,28 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } +static void doSaveLastrow(SqlFunctionCtx *pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + if (colDataIsNull_s(pInputCol, rowIndex)) { + pInfo->isNull = true; + } else { + pInfo->isNull = false; + + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + } + + pInfo->ts = cts; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + int32_t lastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; @@ -3007,12 +2980,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; - SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; - TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); @@ -3022,31 +2992,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3054,31 +3003,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3088,21 +3016,6 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); - - return pResInfo->numOfRes; -} - bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; @@ -3425,7 +3338,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -3449,7 +3362,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -3466,7 +3379,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; // todo refactor: move away @@ -3527,7 +3440,7 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS #endif } -void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); int32_t numOfCols = pCtx->subsidiaries.num; @@ -4843,7 +4756,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); } pInfo->numSampled++; } else { @@ -4851,7 +4764,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (j < pInfo->samples) { sampleAssignResult(pInfo, data, j); if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + doCopyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); } } } @@ -5993,7 +5906,7 @@ int32_t interpFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -6002,9 +5915,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; // last_row function does not ignore the null value @@ -6014,28 +5925,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - if (colDataIsNull_s(pInputCol, i)) { - pInfo->isNull = true; - } else { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - - memcpy(pInfo->buf, data, bytes); - } - - pInfo->ts = cts; - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - - pInfo->hasResult = true; + doSaveLastrow(pCtx, data, i, cts, pInfo); } } From 339a0086c052a3881e9b284f834e440974ff7cd0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:42:57 +0800 Subject: [PATCH 63/94] docs: refactor cluster chapter --- docs/zh/10-cluster/01-deploy.md | 26 +--- docs/zh/10-cluster/02-cluster-mgmt.md | 145 +++------------------ docs/zh/10-cluster/03-ha-and-lb.md | 87 ------------- docs/zh/10-cluster/03-high-availability.md | 33 +++++ 4 files changed, 56 insertions(+), 235 deletions(-) delete mode 100644 docs/zh/10-cluster/03-ha-and-lb.md create mode 100644 docs/zh/10-cluster/03-high-availability.md diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-cluster/01-deploy.md index b44d2942f2..9dbfb50f18 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-cluster/01-deploy.md @@ -10,7 +10,7 @@ title: 集群部署 ### 第一步 -如果搭建集群的物理节点中，存有之前的测试数据、装过 1.X 的版本，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 +如果搭建集群的物理节点中，存有之前的测试数据，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 :::note 因为 FQDN 的信息会写进文件，如果之前没有配置或者更改 FQDN，且启动了 TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/\*）； @@ -54,30 +54,16 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是 6030 serverPort 6030 -// 副本数为偶数的时候，需要配置，请参考《Arbitrator 的使用》的部分 -arbitrator ha.taosdata.com:6042 -``` - 一定要修改的参数是 firstEp 和 fqdn。在每个数据节点，firstEp 需全部配置成一样，但 fqdn 一定要配置成其所在数据节点的值。其他参数可不做任何修改，除非你很清楚为什么要修改。 -加入到集群中的数据节点 dnode，涉及集群相关的下表 9 项参数必须完全相同，否则不能成功加入到集群中。 +加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。 | **#** | **配置参数名称** | **含义** | | ----- | ------------------ | ------------------------------------------- | -| 1 | numOfMnodes | 系统中管理节点个数 | -| 2 | mnodeEqualVnodeNum | 一个 mnode 等同于 vnode 消耗的个数 | -| 3 | offlineThreshold | dnode 离线阈值，超过该时间将导致 Dnode 离线 | -| 4 | statusInterval | dnode 向 mnode 报告状态时长 | -| 5 | arbitrator | 系统中裁决器的 End Point | -| 6 | timezone | 时区 | -| 7 | balance | 是否启动负载均衡 | -| 8 | maxTablesPerVnode | 每个 vnode 中能够创建的最大表个数 | -| 9 | maxVgroupsPerDb | 每个 DB 中能够使用的最大 vgroup 个数 | - -:::note -在 2.0.19.0 及更早的版本中，除以上 9 项参数外，dnode 加入集群时，还会要求 locale 和 charset 参数的取值也一致。 - -::: +| 1 | statusInterval | dnode 向 mnode 报告状态时长 | +| 2 | timezone | 时区 | +| 3 | locale | 系统区位信息及编码格式 | +| 4 | charset | 字符集编码 | ## 启动集群 diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md index 6ab8ec091b..f02a105826 100644 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ b/docs/zh/10-cluster/02-cluster-mgmt.md @@ -24,15 +24,15 @@ SHOW DNODES; ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | +Query OK, 1 rows affected (0.006684s) ``` ## 查看虚拟节点组 -为充分利用多核技术，并提供 scalability，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 +为充分利用多核技术，并提供横向扩展能力，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 启动 CLI 程序 taos，然后执行： @@ -44,26 +44,15 @@ SHOW VGROUPS; 输出如下（具体内容仅供参考，取决于实际的集群配置） ``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) - taos> use db; Database changed. taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 1 | master | 0 | - 15 | 38000 | ready | 1 | 1 | master | 0 | - 16 | 38000 | ready | 1 | 1 | master | 0 | - 17 | 38000 | ready | 1 | 1 | master | 0 | - 18 | 37001 | ready | 1 | 1 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | status | nfiles | file_size | tsma | +================================================================================================================================================================================================ + 2 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 3 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 4 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | Query OK, 8 row(s) in set (0.001154s) ``` @@ -77,35 +66,21 @@ CREATE DNODE "fqdn:port"; 将新数据节点的 End Point 添加进集群的 EP 列表。“fqdn:port“需要用双引号引起来，否则出错。一个数据节点对外服务的 fqdn 和 port 可以通过配置文件 taos.cfg 进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置 FQDN，可能导致生成的数据节点的 End Point 不是所期望的】 -示例如下： -``` -taos> create dnode "localhost:7030"; -Query OK, 0 of 0 row(s) in database (0.008203s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) -``` - -在上面的示例中可以看到新创建的 dnode 的状态为 offline，待该 dnode 被启动并连接上配置文件中指定的 firstEp后再次查看，得到如下结果（示例） +然后启动新加入的数据节点的 taosd 进程，再通过 taos 查看数据节点状态： ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 3 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 6 | 8 | ready | any | 2022-04-19 08:14:59.165 | | -Query OK, 2 row(s) in set (0.001316s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | + 2 | trd04:6030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | +Query OK, 2 rows affected (0.007031s) ``` 从中可以看到两个 dnode 状态都为 ready - ## 删除数据节点 -启动 CLI 程序 taos，然后执行： +先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： ```sql DROP DNODE "fqdn:port"; @@ -117,26 +92,6 @@ DROP DNODE dnodeId; 通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 -示例如下： -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) - -taos> drop dnode 2; -Query OK, 0 of 0 row(s) in database (0.000518s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.001137s) -``` - -上面的示例中，初次执行 `show dnodes` 列出了两个 dnode, 执行 `drop dnode 2` 删除其中 ID 为 2 的 dnode 之后再次执行 `show dnodes`，可以看到只剩下 ID 为 1 的 dnode 。 :::warning @@ -147,70 +102,4 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 ::: -## 手动迁移数据节点 - -手动将某个 vnode 迁移到指定的 dnode。 - -启动 CLI 程序 taos，然后执行： - -```sql -ALTER DNODE BALANCE "VNODE:-DNODE:"; -``` - -其中：source-dnodeId 是源 dnodeId，也就是待迁移的 vnode 所在的 dnodeID；vgId 可以通过 SHOW VGROUPS 获得，列表的第一列；dest-dnodeId 是目标 dnodeId。 - -首先执行 `show vgroups` 查看 vgroup 的分布情况 -``` -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 3 | master | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | - 18 | 37001 | ready | 1 | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | -Query OK, 8 row(s) in set (0.001314s) -``` - -从中可以看到在 dnode 3 中有5个 vgroup，而 dnode 1 有 3 个 vgroup，假定我们想将其中 vgId 为18 的 vgroup 从 dnode 3 迁移到 dnode 1 - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; - -DB error: Balance already enabled (0.00755 -``` - -上面的结果表明目前所在数据库已经启动了 balance 选项，所以无法进行手动迁移。 - -停止整个集群，将两个 dnode 的配置文件中的 balance 都设置为 0 （默认为1）之后，重新启动集群，再次执行 ` alter dnode` 和 `show vgroups` 命令如下 -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; -Query OK, 0 row(s) in set (0.000575s) - -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | -================================================================================================================= - 14 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | slave | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | -Query OK, 8 row(s) in set (0.001242s) -``` - -从上面的输出可以看到 vgId 为 18 的 vnode 被从 dnode 3 迁移到了 dnode 1。 - -:::warning - -只有在集群的自动负载均衡选项关闭时（balance 设置为 0），才允许手动迁移。 -只有处于正常工作状态的 vnode 才能被迁移：master/slave；当处于 offline/unsynced/syncing 状态时，是不能迁移的。 -迁移前，务必核实目标 dnode 的资源足够：CPU、内存、硬盘。 - -::: diff --git a/docs/zh/10-cluster/03-ha-and-lb.md b/docs/zh/10-cluster/03-ha-and-lb.md deleted file mode 100644 index 3d15feb11c..0000000000 --- a/docs/zh/10-cluster/03-ha-and-lb.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: 高可用与负载均衡 ---- - -## Vnode 的高可用性 - -TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 - -vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： - -```sql -CREATE DATABASE demo replica 3; -``` - -一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 - -一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 - -因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 - -## Mnode 的高可用性 - -TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，副本数由系统配置参数 numOfMnodes 决定，有效范围为 1-3。为保证元数据的强一致性，mnode 副本之间是通过同步的方式进行数据复制的。 - -一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。多个 dnode 情况下，哪个 dnode 可以作为 mnode 呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： - -```sql -SHOW MNODES; -``` - -来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（master，slave，unsynced 或 offline）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。如果 numOfMnodes 配置为 2，启动第二个 dnode 时，该 dnode 也将运行一个 mnode 实例。 - -为保证 mnode 服务的高可用性，numOfMnodes 必须设置为 2 或更大。因为 mnode 保存的元数据必须是强一致的，如果 numOfMnodes 大于 2，复制参数 quorum 自动设为 2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 - -:::note -一个 TDengine 高可用系统，无论是 vnode 还是 mnode，都必须配置多个副本。 - -::: - -## 负载均衡 - -有三种情况，将触发负载均衡，而且都无需人工干预。 - -当一个新数据节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新数据节点上，无需任何人工干预。 -当一个数据节点从集群中移除时，系统将自动把该数据节点上的数据转移到其他数据节点，无需任何人工干预。 -如果一个数据节点过热（数据量过大），系统将自动进行负载均衡，将该数据节点的一些 vnode 自动挪到其他节点。 -当上述三种情况发生时，系统将启动各个数据节点的负载计算，从而决定如何挪动。 - -:::tip -负载均衡由参数 balance 控制，它决定是否启动自动负载均衡，0 表示禁用，1 表示启用自动负载均衡。 - -::: - -## 数据节点离线处理 - -如果一个数据节点离线，TDengine 集群将自动检测到。有如下两种情况： - -该数据节点离线超过一定时间（taos.cfg 里配置参数 offlineThreshold 控制时长），系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重新上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 - -离线后，在 offlineThreshold 的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 - -:::note -如果一个虚拟节点组（包括 mnode 组）里所归属的每个数据节点都处于离线或 unsynced 状态，必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后，才能选出 Master，该虚拟节点组才能对外提供服务。比如整个集群有 3 个数据节点，副本数为 3，如果 3 个数据节点都宕机，然后 2 个数据节点重启，是无法工作的，只有等 3 个数据节点都重启成功，才能对外服务。 - -::: - -## Arbitrator 的使用 - -如果副本数为偶数，当一个 vnode group 里一半或超过一半的 vnode 不工作时，是无法从中选出 master 的。同理，一半或超过一半的 mnode 不工作时，是无法选出 mnode 的 master 的，因为存在“split brain”问题。 - -为解决这个问题，TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含 Arbitrator 在内，超过半数的 vnode 或 mnode 工作，那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形，如果一个节点 A 离线，但另外一个节点 B 正常，而且能连接到 Arbitrator，那么节点 B 就能正常工作。 - -总之，在目前版本下，TDengine 建议在双副本环境要配置 Arbitrator，以提升系统的可用性。 - -Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求，只需要保证有网络连接，找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤： - -请点击 安装包下载，在 TDengine Arbitrator Linux 一节中，选择合适的版本下载并安装。 -该应用的命令行参数 -p 可以指定其对外服务的端口号，缺省是 6042。 - -修改每个 taosd 实例的配置文件，在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。（如果该参数配置了，当副本数为偶数时，系统将自动连接配置的 Arbitrator。如果副本数为奇数，即使配置了 Arbitrator，系统也不会去建立连接。） - -在配置文件中配置了的 Arbitrator，会出现在 SHOW DNODES 指令的返回结果中，对应的 role 列的值会是“arb”。 -查看集群 Arbitrator 的状态【2.0.14.0 以后支持】 - -```sql -SHOW DNODES; -``` diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md new file mode 100644 index 0000000000..1ec413437f --- /dev/null +++ b/docs/zh/10-cluster/03-high-availability.md @@ -0,0 +1,33 @@ +--- +title: 高可用与负载均衡 +--- + +## Vnode 的高可用性 + +TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 + +vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： + +```sql +CREATE DATABASE demo replica 3; +``` + +一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 + +一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 + +因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 + +## Mnode 的高可用性 + +TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode`来增加新的mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 + +一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： + +```sql +SHOW MNODES; +``` + +来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 + +在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 \ No newline at end of file From 232fb94fe0f7cbd546f26eaa79fbf373dc32a784 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 Jul 2022 17:51:32 +0800 Subject: [PATCH 64/94] chore: add libtaos ws submodule for3.0 (#14954) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 --- .gitmodules | 2 +- tools/taosws-rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9d649bcdd3..31c211bf89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,4 +21,4 @@ url = https://github.com/taosdata/taosadapter.git [submodule "tools/taosws-rs"] path = tools/taosws-rs - url = https://github.com/taosdata/taosws-rs.git + url = https://github.com/taosdata/taosws-rs diff --git a/tools/taosws-rs b/tools/taosws-rs index 7a94ffab45..c5fded266d 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 +Subproject commit c5fded266d3b10508e38bf3285bb7ecf798bc343 From 7696cdeb72612fe7ccf5e6ea33c87e1b4e4eaf17 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:52:39 +0800 Subject: [PATCH 65/94] doc: add a section for load balance --- docs/zh/10-cluster/04-load-balance.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/zh/10-cluster/04-load-balance.md diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md new file mode 100644 index 0000000000..5e79d6b90e --- /dev/null +++ b/docs/zh/10-cluster/04-load-balance.md @@ -0,0 +1,15 @@ +--- +title: 负载均衡 +--- + +TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroups 中，每张表或子表只能由一个 vgroups 处理，一个 vgroups 可能负责处理多个表或子表。 + +创建数据库时可以指定其中的 vgroups 的数量： + +```sql +create database db0 vgroups 100; +``` + +如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 + +通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file From cbd2d8f8a2876f7b7c503fe27b1bf9f471012c6d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 17:54:45 +0800 Subject: [PATCH 66/94] fix(query): add spread function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 42 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index df3e27225f..e0beaff262 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -106,6 +106,7 @@ int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 74c0e54640..208022ad1a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2115,6 +2115,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSpreadFuncEnv, .initFunc = spreadFunctionSetup, .processFunc = spreadFunction, + .sprocessFunc = spreadScalarFunction, .finalizeFunc = spreadFinalize, .invertFunc = NULL, .combineFunc = spreadCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 8808d17454..9a8f570a52 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2338,3 +2338,45 @@ int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return percentileScalarFunction(pInput, inputNum, pOutput); } + +int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + double min, max; + SET_DOUBLE_VAL(&min, DBL_MAX); + SET_DOUBLE_VAL(&max, -DBL_MAX); + + bool hasNull = false; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + + char *in = pInputData->pData; + + double val = 0; + GET_TYPED_DATA(val, double, type, in); + + if (val < GET_DOUBLE_VAL(&min)) { + SET_DOUBLE_VAL(&min, val); + } + + if (val > GET_DOUBLE_VAL(&max)) { + SET_DOUBLE_VAL(&max, val); + } + } + + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + double result = max - min; + colDataAppend(pOutputData, 0, (char *)&result, false); + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} From ce6b52cd72a1064988254e129ab80a6dd202f70a Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:57:38 +0800 Subject: [PATCH 67/94] doc: refine load balance --- docs/zh/10-cluster/04-load-balance.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index 5e79d6b90e..9b582f417d 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -12,4 +12,6 @@ create database db0 vgroups 100; 如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 +此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 + 通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file From d99b682f765a1f8407801e757e1e42fc5c767f34 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 17:58:37 +0800 Subject: [PATCH 68/94] fix: multi-rows function validity check --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/src/parTranslater.c | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d58d5d4b8b..f8c7024591 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -258,6 +258,7 @@ typedef struct SSelectStmt { bool hasAggFuncs; bool hasRepeatScanFuncs; bool hasIndefiniteRowsFunc; + bool hasMultiRowsFunc; bool hasSelectFunc; bool hasSelectValFunc; bool hasOtherVectorFunc; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d2959f21d..33ab1166b0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1058,7 +1058,8 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc) { + if (!isSelectStmt(pCxt->pCurrStmt) || ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -1093,7 +1094,8 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return TSDB_CODE_SUCCESS; } if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || - ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow || NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { @@ -1111,7 +1113,8 @@ static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pF return TSDB_CODE_SUCCESS; } if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || - ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (hasInvalidFuncNesting(pFunc->pParameterList)) { @@ -1204,6 +1207,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId); if (fmIsSelectFunc(pFunc->funcId)) { pSelect->hasSelectFunc = true; ++(pSelect->selectFuncNum); From 9394e3380f6f4d8da734bc0789a2ab0a10dd5648 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 15 Jul 2022 17:48:48 +0800 Subject: [PATCH 69/94] feat(stream): recover from failure --- include/common/tcommon.h | 10 +- include/common/tmsg.h | 29 ++++- include/common/tmsgdef.h | 1 + include/libs/executor/executor.h | 2 + include/libs/stream/tstream.h | 49 +++++++- source/common/src/tmsg.c | 39 +++++- source/dnode/mnode/impl/inc/mndDef.h | 18 +++ source/dnode/mnode/impl/inc/mndTrans.h | 1 + source/dnode/mnode/impl/src/mndDef.c | 2 + source/dnode/mnode/impl/src/mndScheduler.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 66 ++++++++++- source/dnode/mnode/sdb/inc/sdb.h | 25 ++-- source/libs/executor/inc/executorimpl.h | 10 ++ source/libs/executor/src/executorMain.c | 9 ++ source/libs/executor/src/scanoperator.c | 33 ++++-- source/libs/stream/src/stream.c | 9 +- source/libs/stream/src/streamData.c | 2 +- source/libs/stream/src/streamExec.c | 27 ++++- source/libs/stream/src/streamRecover.c | 131 +++++++++++++++++++++ source/libs/stream/src/streamTask.c | 2 + 20 files changed, 423 insertions(+), 43 deletions(-) create mode 100644 source/libs/stream/src/streamRecover.c diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5a7b32d20b..444a17c7f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -57,7 +57,7 @@ enum { // STREAM_INPUT__TABLE_SCAN, STREAM_INPUT__TQ_SCAN, STREAM_INPUT__DATA_RETRIEVE, - STREAM_INPUT__TRIGGER, + STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, STREAM_INPUT__DROP, }; @@ -155,10 +155,10 @@ typedef struct SQueryTableDataCond { int32_t numOfCols; SColumnInfo* colList; int32_t type; // data block load type: -// int32_t numOfTWindows; - STimeWindow twindows; - int64_t startVersion; - int64_t endVersion; + // int32_t numOfTWindows; + STimeWindow twindows; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c8e13fce3d..49feba4d3c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1968,7 +1968,7 @@ typedef struct SVCreateTbReq { int8_t type; union { struct { - char* name; // super table name + char* name; // super table name tb_uid_t suid; SArray* tagName; uint8_t* pTag; @@ -2437,9 +2437,6 @@ typedef struct { int8_t igNotExists; } SMDropStreamReq; -int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); -int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); - typedef struct { int8_t reserved; } SMDropStreamRsp; @@ -2454,6 +2451,27 @@ typedef struct { int8_t reserved; } SVDropStreamTaskRsp; +int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); +int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); + +typedef struct { + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; +} SMRecoverStreamReq; + +typedef struct { + int8_t reserved; +} SMRecoverStreamRsp; + +typedef struct { + int64_t recoverObjUid; + int32_t taskId; + int32_t hasCheckPoint; +} SMVStreamGatherInfoReq; + +int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); + typedef struct { int64_t leftForVer; int32_t vgId; @@ -2876,7 +2894,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp } static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI16(buf, &pRsp->resMsgType); buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen); buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 488bc6346e..a24eda33bb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -131,6 +131,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "alter-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "drop-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RECOVER_STREAM, "recover-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index dd64c5bf71..8b0a836ad2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,6 +192,8 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem); +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 071c539ff3..d03fab0ebc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -34,6 +34,10 @@ typedef struct SStreamTask SStreamTask; enum { TASK_STATUS__NORMAL = 0, TASK_STATUS__DROPPING, + TASK_STATUS__FAIL, + TASK_STATUS__STOP, + TASK_STATUS__PREPARE_RECOVER, + TASK_STATUS__RECOVERING, }; enum { @@ -72,6 +76,7 @@ typedef struct { int8_t type; int32_t srcVgId; + int32_t childId; int64_t sourceVer; SArray* blocks; // SArray @@ -222,6 +227,8 @@ typedef struct { int32_t nodeId; int32_t childId; int32_t taskId; + int64_t checkpointVer; + int64_t processedVer; SEpSet epSet; } SStreamChildEpInfo; @@ -232,6 +239,7 @@ typedef struct SStreamTask { int8_t execType; int8_t sinkType; int8_t dispatchType; + int8_t isStreamDistributed; int16_t dispatchMsgType; int8_t taskStatus; @@ -242,6 +250,13 @@ typedef struct SStreamTask { int32_t nodeId; SEpSet epSet; + // used for semi or single task, + // while final task should have processedVer for each child + int64_t recoverSnapVer; + int64_t startVer; + int64_t checkpointVer; + int64_t processedVer; + // children info SArray* childEpInfo; // SArray @@ -316,12 +331,12 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); - } else if (pItem->type == STREAM_INPUT__TRIGGER) { + } else if (pItem->type == STREAM_INPUT__GET_RES) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); } - if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + if (pItem->type != STREAM_INPUT__GET_RES && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } @@ -420,6 +435,36 @@ typedef struct { int8_t inputStatus; } SStreamTaskRecoverRsp; +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq); +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq); + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp); +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverReq; + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverRsp; + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq); +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq); + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; +} SPStreamTaskRecoverReq; + +typedef struct { + int8_t reserved; +} SPStreamTaskRecoverRsp; + int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9ebfa78b80..788f2cdfba 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4823,6 +4823,35 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * return 0; } +int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); @@ -4945,8 +4974,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; int32_t len = taosArrayGetSize(pReq->ctb.tagName); if (tEncodeI32(pCoder, len) < 0) return -1; - for (int32_t i = 0; i < len; i++){ - char* name = taosArrayGet(pReq->ctb.tagName, i); + for (int32_t i = 0; i < len; i++) { + char *name = taosArrayGet(pReq->ctb.tagName, i); if (tEncodeCStr(pCoder, name) < 0) return -1; } } else if (pReq->type == TSDB_NORMAL_TABLE) { @@ -4982,9 +5011,9 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { int32_t len = 0; if (tDecodeI32(pCoder, &len) < 0) return -1; pReq->ctb.tagName = taosArrayInit(len, TSDB_COL_NAME_LEN); - if(pReq->ctb.tagName == NULL) return -1; - for (int32_t i = 0; i < len; i++){ - char name[TSDB_COL_NAME_LEN] = {0}; + if (pReq->ctb.tagName == NULL) return -1; + for (int32_t i = 0; i < len; i++) { + char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; if (tDecodeCStr(pCoder, &tmp) < 0) return -1; strcpy(name, tmp); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7ac991451e..5cf6e26ee8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -559,6 +559,7 @@ typedef struct { // info int64_t uid; int8_t status; + int8_t isDistributed; // config int8_t igExpired; int8_t trigger; @@ -586,6 +587,23 @@ typedef struct { int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); +typedef struct { + char streamName[TSDB_STREAM_FNAME_LEN]; + int64_t uid; + int64_t streamUid; + SArray* childInfo; // SArray +} SStreamCheckpointObj; + +#if 0 +typedef struct { + int64_t uid; + int64_t streamId; + int8_t isDistributed; + int8_t status; + int8_t stage; +} SStreamRecoverObj; +#endif + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 1497bba11c..6e367fbe24 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -27,6 +27,7 @@ typedef enum { TRANS_STOP_FUNC_TEST = 2, TRANS_START_FUNC_MQ_REB = 3, TRANS_STOP_FUNC_MQ_REB = 4, + TRANS_FUNC_RECOVER_STREAM_STEP_NEXT = 5, } ETrnFunc; typedef enum { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 727b20ef8a..c26424e049 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -27,6 +27,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->isDistributed) < 0) return -1; if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; @@ -72,6 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->isDistributed) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index aa6e7192fc..ec0ea90f46 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -319,6 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); ASSERT(totLevel <= 2); pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); + pStream->isDistributed = totLevel == 2; bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5777df4fa6..8248a26c28 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,7 +36,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); -/*static int32_t mndProcessDropStreamInRsp(SRpcMsg *pRsp);*/ +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -55,6 +55,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); @@ -672,6 +673,69 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + /*SDbObj *pDb = NULL;*/ + /*SUserObj *pUser = NULL;*/ + + SMRecoverStreamReq recoverReq = {0}; + if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) { + ASSERT(0); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + pStream = mndAcquireStream(pMnode, recoverReq.name); + + if (pStream == NULL) { + if (recoverReq.igNotExists) { + mDebug("stream:%s, not exist, ignore not exist is set", recoverReq.name); + sdbRelease(pMnode->pSdb, pStream); + return 0; + } else { + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + return -1; + } + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + if (pTrans == NULL) { + mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); + + // broadcast to recover all tasks + if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + // update stream status + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + sdbRelease(pMnode->pSdb, pStream); + + return TSDB_CODE_ACTION_IN_PROGRESS; +} + int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index be56d901de..4f2486d551 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -137,17 +137,18 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_STREAM = 10, - SDB_OFFSET = 11, - SDB_SUBSCRIBE = 12, - SDB_CONSUMER = 13, - SDB_TOPIC = 14, - SDB_VGROUP = 15, - SDB_SMA = 16, - SDB_STB = 17, - SDB_DB = 18, - SDB_FUNC = 19, - SDB_MAX = 20 + SDB_STREAM_CK = 10, + SDB_STREAM = 11, + SDB_OFFSET = 12, + SDB_SUBSCRIBE = 13, + SDB_CONSUMER = 14, + SDB_TOPIC = 15, + SDB_VGROUP = 16, + SDB_SMA = 17, + SDB_STB = 18, + SDB_DB = 19, + SDB_FUNC = 20, + SDB_MAX = 21 } ESdbType; typedef struct SSdbRaw { @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) ; +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); /** * @brief Cancel a traversal diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1b11941849..99845713a9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -139,6 +139,12 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; +enum { + STREAM_RECOVER_STEP__NONE = 0, + STREAM_RECOVER_STEP__PREPARE, + STREAM_RECOVER_STEP__SCAN, +}; + typedef struct { //TODO remove prepareStatus STqOffsetVal prepareStatus; // for tmq @@ -147,6 +153,10 @@ typedef struct { SSDataBlock* pullOverBlk; // for streaming SWalFilterCond cond; int64_t lastScanUid; + int8_t recoverStep; + SQueryTableDataCond tableCond; + int64_t recoverStartVer; + int64_t recoverEndVer; } SStreamTaskInfo; typedef struct SExecTaskInfo { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index d910b8be34..37d2521e5d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -261,6 +261,15 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { } #endif +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.recoverStartVer = startVer; + pTaskInfo->streamInfo.recoverEndVer = endVer; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; + return 0; +} + void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; return (void*)pInfo->tqReader; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f5b6b0c7a6..dcb34c621d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -517,14 +517,12 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); - int32_t code; - if (pInfo->dataReader != NULL) { - code = tsdbReaderReset(pInfo->dataReader, &pInfo->cond); - ASSERT(code == 0); - } else { - code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, - GET_TASKID(pTaskInfo)); - ASSERT(code == 0); + tsdbReaderClose(pInfo->dataReader); + + int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, + GET_TASKID(pTaskInfo)); + if (code != 0) { + // TODO } } @@ -1264,6 +1262,24 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return NULL; } + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); + pTSInfo->scanTimes = 0; + pTSInfo->currentGroupId = -1; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; + } + + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN) { + SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); + if (pBlock != NULL) { + return pBlock; + } + // TODO fill in bloom filter + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + return NULL; + } + size_t total = taosArrayGetSize(pInfo->pBlockLists); // TODO: refactor if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { @@ -1556,6 +1572,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } taosArrayDestroy(tableIdList); + memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); } // create the pseduo columns info diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 566d9209a8..2a96db3bfc 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -57,7 +57,7 @@ void streamTriggerByTimer(void* param, void* tmrId) { if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); if (trigger == NULL) return; - trigger->type = STREAM_INPUT__TRIGGER; + trigger->type = STREAM_INPUT__GET_RES; trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (trigger->pBlock == NULL) { taosFreeQitem(trigger); @@ -183,8 +183,11 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S // 2.1. idle: exec // 2.2. executing: return // 2.3. closing: keep trying +#if 0 if (pTask->execType != TASK_EXEC__NONE) { - streamExec(pTask, pTask->pMsgCb); +#endif + streamExec(pTask, pTask->pMsgCb); +#if 0 } else { ASSERT(pTask->sinkType != TASK_SINK__NONE); while (1) { @@ -195,11 +198,13 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S } } } +#endif // 3. handle output // 3.1 check and set status // 3.2 dispatch / sink if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); streamDispatch(pTask, pTask->pMsgCb); } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8e16b23e56..dbf6350c93 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -112,7 +112,7 @@ int32_t streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { void streamFreeQitem(SStreamQueueItem* data) { int8_t type = data->type; - if (type == STREAM_INPUT__TRIGGER) { + if (type == STREAM_INPUT__GET_RES) { blockDataDestroy(((SStreamTrigger*)data)->pBlock); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1e46a9622b..cffd519d2a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,7 +20,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; - if (pItem->type == STREAM_INPUT__TRIGGER) { + if (pItem->type == STREAM_INPUT__GET_RES) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { @@ -73,6 +73,15 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) return 0; } +static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + int32_t childId = pBlock->childId; + int64_t ver = pBlock->sourceVer; + SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); + pChildInfo->processedVer = ver; + return 0; +} + static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { int32_t cnt = 0; void* data = NULL; @@ -84,14 +93,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { } if (data == NULL) { data = qItem; + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } streamQueueProcessSuccess(pTask->inputQueue); - continue; } else { if (streamAppendQueueItem(data, qItem) < 0) { streamQueueProcessFail(pTask->inputQueue); break; } else { cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); taosFreeQitem(qItem); @@ -106,6 +118,12 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (data == NULL) return pRes; + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); @@ -125,6 +143,11 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { taosFreeQitem(qRes); return NULL; } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } /*streamQueueProcessSuccess(pTask->inputQueue);*/ pRes = taosArrayInit(0, sizeof(SSDataBlock)); } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c new file mode 100644 index 0000000000..4175f60446 --- /dev/null +++ b/source/libs/stream/src/streamRecover.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInc.h" + +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { + if (pTask->taskStatus != TASK_STATUS__FAIL) { + return 0; + } + + if (pTask->isStreamDistributed) { + if (pTask->isDataScan) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + } else if (pTask->execType != TASK_EXEC__NONE) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + bool hasCheckpoint = false; + int32_t childSz = taosArrayGetSize(pTask->childEpInfo); + for (int32_t i = 0; i < childSz; i++) { + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + if (pEpInfo->checkpointVer == -1) { + hasCheckpoint = true; + break; + } + } + if (hasCheckpoint) { + // load from checkpoint + } else { + // recover child + } + } + } else { + if (pTask->isDataScan) { + if (pTask->checkpointVer != -1) { + // load from checkpoint + } else { + // reset stream query task info + // TODO get snapshot ver + pTask->recoverSnapVer = -1; + qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); + pTask->taskStatus = TASK_STATUS__RECOVERING; + } + } + } + + if (pTask->taskStatus == TASK_STATUS__RECOVERING) { + streamProcessRunReq(pTask); + } + return 0; +} diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 7488c009bd..2d15c31bf1 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -34,6 +34,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1; return 0; } @@ -42,6 +43,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1; return 0; } From cede4b7624d786d224267973d187834f518696d1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 15 Jul 2022 18:01:04 +0800 Subject: [PATCH 70/94] fix: enable tag scan optimization when the scan cols is empty before adding ts col --- include/libs/nodes/plannodes.h | 1 + source/libs/planner/src/planLogicCreater.c | 4 ++++ source/libs/planner/src/planOptimizer.c | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 280e9edf33..3bd02b1b48 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -78,6 +78,7 @@ typedef struct SScanLogicNode { SNodeList* pGroupTags; bool groupSort; int8_t cacheLastMode; + bool hasNormalCols; // neither tag column nor primary key tag column } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4a062a86db..e0b020ce21 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -278,6 +278,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); + if (NULL != pScan->pScanCols) { + pScan->hasNormalCols = true; + } + if (TSDB_CODE_SUCCESS == code) { code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 97858f83fb..aac98e67a1 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2112,7 +2112,7 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) { return false; } SScanLogicNode* pScan = (SScanLogicNode*)pNode; - if (NULL != pScan->pScanCols) { + if (pScan->hasNormalCols) { return false; } if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) || From b86ec034dbea631c54542b6556a9a6f3903900d4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 18:05:51 +0800 Subject: [PATCH 71/94] refactor(sync): modify elect/heartbeat timer --- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 38b9df52b8..81177fd5d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -584,8 +584,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } setPingTimerMS(pVnode->sync, 5000); - setElectTimerMS(pVnode->sync, 500); - setHeartbeatTimerMS(pVnode->sync, 100); + setElectTimerMS(pVnode->sync, 1300); + setHeartbeatTimerMS(pVnode->sync, 900); return 0; } From d02a6090ef755d7322dbaba43ff8aa9b292c3d1d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 18:08:43 +0800 Subject: [PATCH 72/94] fix: the follower of mnode handles transactions incorrectly --- source/dnode/mnode/impl/src/mndSync.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c3864b620..418bb1be0a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -56,23 +56,24 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM sdbSetApplyInfo(pMnode->pSdb, cbMeta.index, cbMeta.term, cbMeta.lastConfigIndex); } - if (pMgmt->transId == transId && transId != 0) { + if (transId <= 0) { + mError("trans:%d, invalid commit msg", transId); + } else if (transId == pMgmt->transId) { if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode)); } pMgmt->transId = 0; tsem_post(&pMgmt->syncSem); } else { -#if 1 - mError("trans:%d, invalid commit msg since trandId not match with %d", transId, pMgmt->transId); -#else STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { + mDebug("trans:%d, execute in mnode which not leader", transId); mndTransExecute(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); + } else { + mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } - // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); -#endif } } From d754cd14f4af50a982a4d36c711dab4ee9d03bcb Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 18:09:01 +0800 Subject: [PATCH 73/94] doc: fix wrong title --- docs/zh/10-cluster/03-high-availability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index 1ec413437f..b8f3b25edf 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -1,5 +1,5 @@ --- -title: 高可用与负载均衡 +title: 高可用 --- ## Vnode 的高可用性 From b1826ef736a52071b07e30ca71a299b64d19d738 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 18:15:06 +0800 Subject: [PATCH 74/94] fix(query): fix a typo --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a921e4a70b..12b796c5ca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2682,7 +2682,7 @@ static void saveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFun } } -static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t currentTs, int32_t type, char* pData) { +static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); From 72e4a8f5f5a14b256324336e7f520985b7b33af0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 18:24:16 +0800 Subject: [PATCH 75/94] doc: refine according to comments --- docs/zh/10-cluster/03-high-availability.md | 4 ++-- docs/zh/10-cluster/04-load-balance.md | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index b8f3b25edf..36f9300366 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -20,7 +20,7 @@ CREATE DATABASE demo replica 3; ## Mnode 的高可用性 -TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode`来增加新的mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 +TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode` 来增加新的 mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： @@ -30,4 +30,4 @@ SHOW MNODES; 来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 -在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 \ No newline at end of file +在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index 9b582f417d..c1a72cf006 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -14,4 +14,6 @@ create database db0 vgroups 100; 此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 -通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file +通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 + +在初始的负载均衡建立起来之后，如果由于删库、删表等动作，特别是删库动作会导致属于它的 vnode 都被删除，这有可能会造成一定程度的负载失衡，在后续版本中会提供重新平衡的方法。但如果有新的数据库建立，TDengine 也能够一定程度自我再平衡而无须人工干预。 From 9273caf5b7b3fe2c7cc6cc72def24e99240df342 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 Jul 2022 18:47:16 +0800 Subject: [PATCH 76/94] feat: update taostools for3.0 (#14964) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..d807c3ffa6 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit d807c3ffa6f750f7765e102917d1328cadf21c13 From 684b550f5e23170ba07fed1981f18a5ec2254931 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 20:17:21 +0800 Subject: [PATCH 77/94] test: add test case to index --- .../2-query/json_tag_large_tables.py | 672 ++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100644 tests/system-test/2-query/json_tag_large_tables.py diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py new file mode 100644 index 0000000000..fc41858580 --- /dev/null +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -0,0 +1,672 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import imp +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import json +import os + + +class TDTestCase: + def caseDescription(self): + ''' + Json tag test case, include create table with json tag, select json tag and query with json tag in where condition, besides, include json tag in group by/order by/join/subquery. + case1: [TD-12452] fix error if json tag is NULL + case2: [TD-12389] describe child table, tag length error if the tag is json tag + ''' + return + + def init(self, conn, logSql): + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + def run(self): + # tdSql.prepare() + tdSql.execute('drop database if exists db') + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + print("============== STEP 1 ===== prepare data & validate json string") + i = 0 + tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + while i <= 100000: + f = "insert into jsons1_{} using jsons1 tags('{\"tag1\":\"fff\",\"tag2\":{}, \"tag3\":true}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')".format + sql = f(i, i) + tdSql.execute(sql) + i = i + 1 + + # test duplicate key using the first one. elimate empty key + #tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") tdSql.query("select jtag from jsons1_8") tdSql.checkRows(0); + #tdSql.query("select ts,jtag from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, '{"tag1":false,"tag2":"beijing"}') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, '{"tag1":null,"tag2":"shanghai","tag3":"hello"}') + + #tdSql.query("select ts,jtag->'tag1' from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '5.000000000') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, 'false') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, 'null') + + ## test empty json string, save as jtag is NULL + #tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") + #tdSql.execute("CREATE TABLE if not exists jsons1_10 using jsons1 tags('')") + #tdSql.execute("CREATE TABLE if not exists jsons1_11 using jsons1 tags(' ')") + #tdSql.execute("CREATE TABLE if not exists jsons1_12 using jsons1 tags('{}')") + #tdSql.execute("CREATE TABLE if not exists jsons1_13 using jsons1 tags('null')") + + ## test invalidate json + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(76)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(hell)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") + + ## test invalidate json key, key must can be printed assic char + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"。loc\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"\t\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"试试\":\"fff\"}')") + + ## test invalidate json value, value number can not be inf,nan TD-12166 + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") + + ##test length limit + #char1= ''.join(['abcd']*64) + #char3= ''.join(['abcd']*1021) + #print(len(char3)) # 4084 + #tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 + #tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 + #tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3) # len(object)=4096 + #tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 + #tdSql.execute("drop table if exists jsons1_15") + #tdSql.execute("drop table if exists jsons1_16") + + #print("============== STEP 2 ===== alter table json tag") + #tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") + #tdSql.error("ALTER STABLE jsons1 drop tag jtag") + #tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") + + #tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") + #tdSql.query("select jtag from jsons1_1") + #tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") + + #tdSql.execute("create table st(ts timestamp, i int) tags(t int)") + #tdSql.error("ALTER STABLE st add tag jtag json") + #tdSql.error("ALTER STABLE st add column jtag json") + + #print("============== STEP 3 ===== query table") + ## test error syntax + #tdSql.error("select * from jsons1 where jtag->tag1='beijing'") + #tdSql.error("select -> from jsons1") + #tdSql.error("select * from jsons1 where contains") + #tdSql.error("select * from jsons1 where jtag->") + #tdSql.error("select jtag->location from jsons1") + #tdSql.error("select jtag contains location from jsons1") + #tdSql.error("select * from jsons1 where jtag contains location") + #tdSql.query("select * from jsons1 where jtag contains''") + #tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") + + ## test function error + #tdSql.error("select avg(jtag->'tag1') from jsons1") + #tdSql.error("select avg(jtag) from jsons1") + #tdSql.error("select min(jtag->'tag1') from jsons1") + #tdSql.error("select min(jtag) from jsons1") + #tdSql.error("select ceil(jtag->'tag1') from jsons1") + #tdSql.error("select ceil(jtag) from jsons1") + + + ##test scalar operation + #tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + + ## test select normal column + #tdSql.query("select dataint from jsons1 order by dataint") + #tdSql.checkRows(9) + #tdSql.checkData(1, 0, 1) + + ## test select json tag + #tdSql.query("select * from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select jtag from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag is null") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag is not null") + #tdSql.checkRows(8) + + ## test jtag is NULL + #tdSql.query("select jtag from jsons1_9") + #tdSql.checkData(0, 0, None) + + ## test select json tag->'key', value is string + #tdSql.query("select jtag->'tag1' from jsons1_1") + #tdSql.checkData(0, 0, '"femail"') + #tdSql.query("select jtag->'tag2' from jsons1_6") + #tdSql.checkData(0, 0, '""') + ## test select json tag->'key', value is int + #tdSql.query("select jtag->'tag2' from jsons1_1") + #tdSql.checkData(0, 0, "35.000000000") + ## test select json tag->'key', value is bool + #tdSql.query("select jtag->'tag3' from jsons1_1") + #tdSql.checkData(0, 0, "true") + ## test select json tag->'key', value is null + #tdSql.query("select jtag->'tag1' from jsons1_4") + #tdSql.checkData(0, 0, None) + ## test select json tag->'key', value is double + #tdSql.query("select jtag->'tag1' from jsons1_5") + #tdSql.checkData(0, 0, "1.232000000") + ## test select json tag->'key', key is not exist + #tdSql.query("select jtag->'tag10' from jsons1_4") + #tdSql.checkData(0, 0, None) + + #tdSql.query("select jtag->'tag1' from jsons1") + #tdSql.checkRows(9) + ## test header name + #res = tdSql.getColNameList("select jtag->'tag1' from jsons1") + #cname_list = [] + #cname_list.append("jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + + ## test where with json tag + #tdSql.query("select * from jsons1_1 where jtag is not null") + #tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + #tdSql.error("select * from jsons1 where jtag->'tag1'={}") + + ## test json error + #tdSql.error("select jtag + 1 from jsons1") + #tdSql.error("select jtag > 1 from jsons1") + #tdSql.error("select jtag like \"1\" from jsons1") + #tdSql.error("select jtag in (\"1\") from jsons1") + ##tdSql.error("select jtag from jsons1 where jtag > 1") + ##tdSql.error("select jtag from jsons1 where jtag like 'fsss'") + ##tdSql.error("select jtag from jsons1 where jtag in (1)") + + + ## where json value is string + #tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing' order by dataint") + #tdSql.checkRows(2) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, 'jsons1_2') + #tdSql.checkData(0, 2, "5.000000000") + #tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, 3) + #tdSql.checkData(1, 1, 'jsons1_3') + #tdSql.checkData(1, 2, 'false') + + + #tdSql.query("select * from jsons1 where jtag->'tag1'='beijing'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>'beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'=''") + #tdSql.checkRows(2) + + ## where json value is int + #tdSql.query("select * from jsons1 where jtag->'tag1'=5") + #tdSql.checkRows(1) + #tdSql.checkData(0, 1, 2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=10") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<54") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>4") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=55") + #tdSql.checkRows(3) + + ## where json value is double + #tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'/0=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'/5=1") + #tdSql.checkRows(1) + + ## where json value is bool + #tdSql.query("select * from jsons1 where jtag->'tag1'=true") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'>false") + #tdSql.checkRows(0) + + ## where json value is null + #tdSql.query("select * from jsons1 where jtag->'tag1'=null") + #tdSql.checkRows(0) + + ## where json key is null + #tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + #tdSql.checkRows(0) + + ## where json value is not exist + #tdSql.query("select * from jsons1 where jtag->'tag1' is null") + #tdSql.checkData(0, 0, 'jsons1_9') + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag4' is null") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag->'tag3' is not null") + #tdSql.checkRows(3) + + ## test contains + #tdSql.query("select * from jsons1 where jtag contains 'tag1'") + #tdSql.checkRows(8) + #tdSql.query("select * from jsons1 where jtag contains 'tag3'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag contains 'tag_no_exist'") + #tdSql.checkRows(0) + + ## test json tag in where condition with and/or + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1' is not null and jtag contains 'tag3'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag contains 'tag3'") + #tdSql.checkRows(2) + + + ## test with between and + #tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") + #tdSql.checkRows(2) + + ## test with tbname/normal column + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=23") + #tdSql.checkRows(1) + + + ## test where condition like + #tdSql.query("select * from jsons1 where jtag->'tag2' like 'bei%'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") + #tdSql.checkRows(2) + + ## test where condition in no support in + #tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')") + + ## test where condition match/nmath + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma$'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag2' match 'jing$'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match '收到'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1' nmatch 'ma'") + #tdSql.checkRows(1) + + ## test distinct + #tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select distinct jtag->'tag1' from jsons1") + #tdSql.checkRows(7) + ## tdSql.query("select distinct jtag from jsons1") + ## tdSql.checkRows(9) + + ##test dumplicate key with normal colomn + #tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") + #tdSql.query("select * from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") + #tdSql.checkRows(1) + ## tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") + ## tdSql.checkRows(1) + + ## test join + #tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") + #tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") + + #tdSql.execute("create table if not exists jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons3_1 using jsons3 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 3, false, 'json3', '你是3')") + #tdSql.execute("insert into jsons3_2 using jsons3 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060638000, 2, true, 'json3', 'sss')") + #tdSql.query("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #tdSql.checkData(0, 0, "sss") + #tdSql.checkData(0, 2, "true") + + #res = tdSql.getColNameList("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #cname_list = [] + #cname_list.append("'sss'") + #cname_list.append("33") + #cname_list.append("a.jtag->'tag3'") + #tdSql.checkColNameList(res, cname_list) + ## + ## test group by & order by json tag + #tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(2, 1, '"收到货"') + #tdSql.checkData(7, 1, "false") + + + #tdSql.error("select count(*) from jsons1 group by jtag") + #tdSql.error("select count(*) from jsons1 partition by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag order by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(1, 0, 2) + #tdSql.checkData(1, 1, '"收到货"') + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "11.000000000") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "false") + + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "false") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "11.000000000") + #tdSql.checkData(7, 0, 2) + #tdSql.checkData(7, 1, '"femail"') + + ## test stddev with group by json tag + #tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(4, 0, 0) + #tdSql.checkData(4, 1, "5.000000000") + #tdSql.checkData(7, 0, 11) + #tdSql.checkData(7, 1, '"femail"') + + #res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + #cname_list = [] + #cname_list.append("stddev(dataint)") + #cname_list.append("jsons1.jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + ## test top/bottom with group by json tag + #tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, None) + ##tdSql.checkData(2, 0, 24) + ##tdSql.checkData(3, 0, 3) + ##tdSql.checkData(3, 1, "false") + ##tdSql.checkData(8, 0, 2) + ##tdSql.checkData(10, 1, '"femail"') + + ## test having + ## tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + ## tdSql.checkRows(3) + + ## subquery with json tag + #tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") + #tdSql.checkRows(11) + #tdSql.checkData(1, 1, 1) + #tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}') + + #tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") + #tdSql.error("select t->'tag1' from (select jtag->'tag1' as t, dataint from jsons1)") + ## tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)") + ## tdSql.checkRows(11) + ## tdSql.checkData(1, 1, "jsons1_1") + ## tdSql.checkData(1, 2, '"femail"') + + ## union all + #tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") + #tdSql.checkRows(3) + + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") + #tdSql.checkRows(3) + #tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") + #tdSql.checkRows(13) + + ##show create table + #tdSql.query("show create table jsons1") + #tdSql.checkData(0, 1, 'CREATE STABLE `jsons1` (`ts` TIMESTAMP, `dataint` INT, `databool` BOOL, `datastr` NCHAR(50), `datastrbin` VARCHAR(150)) TAGS (`jtag` JSON) WATERMARK 5000a, 5000a') + + ##test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares + #tdSql.query("select count(*) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select avg(dataint) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 5.3) + ## tdSql.query("select twa(dataint) from jsons1 where jtag is not null") + ## tdSql.checkData(0, 0, 28.386363636363637) + ## tdSql.query("select irate(dataint) from jsons1 where jtag is not null") + + #tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") + #tdSql.checkData(0, 0, 45) + #tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 4.496912521) + #tdSql.query("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null") + + ##test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp + #tdSql.query("select min(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1) + #tdSql.query("select max(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select first(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 2) + #tdSql.query("select last(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select top(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select bottom(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + ##tdSql.query("select percentile(dataint,20) from jsons1 where jtag->'tag1'>1") + #tdSql.query("select apercentile(dataint, 50) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1.5) + ## tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1") + ## tdSql.query("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1") + + ##test calculation function:diff/derivative/spread/ceil/floor/round/ + #tdSql.query("select diff(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(2) + ## tdSql.checkData(0, 0, -1) + ## tdSql.checkData(1, 0, 10) + #tdSql.query("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1") + ## tdSql.checkData(0, 0, -2) + #tdSql.query("select spread(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + + ##math function + #tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ## function not ready + #tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##str function + #tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + + ##time function + #tdSql.query("select now() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select today() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##test TD-12077 + #tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select jtag->'tag3' from jsons1_16") + #tdSql.checkData(0, 0, '-2.111000000') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + From e840cdaaace17846e7430af636556a608fc14d2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 20:56:29 +0800 Subject: [PATCH 78/94] fix(query): set correct sort buffer format --- source/libs/executor/src/tsort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 8779fe5415..9b15cc7adb 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -621,7 +621,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { pHandle->sortElapsed += el; // All sorted data can fit in memory, external memory sort is not needed. Return to directly - if (size <= sortBufSize) { + if (size <= sortBufSize && pHandle->pBuf == NULL) { pHandle->cmpParam.numOfSources = 1; pHandle->inMemSort = true; From 280ed4f65d88c470c4dfbfa957246b385359427c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 15 Jul 2022 21:29:53 +0800 Subject: [PATCH 79/94] feat(stream): pipelined exec for recovering --- include/libs/stream/tstream.h | 5 ++ source/dnode/mnode/impl/src/mndStream.c | 90 ++++++++++++++++++++++++- source/dnode/snode/inc/sndInt.h | 9 --- source/libs/stream/inc/streamInc.h | 2 + source/libs/stream/src/streamExec.c | 57 ++++++++++++++++ source/libs/stream/src/streamRecover.c | 6 +- 6 files changed, 156 insertions(+), 13 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d03fab0ebc..5c4d8ce250 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -31,6 +31,11 @@ extern "C" { typedef struct SStreamTask SStreamTask; +enum { + STREAM_STATUS__NORMAL = 0, + STREAM_STATUS__RECOVER, +}; + enum { TASK_STATUS__NORMAL = 0, TASK_STATUS__DROPPING, diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 8248a26c28..5e8867fdfb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -177,7 +177,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream taosWLockLatch(&pOldStream->lock); - // TODO handle update + pOldStream->status = pNewStream->status; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -395,6 +395,20 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr return 0; } +static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream) { + SStreamObj streamObj = {0}; + memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); + streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + return 0; +} + static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; @@ -492,6 +506,76 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { + SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pReq->streamId = pTask->streamId; + pReq->taskId = pTask->taskId; + int32_t len; + int32_t code; + tEncodeSize(tEncodeSMStreamTaskRecoverReq, pReq, len, code); + if (code != 0) { + return -1; + } + void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) { + return -1; + } + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSMStreamTaskRecoverReq(&encoder, pReq); + ((SMsgHead *)buf)->vgId = pTask->nodeId; + + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + len; + action.msgType = TDMT_STREAM_TASK_RECOVER; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + if (pStream->isDistributed) { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + SStreamTask *pTask = taosArrayGetP(pTasks, 0); + if (!pTask->isDataScan && pTask->execType != TASK_EXEC__NONE) { + ASSERT(sz == 1); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } else { + continue; + } + } + } else { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pTasks, j); + if (!pTask->isDataScan) break; + ASSERT(pTask->execType != TASK_EXEC__NONE); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } + } + } + return 0; +} + int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { @@ -712,14 +796,14 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); // broadcast to recover all tasks - if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); return -1; } // update stream status - if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); return -1; } diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 8916e2a31c..1f0019ef46 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -30,15 +30,6 @@ extern "C" { #endif -enum { - STREAM_STATUS__RUNNING = 1, - STREAM_STATUS__STOPPED, - STREAM_STATUS__CREATING, - STREAM_STATUS__STOPING, - STREAM_STATUS__RESTORING, - STREAM_STATUS__DELETING, -}; - typedef struct { SHashObj* pHash; // taskId -> SStreamTask } SStreamMeta; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index f9f4e62774..d10ea76c83 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -33,6 +33,8 @@ typedef struct { static SStreamGlobalEnv streamEnv; int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum); + int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cffd519d2a..9d22da8662 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,6 +82,63 @@ static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock return 0; } +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { + ASSERT(pTask->execType != TASK_EXEC__NONE); + + void* exec = pTask->exec.executor; + + while (1) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t batchCnt = 0; + while (1) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(0); + } + if (output == NULL) break; + + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + + if (++batchCnt >= batchNum) break; + } + if (taosArrayGetSize(pRes) == 0) { + taosArrayDestroy(pRes); + break; + } + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return -1; + } + + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + qRes->childId = pTask->selfChildId; + + if (streamTaskOutput(pTask, qRes) < 0) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return -1; + } + + if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + streamDispatch(pTask, pTask->pMsgCb); + } + } + + return 0; +} + static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { int32_t cnt = 0; void* data = NULL; diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 4175f60446..87b27daf60 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -125,7 +125,11 @@ int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* } if (pTask->taskStatus == TASK_STATUS__RECOVERING) { - streamProcessRunReq(pTask); + if (streamPipelineExec(pTask, 10) < 0) { + // set fail + return -1; + } } + return 0; } From 352db0febbfca2948786e16c66eb4b2092d11cb1 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 16 Jul 2022 09:28:15 +0800 Subject: [PATCH 80/94] feat(stream): optimize interval plan --- source/common/src/tdatablock.c | 2 +- source/libs/executor/inc/executorimpl.h | 16 +- source/libs/executor/src/scanoperator.c | 20 ++- source/libs/executor/src/timewindowoperator.c | 21 ++- source/libs/planner/src/planSpliter.c | 3 +- tests/script/tsim/stream/sliding.sim | 139 ++++++++++++++++++ 6 files changed, 183 insertions(+), 18 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 47cfa5d410..cd21de52ed 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1752,7 +1752,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld\n", flag, + len += snprintf(dumpBuf + len, size - len, "%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|======\n", "dumpBlockData", (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId, pDataBlock->info.uid); if (len >= size - 1) return dumpBuf; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f110b1c92e..cd50a13edd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -371,6 +371,13 @@ typedef struct SessionWindowSupporter { uint8_t parentType; } SessionWindowSupporter; +typedef struct STimeWindowSupp { + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. +} STimeWindowAggSupp; + typedef struct SStreamScanInfo { uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; @@ -407,6 +414,7 @@ typedef struct SStreamScanInfo { SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; STimeWindow updateWin; + STimeWindowAggSupp twAggSup; // status for tmq // SSchemaWrapper schema; @@ -452,13 +460,6 @@ typedef struct SAggSupporter { int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; -typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. -} STimeWindowAggSupp; - typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info @@ -952,6 +953,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index afb5e5ed45..a81a796fa7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -802,7 +802,12 @@ static bool isStateWindow(SStreamScanInfo* pInfo) { static bool isIntervalWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; +} + +static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; } static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { @@ -1130,9 +1135,14 @@ static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlo static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* ts = (TSKEY*)pColDataInfo->pData; + TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId]) && out) { + SResultRowInfo dumyInfo; + dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + // must check update info first. + bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); + if ( (update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup)) ) && out) { taosArrayPush(pInfo->tsArray, &rowId); } } @@ -1413,6 +1423,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); if (pInfo->pUpdateRes->info.rows > 0) { if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; @@ -1584,13 +1595,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; + pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); pInfo->pStreamScanOp = pOperator; pInfo->deleteDataIndex = 0; pInfo->pDeleteDataRes = createPullDataBlock(); pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX}; + pInfo->twAggSup = *pTwSup; pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index acc56af5cd..71eb221c64 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -652,8 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) { - qDebug("======printDataBlock Block is Null"); + if (!pBlock || pBlock->info.rows == 0) { + qDebug("======printDataBlock: Block is Null or Empty"); return; } char* pBuf = NULL; @@ -1355,7 +1355,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, int32_t size = taosArrayGetSize(chAy); qDebug("window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qDebug("window %" PRId64 " wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qDebug("window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { @@ -1626,6 +1626,12 @@ SSDataBlock* createDeleteBlock() { return pBlock; } +void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type) { + ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanInfo* pScanInfo = downstream->info; + pScanInfo->sessionSup.parentType = type; +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, @@ -1701,6 +1707,10 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { + initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL); + } + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2476,12 +2486,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } else { int32_t index = -1; SArray* chArray = NULL; + int32_t chId = 0; if (chIds) { chArray = *(void**)chIds; - int32_t chId = getChildIndex(pSDataBlock); + chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { + qDebug("======delete child id %d", chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over @@ -3010,6 +3022,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; } } + void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 871bcf015b..ae0ccb1c51 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -198,8 +198,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || - (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 1ffca2a67b..f34a50de9d 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -366,4 +366,143 @@ if $data32 != 8 then goto loop1 endi +sql drop database IF EXISTS test2; +sql drop stream IF EXISTS streams21; +sql drop stream IF EXISTS streams22; + +sql create database test2 vgroups 2; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,1.1); +sql insert into t1 values(1648791233002,3,3,3,2.1); +sql insert into t1 values(1648791243003,4,4,4,3.1); +sql insert into t1 values(1648791213004,4,5,5,4.1); + +sql insert into t2 values(1648791213000,1,6,6,1.0); +sql insert into t2 values(1648791223001,2,7,7,1.1); +sql insert into t2 values(1648791233002,3,8,8,2.1); +sql insert into t2 values(1648791243003,4,9,9,3.1); +sql insert into t2 values(1648791213004,4,10,10,4.1); + +$loop_count = 0 + +loop2: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt; + +# row 0 +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop2 +endi + +# row 2 +if $data21 != 1 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop2 +endi + +# row 3 +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 4 then + print =====data32=$data32 + goto loop2 +endi + +print step 6 + +$loop_count = 0 + +loop3: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; + +# row 0 +if $data01 != 4 then + print =====data01=$data01 + # goto loop3 +endi + +if $data02 != 10 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop3 +endi + +# row 2 +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 6 then + print =====data22=$data22 + goto loop3 +endi + +# row 3 +if $data31 != 2 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 8 then + print =====data32=$data32 + goto loop3 +endi + system sh/stop_dnodes.sh \ No newline at end of file From af24f2f85d414b636b0e48e0108f3f3a86a7267f Mon Sep 17 00:00:00 2001 From: huolibo Date: Sat, 16 Jul 2022 09:32:31 +0800 Subject: [PATCH 81/94] fix(driver): tmq jdbc get meta --- .../jni/com_taosdata_jdbc_tmq_TMQConnector.h | 2 +- source/client/src/TMQConnector.c | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index b3b098e460..197cd78006 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -161,7 +161,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;ILjava/util/List;)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *, jobject, jlong, jlong, - jobject, jint, jobject); + jobject, jobject); #ifdef __cplusplus } diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index 1d84dcf7a2..3755a591e3 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -278,7 +278,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj, jint flag, + jlong res, jobject rowobj, jobject arrayListObj) { TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); @@ -309,16 +309,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( TAOS_FIELD *fields = taos_fetch_fields(tres); jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, numOfFields); - if (flag) { - for (int i = 0; i < numOfFields; ++i) { - jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); - (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); - (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); - (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); - jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); - (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); - (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); - } + for (int i = 0; i < numOfFields; ++i) { + jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); + (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); + (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); + (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); + jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); + (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); + (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); } (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); From 52327cc2fb578a4a40c91c2d7f7ad995bbb174cf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 09:44:52 +0800 Subject: [PATCH 82/94] fix: multi-rows function validity check --- source/libs/parser/src/parTranslater.c | 5 +++-- tests/system-test/2-query/statecount.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 33ab1166b0..3164623465 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1058,8 +1058,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (!isSelectStmt(pCxt->pCurrStmt) || ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || - ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { + // The auto-generated COUNT function in the DELETE statement is legal + if (isSelectStmt(pCxt->pCurrStmt) && + (((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 90e8bebab4..91e2aa9e47 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -335,7 +335,7 @@ class TDTestCase: # tdSql.checkRows(21) # group by - tdSql.query("select statecount(c1,'GT',1) from ct1 group by c1") + tdSql.error("select statecount(c1,'GT',1) from ct1 group by c1") tdSql.error("select statecount(c1,'GT',1) from ct1 group by tbname") # super table From be8e5325b0aae42e627deffc381582938d3f84c6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 10:47:32 +0800 Subject: [PATCH 83/94] change submodule ref point --- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/taos-tools b/tools/taos-tools index 1163c0f60a..50b68d85f7 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 1163c0f60aa65d6cc58283247c8bf8c56ba43b92 +Subproject commit 50b68d85f7cbaf7a9adfa4082e88ca758770f75e diff --git a/tools/taosadapter b/tools/taosadapter index c3815951fc..d9f9e69960 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit c3815951fc80617ecd171f3743b8b4a4d0bc712e +Subproject commit d9f9e69960385eec7e3f7d0582b932d143014ae1 From 5bbbb5d214bb7d8792316559f47772351d9ea4f8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 10:51:13 +0800 Subject: [PATCH 84/94] restore submodule refpoint --- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/taos-tools b/tools/taos-tools index 50b68d85f7..bd496f76b6 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 50b68d85f7cbaf7a9adfa4082e88ca758770f75e +Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde diff --git a/tools/taosadapter b/tools/taosadapter index d9f9e69960..df8678f070 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit d9f9e69960385eec7e3f7d0582b932d143014ae1 +Subproject commit df8678f070e3f707faf59baebec90065f6e1268b From 598c57e05966b82e577c366bf94d33875c82a2e1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:26:04 +0800 Subject: [PATCH 85/94] feat(query): add irate function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index e0beaff262..d743a9ac02 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -107,6 +107,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c458e2caec..99bcc2cb59 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2217,6 +2217,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getIrateFuncEnv, .initFunc = irateFuncSetup, .processFunc = irateFunction, + .sprocessFunc = irateScalarFunction, .finalizeFunc = irateFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9a8f570a52..87d1f0c683 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2380,3 +2380,29 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + //int64_t count = 0, sum = 0, qSum = 0; + bool hasNull = false; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + } + + double *out = (double *)pOutputData->pData; + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + *out = 0; + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} From 566ab93f839db501d4a9bef816aa4b283aa0db52 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:37:05 +0800 Subject: [PATCH 86/94] add derivative scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index d743a9ac02..582fe5a09b 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -107,6 +107,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 99bcc2cb59..b9e2b6feaf 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2207,6 +2207,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getDerivativeFuncEnv, .initFunc = derivativeFuncSetup, .processFunc = derivativeFunction, + .sprocessFunc = derivativeScalarFunction, .finalizeFunc = functionFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 87d1f0c683..5ccc23c2d4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2381,6 +2381,11 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + pOutput->numOfRows = 0; + return TSDB_CODE_SUCCESS; +} + int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; From 0ed38565c1814266c9fef41073bad5703624e56c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Sat, 16 Jul 2022 11:44:11 +0800 Subject: [PATCH 87/94] doc: english version of cluster chapter and some refine for chinese version --- docs/en/10-cluster/01-deploy.md | 48 ++++------ docs/en/10-cluster/02-cluster-mgmt.md | 101 +++------------------ docs/en/10-cluster/03-ha-and-lb.md | 81 ----------------- docs/en/10-cluster/03-high-availability.md | 30 ++++++ docs/en/10-cluster/04-load-balance.md | 20 ++++ docs/zh/10-cluster/01-deploy.md | 21 +++-- docs/zh/10-cluster/02-cluster-mgmt.md | 10 +- docs/zh/10-cluster/03-high-availability.md | 2 +- docs/zh/10-cluster/04-load-balance.md | 8 +- 9 files changed, 104 insertions(+), 217 deletions(-) delete mode 100644 docs/en/10-cluster/03-ha-and-lb.md create mode 100644 docs/en/10-cluster/03-high-availability.md create mode 100644 docs/en/10-cluster/04-load-balance.md diff --git a/docs/en/10-cluster/01-deploy.md b/docs/en/10-cluster/01-deploy.md index 200da1be3f..d998fd6ad0 100644 --- a/docs/en/10-cluster/01-deploy.md +++ b/docs/en/10-cluster/01-deploy.md @@ -6,9 +6,9 @@ title: Deployment ### Step 1 -The FQDN of all hosts must be setup properly. For e.g. FQDNs may have to be configured in the /etc/hosts file on each host. You must confirm that each FQDN can be accessed from any other host. For e.g. you can do this by using the `ping` command. +The FQDN of all hosts must be setup properly. All FQDNs need to be configured in the /etc/hosts file on each host. You must confirm that each FQDN can be accessed from any other host, you can do this by using the `ping` command. -To get the hostname on any host, the command `hostname -f` can be executed. `ping ` command can be executed on each host to check whether any other host is accessible from it. If any host is not accessible, the network configuration, like /etc/hosts or DNS configuration, needs to be checked and revised, to make any two hosts accessible to each other. +The command `hostname -f` can be executed to get the hostname on any host. `ping ` command can be executed on each host to check whether any other host is accessible from it. If any host is not accessible, the network configuration, like /etc/hosts or DNS configuration, needs to be checked and revised, to make any two hosts accessible to each other. :::note @@ -20,7 +20,7 @@ To get the hostname on any host, the command `hostname -f` can be executed. `pin ### Step 2 -If any previous version of TDengine has been installed and configured on any host, the installation needs to be removed and the data needs to be cleaned up. For details about uninstalling please refer to [Install and Uninstall](/operation/pkg-install). To clean up the data, please use `rm -rf /var/lib/taos/\*` assuming the `dataDir` is configured as `/var/lib/taos`. +If any previous version of TDengine has been installed and configured on any host, the installation needs to be removed and the data needs to be cleaned up. For details about uninstalling please refer to [Install and Uninstall](/operation/pkg-install). To clean up the data, please use `rm -rf /var/lib/taos/\*` assuming the `dataDir` is configured as `/var/lib/taos`. :::note @@ -54,22 +54,12 @@ serverPort 6030 For all the dnodes in a TDengine cluster, the below parameters must be configured exactly the same, any node whose configuration is different from dnodes already in the cluster can't join the cluster. -| **#** | **Parameter** | **Definition** | -| ----- | ------------------ | --------------------------------------------------------------------------------- | -| 1 | numOfMnodes | The number of management nodes in the cluster | -| 2 | mnodeEqualVnodeNum | The ratio of resource consuming of mnode to vnode | -| 3 | offlineThreshold | The threshold of dnode offline, once it's reached the dnode is considered as down | -| 4 | statusInterval | The interval by which dnode reports its status to mnode | -| 5 | arbitrator | End point of the arbitrator component in the cluster | -| 6 | timezone | Timezone | -| 7 | balance | Enable load balance automatically | -| 8 | maxTablesPerVnode | Maximum number of tables that can be created in each vnode | -| 9 | maxVgroupsPerDb | Maximum number vgroups that can be used by each DB | - -:::note -Prior to version 2.0.19.0, besides the above parameters, `locale` and `charset` must also be configured the same for each dnode. - -::: +| **#** | **Parameter** | **Definition** | +| ----- | -------------- | ------------------------------------------------------------- | +| 1 | statusInterval | The time interval for which dnode reports its status to mnode | +| 2 | timezone | Time Zone where the server is located | +| 3 | locale | Location code of the system | +| 4 | charset | Character set of the system | ## Start Cluster @@ -77,19 +67,19 @@ In the following example we assume that first dnode has FQDN h1.taosdata.com and ### Start The First DNODE -The first dnode can be started following the instructions in [Get Started](/get-started/). Then TDengine CLI `taos` can be launched to execute command `show dnodes`, the output is as following for example: +Start the first dnode following the instructions in [Get Started](/get-started/). Then launch TDengine CLI `taos` and execute command `show dnodes`, the output is as following for example: ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. - -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taosdata.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | +Query OK, 1 rows affected (0.007984s) taos> ``` @@ -100,7 +90,7 @@ From the above output, it is shown that the end point of the started dnode is "h There are a few steps necessary to add other dnodes in the cluster. -Let's assume we are starting the second dnode with FQDN, h2.taosdata.com. First we make sure the configuration is correct. +Let's assume we are starting the second dnode with FQDN, h2.taosdata.com. Firstly we make sure the configuration is correct. ```c // firstEp is the end point to connect to when any dnode starts @@ -114,7 +104,7 @@ serverPort 6030 ``` -Second, we can start `taosd` as instructed in [Get Started](/get-started/). +Secondly, we can start `taosd` as instructed in [Get Started](/get-started/). Then, on the first dnode i.e. h1.taosdata.com in our example, use TDengine CLI `taos` to execute the following command to add the end point of the dnode in the cluster. In the command "fqdn:port" should be quoted using double quotes. diff --git a/docs/en/10-cluster/02-cluster-mgmt.md b/docs/en/10-cluster/02-cluster-mgmt.md index bd3386c411..19ee034127 100644 --- a/docs/en/10-cluster/02-cluster-mgmt.md +++ b/docs/en/10-cluster/02-cluster-mgmt.md @@ -39,31 +39,25 @@ USE SOME_DATABASE; SHOW VGROUPS; ``` -The example output is below: - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) +The output is like below: taos> use db; Database changed. taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | +vgId | tables | status | onlines | v1_dnode | v1_status | compacting | ========================================================================================== - 14 | 38000 | ready | 1 | 1 | leader | 0 | - 15 | 38000 | ready | 1 | 1 | leader | 0 | - 16 | 38000 | ready | 1 | 1 | leader | 0 | - 17 | 38000 | ready | 1 | 1 | leader | 0 | - 18 | 37001 | ready | 1 | 1 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | +14 | 38000 | ready | 1 | 1 | leader | 0 | +15 | 38000 | ready | 1 | 1 | leader | 0 | +16 | 38000 | ready | 1 | 1 | leader | 0 | +17 | 38000 | ready | 1 | 1 | leader | 0 | +18 | 37001 | ready | 1 | 1 | leader | 0 | +19 | 37000 | ready | 1 | 1 | leader | 0 | +20 | 37000 | ready | 1 | 1 | leader | 0 | +21 | 37000 | ready | 1 | 1 | leader | 0 | Query OK, 8 row(s) in set (0.001154s) -``` + +```` ## Add DNODE @@ -71,7 +65,7 @@ Launch TDengine CLI `taos` and execute the command below to add the end point of ```sql CREATE DNODE "fqdn:port"; -``` +```` The example output is as below: @@ -142,72 +136,3 @@ In the above example, when `show dnodes` is executed the first time, two dnodes - dnodeID is allocated automatically and can't be manually modified. dnodeID is generated in ascending order without duplication. ::: - -## Move VNODE - -A vnode can be manually moved from one dnode to another. - -Launch TDengine CLI `taos` and execute below command: - -```sql -ALTER DNODE BALANCE "VNODE:-DNODE:"; -``` - -In the above command, `source-dnodeId` is the original dnodeId where the vnode resides, `dest-dnodeId` specifies the target dnode. vgId (vgroup ID) can be shown by `SHOW VGROUPS `. - -First `show vgroups` is executed to show the vgroup distribution. - -``` -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 3 | leader | 0 | - 15 | 38000 | ready | 1 | 3 | leader | 0 | - 16 | 38000 | ready | 1 | 3 | leader | 0 | - 17 | 38000 | ready | 1 | 3 | leader | 0 | - 18 | 37001 | ready | 1 | 3 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | -Query OK, 8 row(s) in set (0.001314s) -``` - -It can be seen that there are 5 vgroups in dnode 3 and 3 vgroups in node 1, now we want to move vgId 18 from dnode 3 to dnode 1. Execute the below command in `taos` - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; - -DB error: Balance already enabled (0.00755 -``` - -However, the operation fails with error message show above, which means automatic load balancing has been enabled in the current database so manual load balance can't be performed. - -Shutdown the cluster, configure `balance` parameter in all the dnodes to 0, then restart the cluster, and execute `alter dnode` and `show vgroups` as below. - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; -Query OK, 0 row(s) in set (0.000575s) - -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | -================================================================================================================= - 14 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | follower | 3 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | -Query OK, 8 row(s) in set (0.001242s) -``` - -It can be seen from above output that vgId 18 has been moved from dnode 3 to dnode 1. - -:::note - -- Manual load balancing can only be performed when the automatic load balancing is disabled, i.e. `balance` is set to 0. -- Only a vnode in normal state, i.e. leader or follower, can be moved. vnode can't be moved when its in status offline, unsynced or syncing. -- Before moving a vnode, it's necessary to make sure the target dnode has enough resources: CPU, memory and disk. - -::: diff --git a/docs/en/10-cluster/03-ha-and-lb.md b/docs/en/10-cluster/03-ha-and-lb.md deleted file mode 100644 index 9780e8f6c6..0000000000 --- a/docs/en/10-cluster/03-ha-and-lb.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -sidebar_label: HA & LB -title: High Availability and Load Balancing ---- - -## High Availability of Vnode - -High availability of vnode and mnode can be achieved through replicas in TDengine. - -A TDengine cluster can have multiple databases. Each database has a number of vnodes associated with it. A different number of replicas can be configured for each DB. When creating a database, the parameter `replica` is used to specify the number of replicas. The default value for `replica` is 1. Naturally, a single replica cannot guarantee high availability since if one node is down, the data service is unavailable. Note that the number of dnodes in the cluster must NOT be lower than the number of replicas set for any DB, otherwise the `create table` operation will fail with error "more dnodes are needed". The SQL statement below is used to create a database named "demo" with 3 replicas. - -```sql -CREATE DATABASE demo replica 3; -``` - -The data in a DB is divided into multiple shards and stored in multiple vgroups. The number of vnodes in each vgroup is determined by the number of replicas set for the DB. The vnodes in each vgroup store exactly the same data. For the purpose of high availability, the vnodes in a vgroup must be located in different dnodes on different hosts. As long as over half of the vnodes in a vgroup are in an online state, the vgroup is able to provide data access. Otherwise the vgroup can't provide data access for reading or inserting data. - -There may be data for multiple DBs in a dnode. When a dnode is down, multiple DBs may be affected. While in theory, the cluster will provide data access for reading or inserting data if over half the vnodes in vgroups are online, because of the possibly complex mapping between vnodes and dnodes, it is difficult to guarantee that the cluster will work properly if over half of the dnodes are online. - -## High Availability of Mnode - -Each TDengine cluster is managed by `mnode`, which is a module of `taosd`. For the high availability of mnode, multiple mnodes can be configured using system parameter `numOfMNodes`. The valid range for `numOfMnodes` is [1,3]. To ensure data consistency between mnodes, data replication between mnodes is performed synchronously. - -There may be multiple dnodes in a cluster, but only one mnode can be started in each dnode. Which one or ones of the dnodes will be designated as mnodes is automatically determined by TDengine according to the cluster configuration and system resources. The command `show mnodes` can be executed in TDengine `taos` to show the mnodes in the cluster. - -```sql -SHOW MNODES; -``` - -The end point and role/status (leader, follower, unsynced, or offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. If `numOfMNodes` is configured to 2, another mnode will be started when the second dnode is launched. - -For the high availability of mnode, `numOfMnodes` needs to be configured to 2 or a higher value. Because the data consistency between mnodes must be guaranteed, the replica confirmation parameter `quorum` is set to 2 automatically if `numOfMNodes` is set to 2 or higher. - -:::note -If high availability is important for your system, both vnode and mnode must be configured to have multiple replicas. - -::: - -## Load Balancing - -Load balancing will be triggered in 3 cases without manual intervention. - -- When a new dnode joins the cluster, automatic load balancing may be triggered. Some data from other dnodes may be transferred to the new dnode automatically. -- When a dnode is removed from the cluster, the data from this dnode will be transferred to other dnodes automatically. -- When a dnode is too hot, i.e. too much data has been stored in it, automatic load balancing may be triggered to migrate some vnodes from this dnode to other dnodes. - -:::tip -Automatic load balancing is controlled by the parameter `balance`, 0 means disabled and 1 means enabled. This is set in the file [taos.cfg](https://docs.tdengine.com/reference/config/#balance). - -::: - -## Dnode Offline - -When a dnode is offline, it can be detected by the TDengine cluster. There are two cases: - -- The dnode comes online before the threshold configured in `offlineThreshold` is reached. The dnode is still in the cluster and data replication is started automatically. The dnode can work properly after the data sync is finished. - -- If the dnode has been offline over the threshold configured in `offlineThreshold` in `taos.cfg`, the dnode will be removed from the cluster automatically. A system alert will be generated and automatic load balancing will be triggered if `balance` is set to 1. When the removed dnode is restarted and becomes online, it will not join the cluster automatically. The system administrator has to manually join the dnode to the cluster. - -:::note -If all the vnodes in a vgroup (or mnodes in mnode group) are in offline or unsynced status, the leader node can only be voted on, after all the vnodes or mnodes in the group become online and can exchange status. Following this, the vgroup (or mnode group) is able to provide service. - -::: - -## Arbitrator - -The "arbitrator" component is used to address the special case when the number of replicas is set to an even number like 2,4 etc. If half of the vnodes in a vgroup don't work, it is impossible to vote and select a leader node. This situation also applies to mnodes if the number of mnodes is set to an even number like 2,4 etc. - -To resolve this problem, a new arbitrator component named `tarbitrator`, an abbreviation of TDengine Arbitrator, was introduced. The `tarbitrator` simulates a vnode or mnode but it's only responsible for network communication and doesn't handle any actual data access. As long as more than half of the vnode or mnode, including Arbitrator, are available the vnode group or mnode group can provide data insertion or query services normally. - -Normally, it's prudent to configure the replica number for each DB or system parameter `numOfMNodes` to be an odd number. However, if a user is very sensitive to storage space, a replica number of 2 plus arbitrator component can be used to achieve both lower cost of storage space and high availability. - -Arbitrator component is installed with the server package. For details about how to install, please refer to [Install](/operation/pkg-install). The `-p` parameter of `tarbitrator` can be used to specify the port on which it provides service. - -In the configuration file `taos.cfg` of each dnode, parameter `arbitrator` needs to be configured to the end point of the `tarbitrator` process. Arbitrator component will be used automatically if the replica is configured to an even number and will be ignored if the replica is configured to an odd number. - -Arbitrator can be shown by executing command in TDengine CLI `taos` with its role shown as "arb". - -```sql -SHOW DNODES; -``` diff --git a/docs/en/10-cluster/03-high-availability.md b/docs/en/10-cluster/03-high-availability.md new file mode 100644 index 0000000000..e2e1c6521e --- /dev/null +++ b/docs/en/10-cluster/03-high-availability.md @@ -0,0 +1,30 @@ +--- +sidebar_label: High Availability +title: High Availability +--- + +## High Availability of Vnode + +High availability of vnode can be achieved through replicas in TDengine. + +A TDengine cluster can have multiple databases. Each database has a number of vnodes associated with it. A different number of replicas can be configured for each DB. When creating a database, the parameter `replica` is used to specify the number of replicas. The default value for `replica` is 1. Naturally, a single replica cannot guarantee high availability since if one node is down, the data service is unavailable. Note that the number of dnodes in the cluster must NOT be lower than the number of replicas set for any DB, otherwise the `create table` operation will fail with error "more dnodes are needed". The SQL statement below is used to create a database named "demo" with 3 replicas. + +```sql +CREATE DATABASE demo replica 3; +``` + +The data in a DB is divided into multiple shards and stored in multiple vgroups. The number of vnodes in each vgroup is determined by the number of replicas set for the DB. The vnodes in each vgroup store exactly the same data. For the purpose of high availability, the vnodes in a vgroup must be located in different dnodes on different hosts. As long as over half of the vnodes in a vgroup are in an online state, the vgroup is able to provide data access. Otherwise the vgroup can't provide data access for reading or inserting data. + +There may be data for multiple DBs in a dnode. When a dnode is down, multiple DBs may be affected. While in theory, the cluster will provide data access for reading or inserting data if over half the vnodes in vgroups are online, because of the possibly complex mapping between vnodes and dnodes, it is difficult to guarantee that the cluster will work properly if over half of the dnodes are online. + +## High Availability of Mnode + +Each TDengine cluster is managed by `mnode`, which is a module of `taosd`. For the high availability of mnode, multiple mnodes can be configured in the system. When a TDengine cluster is started from scratch, there is only one `mnode`, then you can use command `create mnode` to and start corresponding dnode to add more mnodes. + +```sql +SHOW MNODES; +``` + +The end point and role/status (leader, follower, candidate, offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. + +From TDengine 3.0.0, RAFT procotol is used to guarantee the high availability, so the number of mnodes is should be 1 or 3. diff --git a/docs/en/10-cluster/04-load-balance.md b/docs/en/10-cluster/04-load-balance.md new file mode 100644 index 0000000000..7648398059 --- /dev/null +++ b/docs/en/10-cluster/04-load-balance.md @@ -0,0 +1,20 @@ +--- +sidebar_label: Load Balance +title: Load Balance +--- + +The load balance in TDengine is mainly about processing data series data. TDengine employes builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table. + +The number of vgroup can be specified when creating a database, using the parameter `vgroups`. + +```sql +create database db0 vgroups 100; +``` + +The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may downgrad the system performance significantly. If multiple databases are to be created in the system, then the total number of `vroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases. + +Furthermode, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distrubtion is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck. + +TDegnine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput. + +Once the load balance is achieved, after some operations like deleting tables or droping databases, the load across all dnodes may become inbalanced, the method of rebalance will be provided in later versions. However, even without explicit rebalancing, TDengine will try its best to achieve new balance without manual interfering when a new database is created. \ No newline at end of file diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-cluster/01-deploy.md index 9dbfb50f18..cd19f90ba1 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-cluster/01-deploy.md @@ -72,19 +72,22 @@ serverPort 6030 按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 taos shell，从 shell 里执行命令“SHOW DNODES”，如下所示： ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. - -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taos.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | +Query OK, 1 rows affected (0.007984s) taos> -``` + +taos> + +```` 上述命令里，可以看到刚启动的数据节点的 End Point 是：h1.taos.com:6030，就是这个新集群的 firstEp。 @@ -98,7 +101,7 @@ taos> ```sql CREATE DNODE "h2.taos.com:6030"; -``` +```` 将新数据节点的 End Point（准备工作中第四步获知的）添加进集群的 EP 列表。“fqdn:port”需要用双引号引起来，否则出错。请注意将示例的“h2.taos.com:6030” 替换为这个新数据节点的 End Point。 diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md index f02a105826..5c490516f0 100644 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ b/docs/zh/10-cluster/02-cluster-mgmt.md @@ -72,10 +72,11 @@ CREATE DNODE "fqdn:port"; taos> show dnodes; id | endpoint | vnodes | support_vnodes | status | create_time | note | ============================================================================================================================================ - 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | - 2 | trd04:6030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | + 1 | localhost:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | + 2 | localhost:7030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | Query OK, 2 rows affected (0.007031s) ``` + 从中可以看到两个 dnode 状态都为 ready ## 删除数据节点 @@ -85,14 +86,15 @@ Query OK, 2 rows affected (0.007031s) ```sql DROP DNODE "fqdn:port"; ``` + 或者 + ```sql DROP DNODE dnodeId; ``` 通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 - :::warning 数据节点一旦被 drop 之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成 `drop dnode` 操作之前，会将该 dnode 的数据迁移走。 @@ -101,5 +103,3 @@ DROP DNODE dnodeId; dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 ::: - - diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index 36f9300366..ba056b6f16 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -28,6 +28,6 @@ TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管 SHOW MNODES; ``` -来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 +来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate, offline）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index c1a72cf006..2376dd3e61 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -2,17 +2,17 @@ title: 负载均衡 --- -TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroups 中，每张表或子表只能由一个 vgroups 处理，一个 vgroups 可能负责处理多个表或子表。 +TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroup 中，每张表或子表只能由一个 vgroup 处理，一个 vgroup 可能负责处理多个表或子表。 -创建数据库时可以指定其中的 vgroups 的数量： +创建数据库时可以指定其中的 vgroup 的数量： ```sql create database db0 vgroups 100; ``` -如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 +如何指定合适的 vgroup 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroup 数量由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroup 也越多。但也要考虑到磁盘性能，过多的 vgroup 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgroup 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroup。实际中建议首先根据系统资源配置选择一个初始的 vgroup 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroup 数量配置，此为系统中的 vgroup 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroup 进行分配。 -此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 +此外，对于任意数据库的 vgroup，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 From ee5d2c1d8248a1d9c0baf8de3ba17a9d1546fb55 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:46:49 +0800 Subject: [PATCH 88/94] refactor --- source/libs/scalar/src/sclfunc.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5ccc23c2d4..13b29f8ae3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2381,17 +2381,11 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - pOutput->numOfRows = 0; - return TSDB_CODE_SUCCESS; -} - -int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +int32_t nonCalcScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; int32_t type = GET_PARAM_TYPE(pInput); - //int64_t count = 0, sum = 0, qSum = 0; bool hasNull = false; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -2411,3 +2405,11 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} + +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} From 7c585d8ed9d12b0af10949d45a85a854c1af21bc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 11:47:26 +0800 Subject: [PATCH 89/94] fix: database options 'strict' and 'cachelast' syntax adjustments --- include/common/ttokendef.h | 4 +- include/libs/nodes/cmdnodes.h | 4 +- include/util/tdef.h | 33 ++-- source/common/src/systable.c | 6 +- source/dnode/mnode/impl/src/mndDb.c | 31 +++- source/libs/nodes/src/nodesCodeFuncs.c | 6 +- source/libs/parser/inc/parAst.h | 4 +- source/libs/parser/inc/sql.y | 10 +- source/libs/parser/src/parAstCreater.c | 18 +- source/libs/parser/src/parTokenizer.c | 4 +- source/libs/parser/src/parTranslater.c | 48 +++++- source/libs/parser/src/sql.c | 172 ++++++++++---------- source/libs/parser/test/parInitialATest.cpp | 2 +- source/libs/parser/test/parInitialCTest.cpp | 15 +- tests/pytest/alter/alter_cacheLastRow.py | 4 +- tests/pytest/query/last_cache.py | 12 +- tests/pytest/query/last_row_cache.py | 66 ++++---- tests/pytest/util/boundary.py | 2 +- tests/script/tsim/db/alter_option.sim | 12 +- tests/script/tsim/db/basic6.sim | 2 +- tests/script/tsim/db/create_all_options.sim | 2 +- tests/script/tsim/parser/last_cache.sim | 2 +- tests/script/tsim/parser/like.sim | 2 +- tests/system-test/2-query/last_row.py | 8 +- tests/system-test/6-cluster/5dnode1mnode.py | 6 +- 25 files changed, 267 insertions(+), 208 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index e2089f3023..0fce573af9 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -79,8 +79,8 @@ #define TK_NOT 61 #define TK_EXISTS 62 #define TK_BUFFER 63 -#define TK_CACHELAST 64 -#define TK_CACHELASTSIZE 65 +#define TK_CACHEMODEL 64 +#define TK_CACHESIZE 65 #define TK_COMP 66 #define TK_DURATION 67 #define TK_NK_VARIABLE 68 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5311915612..e13f85002c 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -51,7 +51,8 @@ extern "C" { typedef struct SDatabaseOptions { ENodeType type; int32_t buffer; - int8_t cacheLast; + char cacheModelStr[TSDB_CACHE_MODEL_STR_LEN]; + int8_t cacheModel; int32_t cacheLastSize; int8_t compressionLevel; int32_t daysPerFile; @@ -66,6 +67,7 @@ typedef struct SDatabaseOptions { char precisionStr[3]; int8_t precision; int8_t replica; + char strictStr[TSDB_DB_STRICT_STR_LEN]; int8_t strict; int8_t walLevel; int32_t numOfVgroups; diff --git a/include/util/tdef.h b/include/util/tdef.h index 55194d8647..3b31398063 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -53,7 +53,7 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL #define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF #define TSDB_DATA_BINARY_NULL 0xFF @@ -107,9 +107,10 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 -#define TSDB_TICK_PER_SECOND(precision) \ - ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1000LL \ - : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) +#define TSDB_TICK_PER_SECOND(precision) \ + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI \ + ? 1000LL \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) #define T_APPEND_MEMBER(dst, ptr, type, member) \ @@ -328,15 +329,25 @@ typedef enum ELogicConditionType { #define TSDB_MIN_DB_REPLICA 1 #define TSDB_MAX_DB_REPLICA 3 #define TSDB_DEFAULT_DB_REPLICA 1 +#define TSDB_DB_STRICT_STR_LEN sizeof(TSDB_DB_STRICT_OFF_STR) +#define TSDB_DB_STRICT_OFF_STR "off" +#define TSDB_DB_STRICT_ON_STR "on" #define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_ON 1 -#define TSDB_DEFAULT_DB_STRICT 0 -#define TSDB_MIN_DB_CACHE_LAST 0 -#define TSDB_MAX_DB_CACHE_LAST 3 -#define TSDB_DEFAULT_CACHE_LAST 0 -#define TSDB_MIN_DB_CACHE_LAST_SIZE 1 // MB -#define TSDB_MAX_DB_CACHE_LAST_SIZE 65536 -#define TSDB_DEFAULT_CACHE_LAST_SIZE 1 +#define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_OFF +#define TSDB_CACHE_MODEL_STR_LEN sizeof(TSDB_CACHE_MODEL_LAST_VALUE_STR) +#define TSDB_CACHE_MODEL_NONE_STR "none" +#define TSDB_CACHE_MODEL_LAST_ROW_STR "last_row" +#define TSDB_CACHE_MODEL_LAST_VALUE_STR "last_value" +#define TSDB_CACHE_MODEL_BOTH_STR "both" +#define TSDB_CACHE_MODEL_NONE 0 +#define TSDB_CACHE_MODEL_LAST_ROW 1 +#define TSDB_CACHE_MODEL_LAST_VALUE 2 +#define TSDB_CACHE_MODEL_BOTH 3 +#define TSDB_DEFAULT_CACHE_MODEL TSDB_CACHE_MODEL_NONE +#define TSDB_MIN_DB_CACHE_SIZE 1 // MB +#define TSDB_MAX_DB_CACHE_SIZE 65536 +#define TSDB_DEFAULT_CACHE_SIZE 1 #define TSDB_DB_STREAM_MODE_OFF 0 #define TSDB_DB_STREAM_MODE_ON 1 #define TSDB_DEFAULT_DB_STREAM_MODE 0 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 08997bcaf8..c9c1c1b395 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -76,7 +76,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "strict", .bytes = TSDB_DB_STRICT_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "duration", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "keep", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -87,9 +87,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "cache_model", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "cacheModel", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "single_stable_model", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, + {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, // {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "retention", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b1c2b0e277..715340f688 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -293,7 +293,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1; if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1; - if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_LAST_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_LAST_SIZE) return -1; + if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; @@ -312,7 +312,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; - if (pCfg->cacheLast < TSDB_MIN_DB_CACHE_LAST || pCfg->cacheLast > TSDB_MAX_DB_CACHE_LAST) return -1; + if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; if (pCfg->hashMethod != 1) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -341,8 +341,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA; if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; - if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_LAST; - if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_MODEL; + if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; } @@ -1443,6 +1443,22 @@ char *buildRetension(SArray *pRetension) { return p1; } +static const char *getCacheModelStr(int8_t cacheModel) { + switch (cacheModel) { + case TSDB_CACHE_MODEL_NONE: + return TSDB_CACHE_MODEL_NONE_STR; + case TSDB_CACHE_MODEL_LAST_ROW: + return TSDB_CACHE_MODEL_LAST_ROW_STR; + case TSDB_CACHE_MODEL_LAST_VALUE: + return TSDB_CACHE_MODEL_LAST_VALUE_STR; + case TSDB_CACHE_MODEL_BOTH: + return TSDB_CACHE_MODEL_BOTH_STR; + default: + break; + } + return "unknown"; +} + static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb, ESdbStatus objStatus, bool sysinfo) { int32_t cols = 0; @@ -1491,7 +1507,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); - const char *strictStr = pDb->cfg.strict ? "strict" : "no_strict"; + const char *strictStr = pDb->cfg.strict ? "on" : "off"; char strictVstr[24] = {0}; STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1539,8 +1555,11 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); + const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); + char cacheModelVstr[24] = {0}; + STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLast, false); + colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); const char *precStr = NULL; switch (pDb->cfg.precision) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 4955384a9b..f9ebde0657 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3663,7 +3663,7 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { } static const char* jkDatabaseOptionsBuffer = "Buffer"; -static const char* jkDatabaseOptionsCachelast = "Cachelast"; +static const char* jkDatabaseOptionsCacheModel = "CacheModel"; static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel"; static const char* jkDatabaseOptionsDaysPerFileNode = "DaysPerFileNode"; static const char* jkDatabaseOptionsDaysPerFile = "DaysPerFile"; @@ -3687,7 +3687,7 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cacheLast); + code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCacheModel, pNode->cacheModel); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel); @@ -3749,7 +3749,7 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cacheLast); + code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCacheModel, &pNode->cacheModel); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 59fc69f768..f3ea332fe2 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -38,8 +38,8 @@ typedef struct SAstCreateContext { typedef enum EDatabaseOptionType { DB_OPTION_BUFFER = 1, - DB_OPTION_CACHELAST, - DB_OPTION_CACHELASTSIZE, + DB_OPTION_CACHEMODEL, + DB_OPTION_CACHESIZE, DB_OPTION_COMP, DB_OPTION_DAYS, DB_OPTION_FSYNC, diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ddde20e8e9..9cab419c2e 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -172,8 +172,8 @@ exists_opt(A) ::= . db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); } -db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } -db_options(A) ::= db_options(B) CACHELASTSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELASTSIZE, &C); } +db_options(A) ::= db_options(B) CACHEMODEL NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHEMODEL, &C); } +db_options(A) ::= db_options(B) CACHESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHESIZE, &C); } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } @@ -186,7 +186,7 @@ db_options(A) ::= db_options(B) PAGES NK_INTEGER(C). db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGESIZE, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } -db_options(A) ::= db_options(B) STRICT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } +db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } @@ -199,8 +199,8 @@ alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). %type alter_db_option { SAlterOption } %destructor alter_db_option { } alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; } -alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } -alter_db_option(A) ::= CACHELASTSIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHELASTSIZE; A.val = B; } +alter_db_option(A) ::= CACHEMODEL NK_STRING(B). { A.type = DB_OPTION_CACHEMODEL; A.val = B; } +alter_db_option(A) ::= CACHESIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHESIZE; A.val = B; } alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a7806f5e34..0a04ff3e30 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -760,8 +760,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST; - pOptions->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + pOptions->cacheModel = TSDB_DEFAULT_CACHE_MODEL; + pOptions->cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL; pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; @@ -787,7 +787,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->buffer = -1; - pOptions->cacheLast = -1; + pOptions->cacheModel = -1; pOptions->cacheLastSize = -1; pOptions->compressionLevel = -1; pOptions->daysPerFile = -1; @@ -815,10 +815,10 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_BUFFER: ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; - case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cacheLast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + case DB_OPTION_CACHEMODEL: + COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->cacheModelStr, (SToken*)pVal); break; - case DB_OPTION_CACHELASTSIZE: + case DB_OPTION_CACHESIZE: ((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: @@ -858,7 +858,7 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->strictStr, (SToken*)pVal); break; case DB_OPTION_WAL: ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); @@ -872,10 +872,6 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; break; - // case DB_OPTION_SCHEMALESS: - // ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); - // ((SDatabaseOptions*)pOptions)->schemaless = 0; - // break; default: break; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 0d176cef09..b9d907c600 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -52,8 +52,8 @@ static SKeyword keywordTable[] = { {"BUFSIZE", TK_BUFSIZE}, {"BY", TK_BY}, {"CACHE", TK_CACHE}, - {"CACHELAST", TK_CACHELAST}, - {"CACHELASTSIZE", TK_CACHELASTSIZE}, + {"CACHEMODEL", TK_CACHEMODEL}, + {"CACHESIZE", TK_CACHESIZE}, {"CAST", TK_CAST}, {"CLIENT_VERSION", TK_CLIENT_VERSION}, {"CLUSTER", TK_CLUSTER}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3164623465..39f60a7a3e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2938,7 +2938,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->compression = pStmt->pOptions->compressionLevel; pReq->replications = pStmt->pOptions->replica; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLast = pStmt->pOptions->cacheModel; pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->schemaless = pStmt->pOptions->schemaless; pReq->ignoreExist = pStmt->ignoreExists; @@ -3019,13 +3019,31 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt return TSDB_CODE_SUCCESS; } +static int32_t checkDbCacheModelOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if ('\0' != pOptions->cacheModelStr[0]) { + if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_NONE_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_NONE; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_LAST_ROW_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_LAST_ROW; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_LAST_VALUE_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_LAST_VALUE; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_BOTH_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_BOTH; + } else { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "cacheModel", + pOptions->cacheModelStr); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { if ('\0' != pOptions->precisionStr[0]) { - if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) { + if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) { pOptions->precision = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MICRO_STR)) { + } else if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MICRO_STR)) { pOptions->precision = TSDB_TIME_PRECISION_MICRO; - } else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_NANO_STR)) { + } else if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_NANO_STR)) { pOptions->precision = TSDB_TIME_PRECISION_NANO; } else { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pOptions->precisionStr); @@ -3034,6 +3052,19 @@ static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* return TSDB_CODE_SUCCESS; } +static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if ('\0' != pOptions->strictStr[0]) { + if (0 == strcasecmp(pOptions->strictStr, TSDB_DB_STRICT_OFF_STR)) { + pOptions->strict = TSDB_DB_STRICT_OFF; + } else if (0 == strcasecmp(pOptions->strictStr, TSDB_DB_STRICT_ON_STR)) { + pOptions->strict = TSDB_DB_STRICT_ON; + } else { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "strict", pOptions->strictStr); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2); @@ -3100,11 +3131,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST, TSDB_MAX_DB_CACHE_LAST); + code = checkDbCacheModelOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_LAST_SIZE, - TSDB_MAX_DB_CACHE_LAST_SIZE); + code = checkRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); @@ -3140,7 +3170,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "strict", pOptions->strict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON); + code = checkDbStrictOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "walLevel", pOptions->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL); @@ -3225,7 +3255,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLast = pStmt->pOptions->cacheModel; pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->replications = pStmt->pOptions->replica; return; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2200b43db1..57a243257b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -384,32 +384,32 @@ static const YYACTIONTYPE yy_action[] = { /* 1630 */ 464, 1792, 468, 467, 1512, 469, 1511, 1496, 1587, 1824, /* 1640 */ 1154, 50, 196, 295, 1793, 586, 1795, 1796, 582, 1824, /* 1650 */ 577, 1153, 1586, 290, 1793, 586, 1795, 1796, 582, 1810, - /* 1660 */ 577, 1079, 632, 1078, 634, 1077, 1076, 584, 1073, 1521, - /* 1670 */ 1072, 319, 1761, 320, 583, 1071, 1070, 1516, 1514, 321, - /* 1680 */ 496, 1792, 1495, 498, 1494, 500, 1493, 502, 493, 1732, - /* 1690 */ 94, 551, 15, 1792, 1237, 1726, 140, 509, 1713, 1824, - /* 1700 */ 1711, 1712, 1710, 146, 1793, 586, 1795, 1796, 582, 1810, - /* 1710 */ 577, 1709, 1707, 56, 1699, 1247, 229, 581, 510, 227, - /* 1720 */ 214, 1810, 1761, 16, 583, 232, 339, 225, 322, 584, - /* 1730 */ 219, 78, 515, 41, 1761, 17, 583, 47, 79, 23, - /* 1740 */ 524, 1437, 84, 234, 13, 243, 236, 1419, 1951, 1824, + /* 1660 */ 577, 1079, 632, 1078, 634, 1077, 1076, 584, 1073, 1071, + /* 1670 */ 1072, 1521, 1761, 319, 583, 1070, 1516, 320, 1514, 321, + /* 1680 */ 496, 1792, 1495, 498, 1494, 500, 1493, 502, 493, 94, + /* 1690 */ 1732, 551, 509, 1792, 1237, 1726, 140, 1713, 1711, 1824, + /* 1700 */ 1712, 1710, 1709, 146, 1793, 586, 1795, 1796, 582, 1810, + /* 1710 */ 577, 1247, 1707, 56, 1699, 41, 227, 581, 510, 84, + /* 1720 */ 214, 1810, 1761, 16, 583, 232, 339, 15, 322, 584, + /* 1730 */ 219, 225, 515, 243, 1761, 1437, 583, 47, 78, 79, + /* 1740 */ 524, 23, 242, 229, 236, 234, 25, 1419, 1951, 1824, /* 1750 */ 1421, 238, 147, 294, 1793, 586, 1795, 1796, 582, 241, - /* 1760 */ 577, 1824, 1843, 242, 1782, 295, 1793, 586, 1795, 1796, - /* 1770 */ 582, 1792, 577, 24, 25, 252, 46, 1414, 83, 18, - /* 1780 */ 1781, 1792, 1394, 1393, 151, 1449, 1448, 333, 1453, 1454, - /* 1790 */ 1443, 1452, 334, 10, 1280, 1356, 1331, 45, 19, 1810, - /* 1800 */ 1827, 576, 1311, 1329, 341, 1328, 31, 584, 152, 1810, - /* 1810 */ 12, 20, 1761, 165, 583, 21, 589, 584, 585, 587, - /* 1820 */ 342, 1140, 1761, 1137, 583, 591, 594, 593, 1134, 596, - /* 1830 */ 597, 1792, 1128, 599, 600, 602, 1132, 1131, 1117, 1824, - /* 1840 */ 609, 1792, 1149, 295, 1793, 586, 1795, 1796, 582, 1824, - /* 1850 */ 577, 1792, 1126, 281, 1793, 586, 1795, 1796, 582, 1810, - /* 1860 */ 577, 603, 85, 86, 62, 263, 1145, 584, 1130, 1810, - /* 1870 */ 1129, 1042, 1761, 618, 583, 264, 1067, 584, 1086, 1810, - /* 1880 */ 621, 1065, 1761, 1064, 583, 1063, 1062, 584, 1061, 1060, - /* 1890 */ 1059, 1058, 1761, 1083, 583, 1081, 1055, 1054, 1053, 1824, - /* 1900 */ 1050, 1049, 1528, 282, 1793, 586, 1795, 1796, 582, 1824, - /* 1910 */ 577, 1048, 1047, 289, 1793, 586, 1795, 1796, 582, 1824, + /* 1760 */ 577, 1824, 1843, 1782, 17, 295, 1793, 586, 1795, 1796, + /* 1770 */ 582, 1792, 577, 24, 252, 1414, 83, 46, 1781, 1394, + /* 1780 */ 1449, 1792, 1393, 151, 18, 1448, 333, 1453, 1452, 10, + /* 1790 */ 1454, 45, 1443, 334, 1280, 1356, 1827, 1311, 19, 1810, + /* 1800 */ 589, 1331, 1329, 13, 341, 576, 31, 584, 1328, 1810, + /* 1810 */ 152, 12, 1761, 165, 583, 20, 21, 584, 585, 587, + /* 1820 */ 342, 1140, 1761, 591, 583, 1137, 593, 594, 596, 1134, + /* 1830 */ 1128, 1792, 597, 599, 600, 602, 1132, 1117, 1131, 1824, + /* 1840 */ 1149, 1792, 1126, 295, 1793, 586, 1795, 1796, 582, 1824, + /* 1850 */ 577, 1792, 263, 281, 1793, 586, 1795, 1796, 582, 1810, + /* 1860 */ 577, 603, 85, 609, 86, 62, 1130, 584, 1129, 1810, + /* 1870 */ 1145, 618, 1761, 1042, 583, 1086, 1067, 584, 621, 1810, + /* 1880 */ 264, 1065, 1761, 1062, 583, 1064, 1063, 584, 1061, 1060, + /* 1890 */ 1059, 1058, 1761, 1083, 583, 1081, 1048, 1055, 1054, 1824, + /* 1900 */ 1053, 1050, 1528, 282, 1793, 586, 1795, 1796, 582, 1824, + /* 1910 */ 577, 1049, 1047, 289, 1793, 586, 1795, 1796, 582, 1824, /* 1920 */ 577, 643, 644, 291, 1793, 586, 1795, 1796, 582, 645, /* 1930 */ 577, 1526, 1792, 647, 648, 649, 1524, 1522, 651, 652, /* 1940 */ 653, 655, 656, 657, 1510, 659, 1004, 1492, 267, 663, @@ -641,30 +641,30 @@ static const YYCODETYPE yy_lookahead[] = { /* 1630 */ 47, 259, 47, 35, 0, 39, 0, 0, 0, 327, /* 1640 */ 35, 94, 92, 331, 332, 333, 334, 335, 336, 327, /* 1650 */ 338, 22, 0, 331, 332, 333, 334, 335, 336, 287, - /* 1660 */ 338, 35, 43, 35, 43, 35, 35, 295, 35, 0, - /* 1670 */ 35, 22, 300, 22, 302, 35, 35, 0, 0, 22, - /* 1680 */ 35, 259, 0, 35, 0, 35, 0, 22, 49, 0, - /* 1690 */ 20, 369, 85, 259, 35, 0, 172, 22, 0, 327, + /* 1660 */ 338, 35, 43, 35, 43, 35, 35, 295, 35, 22, + /* 1670 */ 35, 0, 300, 22, 302, 35, 0, 22, 0, 22, + /* 1680 */ 35, 259, 0, 35, 0, 35, 0, 22, 49, 20, + /* 1690 */ 0, 369, 22, 259, 35, 0, 172, 0, 0, 327, /* 1700 */ 0, 0, 0, 331, 332, 333, 334, 335, 336, 287, - /* 1710 */ 338, 0, 0, 153, 0, 181, 149, 295, 153, 39, + /* 1710 */ 338, 181, 0, 153, 0, 43, 39, 295, 153, 95, /* 1720 */ 150, 287, 300, 230, 302, 46, 292, 85, 153, 295, - /* 1730 */ 86, 85, 155, 43, 300, 230, 302, 43, 85, 85, - /* 1740 */ 151, 86, 95, 85, 230, 46, 86, 86, 376, 327, + /* 1730 */ 86, 85, 155, 46, 300, 86, 302, 43, 85, 85, + /* 1740 */ 151, 85, 43, 149, 86, 85, 43, 86, 376, 327, /* 1750 */ 86, 85, 85, 331, 332, 333, 334, 335, 336, 85, - /* 1760 */ 338, 327, 340, 43, 46, 331, 332, 333, 334, 335, - /* 1770 */ 336, 259, 338, 85, 43, 46, 43, 86, 85, 43, - /* 1780 */ 46, 259, 86, 86, 46, 35, 35, 35, 35, 86, - /* 1790 */ 86, 35, 35, 2, 22, 193, 86, 224, 43, 287, - /* 1800 */ 85, 85, 22, 86, 292, 86, 85, 295, 46, 287, - /* 1810 */ 85, 85, 300, 46, 302, 85, 35, 295, 195, 96, - /* 1820 */ 35, 86, 300, 86, 302, 85, 85, 35, 86, 35, - /* 1830 */ 85, 259, 86, 35, 85, 35, 109, 109, 22, 327, - /* 1840 */ 97, 259, 35, 331, 332, 333, 334, 335, 336, 327, - /* 1850 */ 338, 259, 86, 331, 332, 333, 334, 335, 336, 287, - /* 1860 */ 338, 85, 85, 85, 85, 43, 22, 295, 109, 287, - /* 1870 */ 109, 62, 300, 61, 302, 43, 35, 295, 68, 287, - /* 1880 */ 83, 35, 300, 35, 302, 35, 35, 295, 35, 22, - /* 1890 */ 35, 35, 300, 68, 302, 35, 35, 35, 35, 327, + /* 1760 */ 338, 327, 340, 46, 230, 331, 332, 333, 334, 335, + /* 1770 */ 336, 259, 338, 85, 46, 86, 85, 43, 46, 86, + /* 1780 */ 35, 259, 86, 46, 43, 35, 35, 35, 35, 2, + /* 1790 */ 86, 224, 86, 35, 22, 193, 85, 22, 43, 287, + /* 1800 */ 35, 86, 86, 230, 292, 85, 85, 295, 86, 287, + /* 1810 */ 46, 85, 300, 46, 302, 85, 85, 295, 195, 96, + /* 1820 */ 35, 86, 300, 85, 302, 86, 35, 85, 35, 86, + /* 1830 */ 86, 259, 85, 35, 85, 35, 109, 22, 109, 327, + /* 1840 */ 35, 259, 86, 331, 332, 333, 334, 335, 336, 327, + /* 1850 */ 338, 259, 43, 331, 332, 333, 334, 335, 336, 287, + /* 1860 */ 338, 85, 85, 97, 85, 85, 109, 295, 109, 287, + /* 1870 */ 22, 61, 300, 62, 302, 68, 35, 295, 83, 287, + /* 1880 */ 43, 35, 300, 22, 302, 35, 35, 295, 35, 22, + /* 1890 */ 35, 35, 300, 68, 302, 35, 22, 35, 35, 327, /* 1900 */ 35, 35, 0, 331, 332, 333, 334, 335, 336, 327, /* 1910 */ 338, 35, 35, 331, 332, 333, 334, 335, 336, 327, /* 1920 */ 338, 35, 47, 331, 332, 333, 334, 335, 336, 39, @@ -783,23 +783,23 @@ static const unsigned short int yy_shift_ofst[] = { /* 450 */ 1603, 1606, 1608, 1553, 1611, 1613, 1581, 1571, 1580, 1620, /* 460 */ 1586, 1576, 1587, 1625, 1593, 1583, 1590, 1627, 1598, 1585, /* 470 */ 1596, 1634, 1636, 1637, 1638, 1547, 1550, 1605, 1629, 1652, - /* 480 */ 1626, 1628, 1630, 1631, 1619, 1621, 1633, 1635, 1640, 1641, - /* 490 */ 1669, 1649, 1677, 1651, 1639, 1678, 1657, 1645, 1682, 1648, - /* 500 */ 1684, 1650, 1686, 1665, 1670, 1689, 1560, 1659, 1695, 1524, - /* 510 */ 1675, 1565, 1570, 1698, 1700, 1575, 1577, 1701, 1702, 1711, - /* 520 */ 1607, 1644, 1534, 1712, 1642, 1589, 1646, 1714, 1680, 1567, - /* 530 */ 1653, 1647, 1679, 1690, 1493, 1654, 1655, 1658, 1660, 1661, - /* 540 */ 1666, 1694, 1664, 1667, 1674, 1688, 1691, 1720, 1699, 1718, - /* 550 */ 1693, 1731, 1505, 1696, 1697, 1729, 1573, 1733, 1734, 1738, - /* 560 */ 1703, 1736, 1514, 1704, 1750, 1751, 1752, 1753, 1756, 1757, - /* 570 */ 1704, 1791, 1772, 1602, 1755, 1715, 1710, 1716, 1717, 1721, - /* 580 */ 1719, 1762, 1725, 1726, 1767, 1780, 1623, 1730, 1723, 1735, - /* 590 */ 1781, 1785, 1740, 1737, 1792, 1741, 1742, 1794, 1745, 1746, - /* 600 */ 1798, 1749, 1766, 1800, 1776, 1727, 1728, 1759, 1761, 1816, - /* 610 */ 1743, 1777, 1778, 1807, 1779, 1822, 1822, 1844, 1809, 1812, - /* 620 */ 1841, 1810, 1797, 1832, 1846, 1848, 1850, 1851, 1853, 1867, - /* 630 */ 1855, 1856, 1825, 1619, 1860, 1621, 1861, 1862, 1863, 1865, - /* 640 */ 1866, 1876, 1877, 1902, 1886, 1875, 1890, 1931, 1898, 1887, + /* 480 */ 1626, 1628, 1630, 1631, 1619, 1621, 1633, 1635, 1647, 1640, + /* 490 */ 1671, 1651, 1676, 1655, 1639, 1678, 1657, 1645, 1682, 1648, + /* 500 */ 1684, 1650, 1686, 1665, 1669, 1690, 1560, 1659, 1695, 1524, + /* 510 */ 1670, 1565, 1570, 1697, 1698, 1575, 1577, 1700, 1701, 1702, + /* 520 */ 1642, 1644, 1530, 1712, 1646, 1589, 1653, 1714, 1677, 1594, + /* 530 */ 1654, 1624, 1679, 1672, 1493, 1656, 1649, 1660, 1658, 1661, + /* 540 */ 1666, 1694, 1664, 1667, 1674, 1688, 1689, 1699, 1687, 1717, + /* 550 */ 1691, 1703, 1534, 1693, 1696, 1728, 1567, 1734, 1732, 1737, + /* 560 */ 1704, 1741, 1573, 1706, 1745, 1750, 1751, 1752, 1753, 1758, + /* 570 */ 1706, 1787, 1772, 1602, 1755, 1711, 1715, 1720, 1716, 1721, + /* 580 */ 1722, 1764, 1726, 1730, 1767, 1775, 1623, 1731, 1723, 1735, + /* 590 */ 1765, 1785, 1738, 1739, 1791, 1742, 1743, 1793, 1747, 1744, + /* 600 */ 1798, 1749, 1756, 1800, 1776, 1727, 1729, 1757, 1759, 1815, + /* 610 */ 1766, 1777, 1779, 1805, 1780, 1809, 1809, 1848, 1811, 1810, + /* 620 */ 1841, 1807, 1795, 1837, 1846, 1850, 1851, 1861, 1853, 1867, + /* 630 */ 1855, 1856, 1825, 1619, 1860, 1621, 1862, 1863, 1865, 1866, + /* 640 */ 1876, 1874, 1877, 1902, 1886, 1875, 1890, 1931, 1898, 1887, /* 650 */ 1896, 1936, 1903, 1892, 1901, 1937, 1906, 1895, 1904, 1944, /* 660 */ 1910, 1911, 1947, 1926, 1928, 1930, 1932, 1929, 1933, }; @@ -987,8 +987,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ 0, /* BUFFER => nothing */ - 0, /* CACHELAST => nothing */ - 0, /* CACHELASTSIZE => nothing */ + 0, /* CACHEMODEL => nothing */ + 0, /* CACHESIZE => nothing */ 0, /* COMP => nothing */ 0, /* DURATION => nothing */ 0, /* NK_VARIABLE => nothing */ @@ -1330,8 +1330,8 @@ static const char *const yyTokenName[] = { /* 61 */ "NOT", /* 62 */ "EXISTS", /* 63 */ "BUFFER", - /* 64 */ "CACHELAST", - /* 65 */ "CACHELASTSIZE", + /* 64 */ "CACHEMODEL", + /* 65 */ "CACHESIZE", /* 66 */ "COMP", /* 67 */ "DURATION", /* 68 */ "NK_VARIABLE", @@ -1726,8 +1726,8 @@ static const char *const yyRuleName[] = { /* 71 */ "exists_opt ::=", /* 72 */ "db_options ::=", /* 73 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 74 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 75 */ "db_options ::= db_options CACHELASTSIZE NK_INTEGER", + /* 74 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 75 */ "db_options ::= db_options CACHESIZE NK_INTEGER", /* 76 */ "db_options ::= db_options COMP NK_INTEGER", /* 77 */ "db_options ::= db_options DURATION NK_INTEGER", /* 78 */ "db_options ::= db_options DURATION NK_VARIABLE", @@ -1740,7 +1740,7 @@ static const char *const yyRuleName[] = { /* 85 */ "db_options ::= db_options PAGESIZE NK_INTEGER", /* 86 */ "db_options ::= db_options PRECISION NK_STRING", /* 87 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 88 */ "db_options ::= db_options STRICT NK_INTEGER", + /* 88 */ "db_options ::= db_options STRICT NK_STRING", /* 89 */ "db_options ::= db_options WAL NK_INTEGER", /* 90 */ "db_options ::= db_options VGROUPS NK_INTEGER", /* 91 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", @@ -1749,8 +1749,8 @@ static const char *const yyRuleName[] = { /* 94 */ "alter_db_options ::= alter_db_option", /* 95 */ "alter_db_options ::= alter_db_options alter_db_option", /* 96 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 97 */ "alter_db_option ::= CACHELAST NK_INTEGER", - /* 98 */ "alter_db_option ::= CACHELASTSIZE NK_INTEGER", + /* 97 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 98 */ "alter_db_option ::= CACHESIZE NK_INTEGER", /* 99 */ "alter_db_option ::= FSYNC NK_INTEGER", /* 100 */ "alter_db_option ::= KEEP integer_list", /* 101 */ "alter_db_option ::= KEEP variable_list", @@ -2816,8 +2816,8 @@ static const struct { { 271, 0 }, /* (71) exists_opt ::= */ { 270, 0 }, /* (72) db_options ::= */ { 270, -3 }, /* (73) db_options ::= db_options BUFFER NK_INTEGER */ - { 270, -3 }, /* (74) db_options ::= db_options CACHELAST NK_INTEGER */ - { 270, -3 }, /* (75) db_options ::= db_options CACHELASTSIZE NK_INTEGER */ + { 270, -3 }, /* (74) db_options ::= db_options CACHEMODEL NK_STRING */ + { 270, -3 }, /* (75) db_options ::= db_options CACHESIZE NK_INTEGER */ { 270, -3 }, /* (76) db_options ::= db_options COMP NK_INTEGER */ { 270, -3 }, /* (77) db_options ::= db_options DURATION NK_INTEGER */ { 270, -3 }, /* (78) db_options ::= db_options DURATION NK_VARIABLE */ @@ -2830,7 +2830,7 @@ static const struct { { 270, -3 }, /* (85) db_options ::= db_options PAGESIZE NK_INTEGER */ { 270, -3 }, /* (86) db_options ::= db_options PRECISION NK_STRING */ { 270, -3 }, /* (87) db_options ::= db_options REPLICA NK_INTEGER */ - { 270, -3 }, /* (88) db_options ::= db_options STRICT NK_INTEGER */ + { 270, -3 }, /* (88) db_options ::= db_options STRICT NK_STRING */ { 270, -3 }, /* (89) db_options ::= db_options WAL NK_INTEGER */ { 270, -3 }, /* (90) db_options ::= db_options VGROUPS NK_INTEGER */ { 270, -3 }, /* (91) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ @@ -2839,8 +2839,8 @@ static const struct { { 272, -1 }, /* (94) alter_db_options ::= alter_db_option */ { 272, -2 }, /* (95) alter_db_options ::= alter_db_options alter_db_option */ { 276, -2 }, /* (96) alter_db_option ::= BUFFER NK_INTEGER */ - { 276, -2 }, /* (97) alter_db_option ::= CACHELAST NK_INTEGER */ - { 276, -2 }, /* (98) alter_db_option ::= CACHELASTSIZE NK_INTEGER */ + { 276, -2 }, /* (97) alter_db_option ::= CACHEMODEL NK_STRING */ + { 276, -2 }, /* (98) alter_db_option ::= CACHESIZE NK_INTEGER */ { 276, -2 }, /* (99) alter_db_option ::= FSYNC NK_INTEGER */ { 276, -2 }, /* (100) alter_db_option ::= KEEP integer_list */ { 276, -2 }, /* (101) alter_db_option ::= KEEP variable_list */ @@ -3543,12 +3543,12 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 74: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + case 74: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 75: /* db_options ::= db_options CACHELASTSIZE NK_INTEGER */ -{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHELASTSIZE, &yymsp[0].minor.yy0); } + case 75: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; case 76: /* db_options ::= db_options COMP NK_INTEGER */ @@ -3593,7 +3593,7 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 88: /* db_options ::= db_options STRICT NK_INTEGER */ + case 88: /* db_options ::= db_options STRICT NK_STRING */ { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; @@ -3628,11 +3628,11 @@ static YYACTIONTYPE yy_reduce( case 96: /* alter_db_option ::= BUFFER NK_INTEGER */ { yymsp[-1].minor.yy409.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; - case 97: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } + case 97: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; - case 98: /* alter_db_option ::= CACHELASTSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHELASTSIZE; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } + case 98: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; case 99: /* alter_db_option ::= FSYNC NK_INTEGER */ { yymsp[-1].minor.yy409.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index f4d0ba1cc8..079a9540c3 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -38,7 +38,7 @@ TEST_F(ParserInitialATest, alterDnode) { TEST_F(ParserInitialATest, alterDatabase) { useDb("root", "test"); - run("ALTER DATABASE test CACHELAST 1 FSYNC 200 WAL 1"); + run("ALTER DATABASE test CACHEMODEL 'last_row' FSYNC 200 WAL 1"); run("ALTER DATABASE test KEEP 2400"); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index e9c8fb5326..7ea4fab470 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -43,7 +43,8 @@ TEST_F(ParserInitialCTest, createBnode) { * * database_option: { * BUFFER value - * | CACHELAST value + * | CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} + * | CACHESIZE value * | COMP {0 | 1 | 2} * | DURATION value * | FSYNC value @@ -55,7 +56,7 @@ TEST_F(ParserInitialCTest, createBnode) { * | PRECISION {'ms' | 'us' | 'ns'} * | REPLICA value * | RETENTIONS ingestion_duration:keep_duration ... - * | STRICT value + * | STRICT {'off' | 'on'} * | WAL value * | VGROUPS value * | SINGLE_STABLE {0 | 1} @@ -76,8 +77,8 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.db[len] = '\0'; expect.ignoreExist = igExists; expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - expect.cacheLast = TSDB_DEFAULT_CACHE_LAST; - expect.cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + expect.cacheLast = TSDB_DEFAULT_CACHE_MODEL; + expect.cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; expect.compression = TSDB_DEFAULT_COMP_LEVEL; expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; @@ -203,8 +204,8 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbSchemalessFunc(1); run("CREATE DATABASE IF NOT EXISTS wxy_db " "BUFFER 64 " - "CACHELAST 2 " - "CACHELASTSIZE 20 " + "CACHEMODEL 'last_value' " + "CACHESIZE 20 " "COMP 1 " "DURATION 100 " "FSYNC 100 " @@ -216,7 +217,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "PRECISION 'ns' " "REPLICA 3 " "RETENTIONS 15s:7d,1m:21d,15m:500d " - "STRICT 1 " + "STRICT 'on' " "WAL 2 " "VGROUPS 100 " "SINGLE_STABLE 1 " diff --git a/tests/pytest/alter/alter_cacheLastRow.py b/tests/pytest/alter/alter_cacheLastRow.py index 36a2864d0f..3152e46af1 100644 --- a/tests/pytest/alter/alter_cacheLastRow.py +++ b/tests/pytest/alter/alter_cacheLastRow.py @@ -67,7 +67,7 @@ class TDTestCase: slow = 0 #count time where lastRow on is slower for i in range(5): #switch lastRow to off and check - tdSql.execute('alter database db cachelast 0') + tdSql.execute('alter database db cachemodel 'none'') tdSql.query('show databases') tdSql.checkData(0,15,0) @@ -79,7 +79,7 @@ class TDTestCase: tdLog.debug(f'time used:{lastRow_Off_end-lastRow_Off_start}') #switch lastRow to on and check - tdSql.execute('alter database db cachelast 1') + tdSql.execute('alter database db cachemodel 'last_row'') tdSql.query('show databases') tdSql.checkData(0,15,1) diff --git a/tests/pytest/query/last_cache.py b/tests/pytest/query/last_cache.py index c31d9821e2..9ee5be8c72 100644 --- a/tests/pytest/query/last_cache.py +++ b/tests/pytest/query/last_cache.py @@ -89,36 +89,36 @@ class TDTestCase: tdSql.prepare() # last_cache_0.sim - tdSql.execute("create database test1 cachelast 0") + tdSql.execute("create database test1 cachemodel 'none'") tdSql.execute("use test1") self.insertData() self.executeQueries() - tdSql.execute("alter database test1 cachelast 1") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() - tdSql.execute("alter database test1 cachelast 0") + tdSql.execute("alter database test1 cachemodel 'none'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() # last_cache_1.sim - tdSql.execute("create database test2 cachelast 1") + tdSql.execute("create database test2 cachemodel 'last_row'") tdSql.execute("use test2") self.insertData() self.executeQueries() - tdSql.execute("alter database test2 cachelast 0") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() - tdSql.execute("alter database test2 cachelast 1") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index 0e11e3d60c..de9f08b277 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -142,56 +142,56 @@ class TDTestCase: tdSql.prepare() print("============== Step1: last_row_cache_0.sim") - tdSql.execute("create database test1 cachelast 0") + tdSql.execute("create database test1 cachemodel 'none'") tdSql.execute("use test1") self.insertData() self.executeQueries() self.insertData2() self.executeQueries2() - print("============== Step2: alter database test1 cachelast 1") - tdSql.execute("alter database test1 cachelast 1") + print("============== Step2: alter database test1 cachemodel 'last_row'") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries2() - print("============== Step3: alter database test1 cachelast 2") - tdSql.execute("alter database test1 cachelast 2") + print("============== Step3: alter database test1 cachemodel 'last_value'") + tdSql.execute("alter database test1 cachemodel 'last_value'") self.executeQueries2() - print("============== Step4: alter database test1 cachelast 3") - tdSql.execute("alter database test1 cachelast 3") + print("============== Step4: alter database test1 cachemodel 'both'") + tdSql.execute("alter database test1 cachemodel 'both'") self.executeQueries2() - print("============== Step5: alter database test1 cachelast 0 and restart taosd") - tdSql.execute("alter database test1 cachelast 0") + print("============== Step5: alter database test1 cachemodel 'none' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'none'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step6: alter database test1 cachelast 1 and restart taosd") - tdSql.execute("alter database test1 cachelast 1") + print("============== Step6: alter database test1 cachemodel 'last_row' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step7: alter database test1 cachelast 2 and restart taosd") - tdSql.execute("alter database test1 cachelast 2") + print("============== Step7: alter database test1 cachemodel 'last_value' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'last_value'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step8: alter database test1 cachelast 3 and restart taosd") - tdSql.execute("alter database test1 cachelast 3") + print("============== Step8: alter database test1 cachemodel 'both' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'both'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step9: create database test2 cachelast 1") - tdSql.execute("create database test2 cachelast 1") + print("============== Step9: create database test2 cachemodel 'last_row'") + tdSql.execute("create database test2 cachemodel 'last_row'") tdSql.execute("use test2") self.insertData() self.executeQueries() @@ -201,45 +201,45 @@ class TDTestCase: tdDnodes.start(1) self.executeQueries2() - print("============== Step8: alter database test2 cachelast 0") - tdSql.execute("alter database test2 cachelast 0") + print("============== Step8: alter database test2 cachemodel 'none'") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries2() - print("============== Step9: alter database test2 cachelast 1") - tdSql.execute("alter database test2 cachelast 1") + print("============== Step9: alter database test2 cachemodel 'last_row'") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries2() - print("============== Step10: alter database test2 cachelast 2") - tdSql.execute("alter database test2 cachelast 2") + print("============== Step10: alter database test2 cachemodel 'last_value'") + tdSql.execute("alter database test2 cachemodel 'last_value'") self.executeQueries2() - print("============== Step11: alter database test2 cachelast 3") - tdSql.execute("alter database test2 cachelast 3") + print("============== Step11: alter database test2 cachemodel 'both'") + tdSql.execute("alter database test2 cachemodel 'both'") self.executeQueries2() - print("============== Step12: alter database test2 cachelast 0 and restart taosd") - tdSql.execute("alter database test2 cachelast 0") + print("============== Step12: alter database test2 cachemodel 'none' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step13: alter database test2 cachelast 1 and restart taosd") - tdSql.execute("alter database test2 cachelast 1") + print("============== Step13: alter database test2 cachemodel 'last_row' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step14: alter database test2 cachelast 2 and restart taosd") - tdSql.execute("alter database test2 cachelast 2") + print("============== Step14: alter database test2 cachemodel 'last_value' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'last_value'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step15: alter database test2 cachelast 3 and restart taosd") - tdSql.execute("alter database test2 cachelast 3") + print("============== Step15: alter database test2 cachemodel 'both' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'both'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) diff --git a/tests/pytest/util/boundary.py b/tests/pytest/util/boundary.py index 4cef926f2e..086821e7cf 100644 --- a/tests/pytest/util/boundary.py +++ b/tests/pytest/util/boundary.py @@ -39,6 +39,6 @@ class DataBoundary: self.DB_PARAM_PRECISION_CONFIG = {"create_name": "precision", "query_name": "precision", "vnode_json_key": "", "boundary": ['ms', 'us', 'ns'], "default": "ms"} self.DB_PARAM_REPLICA_CONFIG = {"create_name": "replica", "query_name": "replica", "vnode_json_key": "", "boundary": [1], "default": 1} self.DB_PARAM_SINGLE_STABLE_CONFIG = {"create_name": "single_stable", "query_name": "single_stable_model", "vnode_json_key": "", "boundary": [0, 1], "default": 0} - self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"no_strict": 0, "strict": 1}, "default": "no_strict"} + self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"off": 0, "strict": 1}, "default": "off"} self.DB_PARAM_VGROUPS_CONFIG = {"create_name": "vgroups", "query_name": "vgroups", "vnode_json_key": "", "boundary": [1, 32], "default": 2} self.DB_PARAM_WAL_CONFIG = {"create_name": "wal", "query_name": "wal", "vnode_json_key": "", "boundary": [1, 2], "default": 1} \ No newline at end of file diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 1eb4e36da6..cda1656888 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -69,7 +69,7 @@ endi if $data4_db != 3 then # replica return -1 endi -if $data5_db != no_strict then # strict +if $data5_db != off then # strict return -1 endi if $data6_db != 345600m then # duration @@ -333,31 +333,31 @@ sql_error alter database db comp 5 sql_error alter database db comp -1 print ============== modify cachelast [0, 1, 2, 3] -sql alter database db cachelast 2 +sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db if $data16_db != 2 then return -1 endi -sql alter database db cachelast 1 +sql alter database db cachemodel 'last_row' sql show databases print cachelast $data16_db if $data16_db != 1 then return -1 endi -sql alter database db cachelast 0 +sql alter database db cachemodel 'none' sql show databases print cachelast $data16_db if $data16_db != 0 then return -1 endi -sql alter database db cachelast 2 +sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db if $data16_db != 2 then return -1 endi -sql alter database db cachelast 3 +sql alter database db cachemodel 'both' sql show databases print cachelast $data16_db if $data16_db != 3 then diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 7525fe2087..1daccb03bc 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -15,7 +15,7 @@ $tb = $tbPrefix . $i print =============== step1 # quorum presicion -sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' +sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachemodel 'last_value' precision 'us' sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index bdfc556e18..ee65229be1 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -89,7 +89,7 @@ if $data4_db != 1 then # replica print expect 1, actual: $data4_db return -1 endi -if $data5_db != no_strict then # strict +if $data5_db != off then # strict return -1 endi if $data6_db != 14400m then # duration diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 5f45b64796..7ffb3749aa 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -7,7 +7,7 @@ print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachelast 2 +sql create database $db cachemodel 'last_value' sql use $db sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index 3d1bff6abf..e7c191ed92 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -8,7 +8,7 @@ print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachelast 2 +sql create database $db cachemodel 'last_value' sql use $db $table1 = table_name diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 449f5c7806..fce3cc7043 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -25,7 +25,7 @@ class TDTestCase: def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(" create database test cachelast 1 ") + tdSql.execute(" create database test cachemodel 'last_row' ") tdSql.execute(" use test ") tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") @@ -63,7 +63,7 @@ class TDTestCase: def prepare_datas(self): - tdSql.execute("create database if not exists db keep 3650 duration 1000 cachelast 1") + tdSql.execute("create database if not exists db keep 3650 duration 1000 cachemodel 'last_row'") tdSql.execute("use db") tdSql.execute( '''create table stb1 @@ -124,7 +124,7 @@ class TDTestCase: def prepare_tag_datas(self): # prepare datas tdSql.execute( - "create database if not exists testdb keep 3650 duration 1000 cachelast 1") + "create database if not exists testdb keep 3650 duration 1000 cachemodel 'last_row'") tdSql.execute(" use testdb ") tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ @@ -528,7 +528,7 @@ class TDTestCase: def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test cachelast 2") + tdSql.execute("create database if not exists bound_test cachemodel 'last_value'") time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 4611726c14..12b14b0661 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -122,9 +122,9 @@ class TDTestCase: tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') tdSql.query('show databases;') - tdSql.checkData(2,5,'no_strict') - tdSql.error('alter database db strict 0') - # tdSql.execute('alter database db strict 1') + tdSql.checkData(2,5,'off') + tdSql.error('alter database db strict 'off'') + # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') # tdSql.checkData(2,5,'strict') From 8caa4bcad0483ee70275ce8178f6e167da5fcead Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 13:34:24 +0800 Subject: [PATCH 90/94] feat(query): add twa function scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 582fe5a09b..a3ec033773 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -109,6 +109,7 @@ int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScala int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b9e2b6feaf..fd50b12515 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2320,6 +2320,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTwaFuncEnv, .initFunc = twaFunctionSetup, .processFunc = twaFunction, + .sprocessFunc = twaScalarFunction, .finalizeFunc = twaFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 13b29f8ae3..c7bb8b3d80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2413,3 +2413,7 @@ int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return nonCalcScalarFunction(pInput, inputNum, pOutput); } + +int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return avgScalarFunction(pInput, inputNum, pOutput); +} From 59839f0d0aeb3ba31364ebfebb2ca206c154cea3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 13:41:42 +0800 Subject: [PATCH 91/94] feat(query): add mavg scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index a3ec033773..5cbe380234 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -110,6 +110,7 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fd50b12515..31776610b6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2447,6 +2447,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, .processFunc = mavgFunction, + .sprocessFunc = mavgScalarFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c7bb8b3d80..9687528136 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2417,3 +2417,7 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return avgScalarFunction(pInput, inputNum, pOutput); } + +int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return avgScalarFunction(pInput, inputNum, pOutput); +} From 00d7dba673f10ca6d8c373eec9831df694a05e65 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 13:37:46 +0800 Subject: [PATCH 92/94] test: valgrind case --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/parser/auto_create_tb.sim | 34 +++--- tests/script/tsim/valgrind/basic1.sim | 113 +++++++------------- tests/script/tsim/valgrind/basic2.sim | 79 +++++++------- 4 files changed, 100 insertions(+), 128 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6c1938d9fc..01896ec161 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -87,7 +87,7 @@ ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim -# jira ./test.sh -f tsim/parser/auto_create_tb.sim +# nojira ./test.sh -f tsim/parser/auto_create_tb.sim ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim diff --git a/tests/script/tsim/parser/auto_create_tb.sim b/tests/script/tsim/parser/auto_create_tb.sim index 673a472cf7..485f4f480c 100644 --- a/tests/script/tsim/parser/auto_create_tb.sim +++ b/tests/script/tsim/parser/auto_create_tb.sim @@ -184,23 +184,26 @@ if $data(3)[8] != 涛思数据3 then return -1 endi -sql select count(*), first(c9) from $stb group by t1 order by t1 asc slimit 2 soffset 1 -if $rows != 2 then +sql select t1, count(*), first(c9) from $stb partition by t1 order by t1 asc slimit 3 +if $rows != 3 then return -1 endi -if $data00 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data01 != 涛思数据2 then +if $data(1)[2] != 涛思数据1 then return -1 endi -if $data02 != 2 then +if $data(2)[1] != 1 then return -1 endi -if $data11 != 涛思数据3 then +if $data(2)[2] != 涛思数据2 then return -1 endi -if $data12 != 3 then +if $data(3)[1] != 1 then + return -1 +endi +if $data(3)[2] != 涛思数据3 then return -1 endi @@ -248,23 +251,26 @@ if $data(3)[8] != 涛思数据3 then return -1 endi -sql select count(*), first(c9) from $stb group by t1 order by t1 asc slimit 2 soffset 1 -if $rows != 2 then +sql select t1, count(*), first(c9) from $stb partition by t1 order by t1 asc slimit 3 +if $rows != 3 then return -1 endi -if $data00 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data01 != 涛思数据2 then +if $data(1)[2] != 涛思数据1 then return -1 endi -if $data02 != 2 then +if $data(2)[1] != 1 then return -1 endi -if $data11 != 涛思数据3 then +if $data(2)[2] != 涛思数据2 then return -1 endi -if $data12 != 3 then +if $data(3)[1] != 1 then + return -1 +endi +if $data(3)[2] != 涛思数据3 then return -1 endi diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 86926509bb..3e39f35fa7 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,87 +1,50 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode1 -s start -v sql connect -print =============== step1: create drop show dnodes -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ---> dnode not ready! - return -1 - endi -sql show dnodes -print ---> $data00 $data01 $data02 $data03 $data04 $data05 -if $rows != 1 then +print ======== step1 +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql use db1; +sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); +sql create table tb1 using st1 tags(1); +sql insert into tb1 values ('2022-07-07 10:01:01', 11, "aaa"); +sql insert into tb1 values ('2022-07-07 11:01:02', 12, "bbb"); +sql create table tb2 using st1 tags(2); +sql insert into tb2 values ('2022-07-07 10:02:01', 21, "aaa"); +sql insert into tb2 values ('2022-07-07 11:02:02', 22, "bbb"); +sql create table tb3 using st1 tags(3); +sql insert into tb3 values ('2022-07-07 10:03:01', 31, "aaa"); +sql insert into tb3 values ('2022-07-07 11:03:02', 32, "bbb"); +sql create table tb4 using st1 tags(4); + +sql insert into tb4 select * from tb1; + +goto _OVER + +sql select * from tb4; +if $rows != 2 then return -1 endi -if $data(1)[4] != ready then - goto step1 -endi - -print =============== step2: create db -sql create database d1 vgroups 3 buffer 3 -sql show databases -sql use d1 -sql show vgroups - -print =============== step3: create show stable, include all type -sql create table if not exists stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(16), c9 nchar(16), c10 timestamp, c11 tinyint unsigned, c12 smallint unsigned, c13 int unsigned, c14 bigint unsigned) tags (t1 bool, t2 tinyint, t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 binary(16), t9 nchar(16), t10 timestamp, t11 tinyint unsigned, t12 smallint unsigned, t13 int unsigned, t14 bigint unsigned) -sql create stable if not exists stb_1 (ts timestamp, c1 int) tags (j int) -sql create table stb_2 (ts timestamp, c1 int) tags (t1 int) -sql create stable stb_3 (ts timestamp, c1 int) tags (t1 int) -sql show stables -if $rows != 4 then +sql insert into tb4 select ts,f1,f2 from st1; +sql select * from tb4; +if $rows != 6 then return -1 endi - -print =============== step4: ccreate child table -sql create table c1 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql create table c2 using stb tags(false, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 2', 'child tbl 2', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql show tables -if $rows != 2 then +sql create table tba (ts timestamp, f1 binary(10), f2 bigint, f3 double); +sql_error insert into tba select * from tb1; +sql insert into tba (ts,f2,f1) select * from tb1; +sql select * from tba; +if $rows != 2 then + return -1 +endi +sql create table tbb (ts timestamp, f1 binary(10), f2 bigint, f3 double); +sql insert into tbb (f2,f1,ts) select f1+1,f2,ts+3 from tb2; +sql select * from tbb; +if $rows != 2 then return -1 endi - -print =============== step5: insert data -sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -print =============== step6: alter insert -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -print =============== restart -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start -v - -print =============== stepa: query data -sql select * from c1 -sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 -sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 - -print =============== stepb: count -sql select count(*) from c1; -sql select count(*) from stb; -sql select count(ts), count(c1), count(c2), count(c3) from c1 -sql select count(ts), count(c1), count(c2), count(c3) from stb - -print =============== stepc: func -sql select first(ts), first(c1), first(c2), first(c3) from c1 -sql select min(c2), min(c3), min(c4) from c1 -sql select max(c2), max(c3), max(c4) from c1 -sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -96,4 +59,4 @@ endi if $system_content == $null then return -1 -endi +endi \ No newline at end of file diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index 15e092da35..d3c72d1e5c 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -23,62 +23,65 @@ if $data(1)[4] != ready then endi print =============== step2: create db -sql create database d1 vgroups 3 buffer 3 +sql create database d1 vgroups 2 buffer 3 sql show databases sql use d1 sql show vgroups -print =============== step3: create show stable, include all type -sql create table if not exists stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(16), c9 nchar(16), c10 timestamp, c11 tinyint unsigned, c12 smallint unsigned, c13 int unsigned, c14 bigint unsigned) tags (t1 bool, t2 tinyint, t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 binary(16), t9 nchar(16), t10 timestamp, t11 tinyint unsigned, t12 smallint unsigned, t13 int unsigned, t14 bigint unsigned) -sql create stable if not exists stb_1 (ts timestamp, c1 int) tags (j int) -sql create table stb_2 (ts timestamp, c1 int) tags (t1 int) -sql create stable stb_3 (ts timestamp, c1 int) tags (t1 int) +print =============== step3: create show stable +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) sql show stables -if $rows != 4 then +if $rows != 1 then return -1 endi -print =============== step4: ccreate child table -sql create table c1 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql create table c2 using stb tags(false, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 2', 'child tbl 2', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +print =============== step4: create show table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then +if $rows != 3 then return -1 endi print =============== step5: insert data -sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) -print =============== step6: alter insert -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -goto _OVER -print =============== stepa: query data -sql select * from c1 +print =============== step6: query data +sql select * from ct1 sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 +sql select c1, c2, c3 from ct1 sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 -print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +print =============== step7: count +sql select count(*) from ct1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +sql select count(ts), count(c1), count(c2), count(c3) from stb -print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c1), min(c2), min(c3) from c1 -#sql select max(c1), max(c2), max(c3) from c1 -#sql select sum(c1), sum(c2), sum(c3) from c1 +print =============== step8: func +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +sql select min(c1), min(c2), min(c3) from ct1 +sql select max(c1), max(c2), max(c3) from ct1 +sql select sum(c1), sum(c2), sum(c3) from ct1 + +print =============== step9: insert select +sql create table ct4 using stb tags(4000); +sql insert into ct4 select * from ct1; +sql select * from ct4; +sql insert into ct4 select ts,c1,c2,c3 from stb; + +sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +sql select * from tb1; + +sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +sql select * from tb2; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 299e117d382379a1e49abe03ac05f907c27e22ff Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 14:22:30 +0800 Subject: [PATCH 93/94] fix: database options 'strict' and 'cachelast' syntax adjustments --- source/libs/command/src/command.c | 16 ++++++------ tests/pytest/util/constant.py | 2 +- tests/script/tsim/db/alter_option.sim | 18 ++++++------- tests/script/tsim/db/create_all_options.sim | 22 ++++++++-------- tests/system-test/0-others/cachelast.py | 28 +++++++++++++-------- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- 6 files changed, 47 insertions(+), 41 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d22b3d88b4..3cb2553f2e 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -15,10 +15,10 @@ #include "command.h" #include "catalog.h" -#include "tdatablock.h" -#include "tglobal.h" #include "commandInt.h" #include "scheduler.h" +#include "tdatablock.h" +#include "tglobal.h" extern SConfig* tsCfg; @@ -222,7 +222,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S char* retentions = buildRetension(pCfg->pRetensions); len += sprintf(buf2 + VARSTR_HEADER_SIZE, - "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " + "CREATE DATABASE `%s` BUFFER %d CACHEMODEL %d COMP %d DURATION %dm " "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, @@ -483,7 +483,7 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { int32_t code = 0; - + if (0 == strcasecmp(cmd, COMMAND_RESET_LOG)) { taosResetLog(); cfgDumpCfg(tsCfg, 0, false); @@ -502,13 +502,13 @@ _return: if (code) { terrno = code; } - - return code; + + return code; } static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { bool processed = false; - + if (execAlterCmd(pStmt->config, pStmt->value, &processed)) { return terrno; } @@ -516,7 +516,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { if (processed) { goto _return; } - + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { return terrno; } diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index 509d87e489..c4541386b4 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -54,7 +54,7 @@ TAOS_KEYWORDS = [ "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", "CACHE", "EXPLAIN", "LP", "ROW", "UNION", - "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CACHEMODEL", "FAIL", "LSHIFT", "RP", "UNSIGNED", "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index cda1656888..1648eb56fa 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -40,13 +40,13 @@ print ============= create database #database_option: { # | BUFFER value [3~16384, default: 96] # | PAGES value [64~16384, default: 256] -# | CACHELAST value [0, 1, 2, 3] +# | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | FSYNC value [0 ~ 180000 ms] # | KEEP value [duration, 365000] # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DURATION 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -102,7 +102,7 @@ endi if $data15_db != 0 then # comp return -1 endi -if $data16_db != 3 then # cachelast +if $data16_db != both then # cachelast return -1 endi if $data17_db != ns then # precision @@ -336,37 +336,37 @@ print ============== modify cachelast [0, 1, 2, 3] sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql alter database db cachemodel 'last_row' sql show databases print cachelast $data16_db -if $data16_db != 1 then +if $data16_db != last_row then return -1 endi sql alter database db cachemodel 'none' sql show databases print cachelast $data16_db -if $data16_db != 0 then +if $data16_db != none then return -1 endi sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql alter database db cachemodel 'both' sql show databases print cachelast $data16_db -if $data16_db != 3 then +if $data16_db != both then return -1 endi sql_error alter database db cachelast 4 sql_error alter database db cachelast 10 -sql_error alter database db cachelast -1 +sql_error alter database db cachelast 'other' print ============== modify precision sql_error alter database db precision 'ms' diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index ee65229be1..9de44ca335 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -40,7 +40,7 @@ print ============= create database with all options # | BUFFER value [3~16384, default: 96] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] -# | CACHELAST value [0, 1, 2, 3, default: 0] +# | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] # | COMP [0 | 1 | 2, default: 2] # | DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] # | FSYNC value [0 ~ 180000 ms, default: 3000] @@ -122,7 +122,7 @@ endi if $data15_db != 2 then # comp return -1 endi -if $data16_db != 0 then # cachelast +if $data16_db != none then # cachelast return -1 endi if $data17_db != ms then # precision @@ -167,32 +167,32 @@ sql drop database db #endi #sql drop database db -print ====> CACHELAST value [0, 1, 2, 3, default: 0] -sql create database db CACHELAST 1 +print ====> CACHEMODEL value [0, 1, 2, 3, default: 0] +sql create database db CACHEMODEL 'last_row' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 1 then +if $data16_db != last_row then return -1 endi sql drop database db -sql create database db CACHELAST 2 +sql create database db CACHEMODEL 'last_value' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql drop database db -sql create database db CACHELAST 3 +sql create database db CACHEMODEL 'both' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 3 then +if $data16_db != both then return -1 endi sql drop database db -sql_error create database db CACHELAST 4 -sql_error create database db CACHELAST -1 +sql_error create database db CACHEMODEL 'other' +sql_error create database db CACHEMODEL '-1' print ====> COMP [0 | 1 | 2, default: 2] sql create database db COMP 1 diff --git a/tests/system-test/0-others/cachelast.py b/tests/system-test/0-others/cachelast.py index b7b4148179..2df6b8c9cc 100644 --- a/tests/system-test/0-others/cachelast.py +++ b/tests/system-test/0-others/cachelast.py @@ -36,22 +36,31 @@ class TDTestCase: def illegal_params(self): - illegal_params = ["1","0","NULL","None","False","True" ,"keep","now" ,"*" , "," ,"_" , "abc" ,"keep"] + illegal_params = ["1","0","NULL","False","True" ,"keep","now" ,"*" , "," ,"_" , "abc" ,"keep"] for value in illegal_params: - tdSql.error("create database testdb replica 1 cachelast '%s' " %value) + tdSql.error("create database testdb replica 1 cachemodel '%s' " %value) unexpected_numbers = [-1 , 0.0 , 3.0 , 4, 10 , 100] for number in unexpected_numbers: - tdSql.error("create database testdb replica 1 cachelast %s " %number) + tdSql.error("create database testdb replica 1 cachemodel %s " %number) + def getCacheModelStr(self, value): + numbers = { + 0 : "none", + 1 : "last_row", + 2 : "last_value", + 3 : "both" + } + return numbers.get(value, 'other') def prepare_datas(self): for i in range(4): - tdSql.execute("create database test_db_%d replica 1 cachelast %d " %(i,i)) - tdSql.execute("use test_db_%d"%i) + str = self.getCacheModelStr(i) + tdSql.execute("create database testdb_%s replica 1 cachemodel '%s' " %(str, str)) + tdSql.execute("use testdb_%s"%str) tdSql.execute("create stable st(ts timestamp , c1 int ,c2 float ) tags(ind int) ") tdSql.execute("create table tb1 using st tags(1) ") tdSql.execute("create table tb2 using st tags(2) ") @@ -81,10 +90,10 @@ class TDTestCase: # cache_last_set value for k , v in cache_lasts.items(): - if k.split("_")[-1]==str(v): - tdLog.info(" database %s cache_last value check pass, value is %d "%(k,v) ) + if k=="testdb_"+str(v): + tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) else: - tdLog.exit(" database %s cache_last value check fail, value is %d "%(k,v) ) + tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) # # check storage layer implementation @@ -132,13 +141,10 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - - self.illegal_params() self.prepare_datas() self.check_cache_last_sets() self.restart_check_cache_last_sets() - def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 12b14b0661..391cf3396d 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -123,7 +123,7 @@ class TDTestCase: tdSql.query('show databases;') tdSql.checkData(2,5,'off') - tdSql.error('alter database db strict 'off'') + tdSql.error("alter database db strict 'off'") # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') # tdSql.checkData(2,5,'strict') From 3b094cd21a451662d66660d2c96993470190f400 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 15:02:04 +0800 Subject: [PATCH 94/94] test: valgrind case --- tests/script/tsim/valgrind/basic3.sim | 139 +++++---------------- tests/script/tsim/valgrind/checkError2.sim | 30 ++++- 2 files changed, 56 insertions(+), 113 deletions(-) diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index 7fba66c468..6a42a8eb7f 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -1,13 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v -system sh/exec.sh -n dnode2 -s start -v sql connect -print =============== add dnode2 into cluster -sql create dnode $hostname port 7200 - +print =============== step1: create drop show dnodes $x = 0 step1: $x = $x + 1 @@ -18,135 +15,59 @@ step1: endi sql show dnodes print ---> $data00 $data01 $data02 $data03 $data04 $data05 -print ---> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then +if $rows != 1 then return -1 endi if $data(1)[4] != ready then goto step1 endi -if $data(2)[4] != ready then - goto step1 -endi - -print =============== create database, stable, table -sql create database db vgroups 3 -sql use db -sql create table stb (ts timestamp, c int) tags (t int) -sql create table t0 using stb tags (0) -sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); - -print =============== run show xxxx -sql show dnodes -if $rows != 2 then - return -1 -endi - -sql show mnodes -if $rows != 1 then - return -1 -endi +print =============== step2: create db +sql create database d1 vgroups 2 buffer 3 sql show databases -if $rows != 3 then - return -1 -endi +sql use d1 +sql show vgroups +print =============== step3: create show stable +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) sql show stables if $rows != 1 then return -1 endi +print =============== step4: create show table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then - return -1 -endi - -sql show users -if $rows != 1 then - return -1 -endi - -sql show vgroups if $rows != 3 then return -1 endi -print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` -if $rows != 2 then - return -1 -endi +print =============== step5: insert data +sql insert into ct1 values(now+0d, 10, 2.0, 3.0) +sql insert into ct1 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3) +sql insert into ct2 values(now+0d, 10, 2.0, 3.0) +sql insert into ct2 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3) +sql insert into ct3 values('2022-01-01 00:00:00.000', 10, 2.0, 3.0) -sql select * from information_schema.`mnodes` -if $rows != 1 then - return -1 -endi +print =============== step6: query data +sql select * from ct1 where ts < now -1d and ts > now +1d +sql select * from stb where ts < now -1d and ts > now +1d +sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc +sql select * from stb where ts < now -1d and ts > now +1d order by ts desc -sql select * from information_schema.user_databases -if $rows != 3 then - return -1 -endi - -sql select * from information_schema.user_stables -if $rows != 1 then - return -1 -endi - -sql select * from information_schema.user_tables -if $rows != 31 then - return -1 -endi - -sql select * from information_schema.user_users -if $rows != 1 then - return -1 -endi - -sql select * from information_schema.`vgroups` -if $rows != 3 then - return -1 -endi - -sql show variables; -if $rows != 4 then - return -1 -endi - -sql show dnode 1 variables; -if $rows <= 0 then - return -1 -endi - -sql show local variables; -if $rows <= 0 then - return -1 -endi - -print ==== stop dnode1 and dnode2, and restart dnodes +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT +print =============== check +$null= -print =============== check dnode1 system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content <= 0 then - return 0 +if $system_content > 1 then + return -1 endi -$null= if $system_content == $null then - return 0 -endi - -print =============== check dnode2 -system_content sh/checkValgrind.sh -n dnode2 -print cmd return result ----> [ $system_content ] -if $system_content <= 0 then - return 0 -endi - -$null= -if $system_content == $null then - return 0 + return -1 endi diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 3a82218015..9746fe9ecf 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -23,7 +23,7 @@ if $data(1)[4] != ready then endi print =============== step2: create db -sql create database d1 vgroups 1 buffer 3 +sql create database d1 vgroups 2 buffer 3 sql show databases sql use d1 sql show vgroups @@ -44,24 +44,32 @@ if $rows != 3 then return -1 endi -print =============== step5: insert data +print =============== step5: insert data (null / update) sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) +sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0) +sql insert into ct3 values('2022-03-02 16:59:00.010', 3 , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4, 4, 5), ('2022-04-01 16:59:00.011', 6, 4, 5), ('2022-03-06 16:59:00.013', 8, 4, 5); +sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5); print =============== step6: query data sql select * from ct1 sql select * from stb sql select c1, c2, c3 from ct1 sql select ts, c1, c2, c3 from stb +sql select * from ct1 where ts < now -1d and ts > now +1d +sql select * from stb where ts < now -1d and ts > now +1d +#sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc +#sql select * from stb where ts < now -1d and ts > now +1d order by ts desc print =============== step7: count sql select count(*) from ct1; sql select count(*) from stb; sql select count(ts), count(c1), count(c2), count(c3) from ct1 sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from ct1 where ts < now -1d and ts > now +1d +sql select count(*) from stb where ts < now -1d and ts > now +1d print =============== step8: func sql select first(ts), first(c1), first(c2), first(c3) from ct1 @@ -69,6 +77,20 @@ sql select min(c1), min(c2), min(c3) from ct1 sql select max(c1), max(c2), max(c3) from ct1 sql select sum(c1), sum(c2), sum(c3) from ct1 +print =============== step9: insert select +#sql create table ct4 using stb tags(4000); +#sql insert into ct4 select * from ct1; +#sql select * from ct4; +#sql insert into ct4 select ts,c1,c2,c3 from stb; + +#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +#sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +#sql select * from tb1; + +#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +#sql select * from tb2; + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check