From fac103ecaa6332758e33ac722cd21952601e51b5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 17:26:53 +0800 Subject: [PATCH 01/89] chore: taostools sanitizer flag align with tdengine --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index f0a1ad83ad..73409a950a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2849aa4 + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2fbf082f7d7f9a00a7b72047d256a39021001c14 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Nov 2022 19:55:15 +0800 Subject: [PATCH 02/89] enh: support client redirect processing --- include/libs/qcom/query.h | 11 +- include/util/taoserror.h | 1 + source/libs/scheduler/inc/schInt.h | 22 +++- source/libs/scheduler/src/schTask.c | 148 ++++++++++++++++++++++---- source/libs/scheduler/src/schUtil.c | 21 ++++ source/libs/scheduler/src/scheduler.c | 6 ++ source/util/src/terror.c | 1 + 7 files changed, 185 insertions(+), 25 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 96ccef7164..f51aa88485 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -259,9 +259,15 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) + +#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later + #define NEED_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || (_code) == TSDB_CODE_SYN_NOT_LEADER || \ + (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ @@ -270,7 +276,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ - (_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY) + SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_APP_NOT_READY) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636decc60b..26f22f398c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -92,6 +92,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0129) #define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x012A) #define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B) +#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 6884824ba9..7cd8d608c2 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -145,7 +145,8 @@ typedef struct SSchedulerMgmt { bool exit; int32_t jobRef; int32_t jobNum; - SSchStat stat; + SSchStat stat; + void *timer; SRWLatch hbLock; SHashObj *hbConnections; void *queryMgmt; @@ -202,12 +203,30 @@ typedef struct SSchTaskProfile { int64_t endTs; } SSchTaskProfile; +typedef struct SSchRedirectCtx { + int32_t periodMs; + bool inRedirect; + int32_t totalTimes; + int32_t roundTotal; + int32_t roundTimes; // retry times in current round + int64_t startTs; +} SSchRedirectCtx; + +typedef struct SSchTimerParam { + int64_t rId; + uint64_t queryId; + uint64_t taskId; +} SSchTimerParam; + typedef struct SSchTask { uint64_t taskId; // task id SRWLatch lock; // task reentrant lock int32_t maxExecTimes; // task max exec times int32_t maxRetryTimes; // task max retry times int32_t retryTimes; // task retry times + int32_t delayExecMs; // task execution delay time + tmr_h delayTimer; // task delay execution timer + SSchRedirectCtx redirectCtx; // task redirect context bool waitRetry; // wait for retry int32_t execId; // task current execute index SSchLevel *level; // level @@ -529,6 +548,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); +char *schDumpEpSet(SEpSet *pEpSet); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 7e5b3faedb..9262a9257c 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -340,6 +340,67 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { + SSchRedirectCtx *pCtx = &pTask->redirectCtx; + if (!pCtx->inRedirect) { + pCtx->inRedirect = true; + pCtx->periodMs = tsRedirectPeriod; + pCtx->startTs = taosGetTimestampMs(); + + if (SCH_IS_DATA_BIND_TASK(pTask)) { + if (pEpSet) { + pCtx->roundTotal = pEpSet->numOfEps; + } else { + SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + pCtx->roundTotal = pAddr->epSet.numOfEps; + } + } else { + pCtx->roundTotal = 1; + } + + goto _return; + } + + pCtx->totalTimes++; + + int64_t nowTs = taosGetTimestampMs(); + if ((nowTs - pCtx->startTs) > tsMaxRetryWaitTime) { + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); + } + + if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { + pCtx->roundTotal = pEpSet->numOfEps; + pCtx->roundTimes = 0; + + pTask->delayExecMs = 0; + + goto _return; + } + + pCtx->roundTimes++; + + if (pCtx->roundTimes >= pCtx->roundTotal) { + pCtx->periodMs *= tsRedirectFactor; + if (pCtx->periodMs > tsRedirectMaxPeriod) { + pCtx->periodMs = tsRedirectMaxPeriod; + } + + pTask->delayExecMs = pCtx->periodMs; + + goto _return; + } + + pTask->delayExecMs = 0; + +_return: + + SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, pCtx->totalTimes, pTask->delayExecMs); + + return TSDB_CODE_SUCCESS; +} + int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; @@ -349,14 +410,10 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->retryTimes = 0; } - if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) { - SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, - pTask->maxExecTimes, pTask->execId); - schHandleJobFailure(pJob, rspCode); - return TSDB_CODE_SUCCESS; - } + SCH_ERR_JRET(schChkUpdateRedirectCtx(pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); @@ -368,8 +425,12 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); if (SCH_IS_DATA_BIND_TASK(pTask)) { - if (pData) { + if (pData && pData->pEpSet) { SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); + } else if (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(rspCode)) { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SCH_SWITCH_EPSET(addr); + SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -380,7 +441,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); - SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + SCH_ERR_JRET(schDelayLaunchTask(pJob, pTask)); return TSDB_CODE_SUCCESS; } @@ -428,28 +489,24 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); } - if (SCH_IS_DATA_BIND_TASK(pTask)) { + if (SYNC_OTHER_LEADER_REDIRECT_ERROR(rspCode)) { if (NULL == pData->pEpSet) { - SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); - code = rspCode; + SCH_TASK_ELOG("epset updating excepted, error:%s", tstrerror(rspCode)); + code = TSDB_CODE_INVALID_MSG; goto _return; } } code = schDoTaskRedirect(pJob, pTask, pData, rspCode); - taosMemoryFree(pData->pData); - taosMemoryFree(pData->pEpSet); - pData->pData = NULL; - pData->pEpSet = NULL; + taosMemoryFreeClear(pData->pData); + taosMemoryFreeClear(pData->pEpSet); SCH_RET(code); _return: - taosMemoryFree(pData->pData); - taosMemoryFree(pData->pEpSet); - pData->pData = NULL; - pData->pEpSet = NULL; + taosMemoryFreeClear(pData->pData); + taosMemoryFreeClear(pData->pEpSet); SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -715,10 +772,10 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); - SEp *pOld = &pAddr->epSet.eps[pAddr->epSet.inUse]; - SEp *pNew = &pEpSet->eps[pEpSet->inUse]; + char *origEpset = schDumpEpSet(&pAddr->epSet); + char *newEpset = schDumpEpSet(pEpSet); - SCH_TASK_DLOG("update task ep from %s:%d to %s:%d", pOld->fqdn, pOld->port, pNew->fqdn, pNew->port); + SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); @@ -1078,6 +1135,53 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } +void schHandleTimerEvent(void *param, void *tmrId) { + SSchTimerParam *pTimerParam = (SSchTimerParam *)param; + SSchTask *pTask = NULL; + SSchJob *pJob = NULL; + int32_t code = 0; + + SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)); + + SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + + return; + +_return: + + schHandleJobFailure(pJob, code); +} + +int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { + if (pTask->delayExecMs > 0) { + SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); + if (NULL == param) { + SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->rId = pJob->refId; + param->queryId = pJob->queryId; + param->taskId = pTask->taskId; + + if (NULL == pTask->delayTimer) { + pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); + if (NULL == pTask->delayTimer) { + SCH_TASK_ELOG("start delay timer failed"); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void*)param, schMgmt.timer, &pTask->delayTimer); + + return TSDB_CODE_SUCCESS; + } + + SCH_RET(schLaunchTask(pJob, pTask)); +} + int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { SCH_ERR_RET(schChkJobNeedFlowCtrl(pJob, level)); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 1f1288fcfd..a5a15cd1c6 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -36,6 +36,27 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } +char *schDumpEpSet(SEpSet *pEpSet) { + if (NULL == pEpSet) { + return NULL; + } + + int32_t maxSize = 1024; + char *str = taosMemoryMalloc(maxSize); + if (NULL == str) { + return NULL; + } + + int32_t n = 0; + n += snprintf(str + n, maxSize - n, "numOfEps:%d, inUse:%d eps:", pEpSet->numOfEps, pEpSet->inUse); + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + SEp *pEp = &pEpSet->eps[i]; + n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); + } + + return str; +} + char *schGetOpStr(SCH_OP_TYPE type) { switch (type) { case SCH_OP_NULL: diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c4b45649a4..91c3b5d7a1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -48,6 +48,12 @@ int32_t schedulerInit() { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + schMgmt.timer = taosTmrInit(0, 0, 0, "scheduler"); + if (NULL == schMgmt.timer) { + qError("init timer failed, error:%s", tstrerror(terrno)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { qError("generate schdulerId failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 43602a607a..ef1cd6556c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -95,6 +95,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") From ba23ed231740c191e20d6fb3decd3ff24467ba64 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 20 Nov 2022 18:53:48 +0800 Subject: [PATCH 03/89] add simple retry --- include/libs/transport/trpc.h | 5 +++ source/libs/transport/inc/transComm.h | 9 +++++ source/libs/transport/inc/transportInt.h | 5 +++ source/libs/transport/src/trans.c | 5 +++ source/libs/transport/src/transCli.c | 46 +++++++++++++++++++----- 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 8cc37910fd..3083e70574 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -85,6 +85,11 @@ typedef struct SRpcInit { int32_t retryLimit; // retry limit int32_t retryInterval; // retry interval ms + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimouet; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int8_t encryption; // encrypt or not diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index ac54749ae1..1102ddd259 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -146,6 +146,15 @@ typedef struct { SCvtAddr cvtAddr; bool setMaxRetry; + int32_t retryMinInterval; + int32_t retryMaxInterval; + int32_t retryStepFactor; + int32_t retryMaxTimeout; + int64_t retryInitTimestamp; + int64_t retryNextInterval; + bool retryInit; + int32_t retryStep; + int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c8a56081cc..833937aa41 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -52,6 +52,11 @@ typedef struct { int32_t retryLimit; // retry limit int32_t retryInterval; // retry interval ms + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimouet; + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 94bc128de9..415a8766e3 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -51,6 +51,11 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->retryLimit = pInit->retryLimit; pRpc->retryInterval = pInit->retryInterval; + pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval + pRpc->retryStepFactor = pInit->retryStepFactor; + pRpc->retryMaxInterval = pInit->retryMaxInterval; + pRpc->retryMaxTimouet = pInit->retryMaxTimouet; + // register callback handle pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4fb00b1a6d..7581611654 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1371,7 +1371,8 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, pTransInst->retryInterval); + + transDQSched(pThrd->delayQueue, doDelayTask, arg, pCtx->retryNextInterval); } FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { @@ -1419,18 +1420,47 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { int32_t code = pResp->code; bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; + if (retry == true) { + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInit = true; + pCtx->retryStep = 1; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + } else { + pCtx->retryStep++; + int64_t factor = 1; + for (int i = 0; i < pCtx->retryStep - 1; i++) { + factor *= pCtx->retryStepFactor; + } + + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + } + + if (taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + retry = false; + } + } + if (retry) { pMsg->sent = 0; pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, EPSET_GET_SIZE(&pCtx->epSet) * 3); - if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); - EPSET_FORWARD_INUSE(&pCtx->epSet); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } + // if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); + EPSET_FORWARD_INUSE(&pCtx->epSet); + transFreeMsg(pResp->pCont); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; + //} } else { cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, pTransInst->retryLimit); if (pCtx->retryCnt < pCtx->retryLimit) { From 072a73ec3d1d2b3547bc1ba8b2adec3d553978e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 11:35:52 +0800 Subject: [PATCH 04/89] enh: support max retry wait time configuration --- include/common/tglobal.h | 4 +++ source/common/src/tglobal.c | 8 +++++ source/libs/scheduler/inc/schInt.h | 2 ++ source/libs/scheduler/src/schRemote.c | 11 ++++-- source/libs/scheduler/src/schTask.c | 49 ++++++++++++++++----------- 5 files changed, 51 insertions(+), 23 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2076906f70..d23b1f519a 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -90,6 +90,10 @@ extern int32_t tsQueryNodeChunkSize; extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; +extern int32_t tsRedirectPeriod; +extern int32_t tsRedirectFactor; +extern int32_t tsRedirectMaxPeriod; +extern int32_t tsMaxRetryWaitTime; // client extern int32_t tsMinSlidingTime; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 27dcbd5be3..cb82ad300b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -87,6 +87,10 @@ bool tsQueryPlannerTrace = false; int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; +int32_t tsRedirectPeriod = 100; +int32_t tsRedirectFactor = 5; +int32_t tsRedirectMaxPeriod = 10000; +int32_t tsMaxRetryWaitTime = 60000; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -301,6 +305,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); @@ -645,6 +650,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32; tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32; + tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; return 0; } @@ -860,6 +866,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + } else if (strcasecmp("maxRetryWaitTime", name) == 0) { + tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } break; } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7cd8d608c2..83b58a77d2 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -27,6 +27,7 @@ extern "C" { #include "tarray.h" #include "thash.h" #include "trpc.h" +#include "ttimer.h" enum { SCH_READ = 1, @@ -507,6 +508,7 @@ extern SSchedulerMgmt schMgmt; void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); +int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); int32_t schReleaseJob(int64_t refId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index a6a2a6c301..4ebe07cf58 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -887,9 +887,14 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo)); SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); - qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, - epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); - + if (pJob && pTask) { + SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, + epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); + } else { + qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, + epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); + } + if (pTask) { pTask->lastMsgType = msgType; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9262a9257c..0b235e63e9 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -340,7 +340,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { +int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { SSchRedirectCtx *pCtx = &pTask->redirectCtx; if (!pCtx->inRedirect) { pCtx->inRedirect = true; @@ -362,13 +362,6 @@ int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { } pCtx->totalTimes++; - - int64_t nowTs = taosGetTimestampMs(); - if ((nowTs - pCtx->startTs) > tsMaxRetryWaitTime) { - SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", - nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); - SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); - } if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; @@ -382,12 +375,21 @@ int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { pCtx->roundTimes++; if (pCtx->roundTimes >= pCtx->roundTotal) { + int64_t nowTs = taosGetTimestampMs(); + int64_t lastTime = nowTs - pCtx->startTs; + if (lastTime > tsMaxRetryWaitTime) { + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); + } + pCtx->periodMs *= tsRedirectFactor; if (pCtx->periodMs > tsRedirectMaxPeriod) { pCtx->periodMs = tsRedirectMaxPeriod; } - pTask->delayExecMs = pCtx->periodMs; + int64_t leftTime = tsMaxRetryWaitTime - lastTime; + pTask->delayExecMs = leftTime < pCtx->periodMs ? leftTime : pCtx->periodMs; goto _return; } @@ -410,7 +412,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->retryTimes = 0; } - SCH_ERR_JRET(schChkUpdateRedirectCtx(pTask, pData ? pData->pEpSet : NULL)); + SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; @@ -431,6 +433,10 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); + } else { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; + SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1141,15 +1147,13 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchJob *pJob = NULL; int32_t code = 0; - SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)); + if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { + return; + } - SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + code = schLaunchTask(pJob, pTask); - return; - -_return: - - schHandleJobFailure(pJob, code); + schProcessOnCbEnd(pJob, pTask, code); } int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { @@ -1157,7 +1161,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } param->rId = pJob->refId; @@ -1167,8 +1171,8 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { if (NULL == pTask->delayTimer) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { - SCH_TASK_ELOG("start delay timer failed"); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -1203,7 +1207,12 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { while (pIter) { SSchTask *pTask = *(SSchTask **)pIter; + SCH_LOCK_TASK(pTask); + if (pTask->delayTimer) { + taosTmrStopA(&pTask->delayTimer); + } schDropTaskOnExecNode(pJob, pTask); + SCH_UNLOCK_TASK(pTask); pIter = taosHashIterate(list, pIter); } From e7e17302a4e6a019156d3a9094d022deb2e5e627 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 21 Nov 2022 15:35:36 +0800 Subject: [PATCH 05/89] enh(sync): use TSDB_CODE_SYN_NOT_LEADER instead of TSDB_CODE_RPC_REDIRECT --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 38cb534d7f..6a0e79902f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -66,7 +66,7 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { } pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; tmsgSendRedirectRsp(&rsp, &newEpSet); } From 62bf1c024a97e17142d39e118cd1282f972f1008 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Nov 2022 21:25:15 +0800 Subject: [PATCH 06/89] refactor retry --- include/libs/transport/trpc.h | 2 +- source/client/src/clientEnv.c | 4 + source/libs/transport/inc/transComm.h | 7 +- source/libs/transport/src/transCli.c | 149 +++++++++++++++----------- 4 files changed, 93 insertions(+), 69 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3083e70574..d761813db1 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -88,7 +88,7 @@ typedef struct SRpcInit { int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval - int32_t retryMaxTimouet; + int64_t retryMaxTimouet; int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int8_t encryption; // encrypt or not diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3f4e1bb513..48ecb5caac 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -148,6 +148,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.dfp = destroyAhandle; rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; + rpcInit.retryMinInterval = 100; + rpcInit.retryStepFactor = 5; + rpcInit.retryMaxInterval = 10240; + rpcInit.retryMaxTimouet = -1; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 1102ddd259..9add91cdeb 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -137,9 +137,6 @@ typedef struct { tmsg_t msgType; // message type int8_t connType; // connection type cli/srv - int8_t retryCnt; - int8_t retryLimit; - STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -149,12 +146,14 @@ typedef struct { int32_t retryMinInterval; int32_t retryMaxInterval; int32_t retryStepFactor; - int32_t retryMaxTimeout; + int64_t retryMaxTimeout; int64_t retryInitTimestamp; int64_t retryNextInterval; bool retryInit; int32_t retryStep; + int8_t epsetRetryCnt; + int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7581611654..7ffddda777 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -959,7 +959,7 @@ FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { if (code != 0) return false; - if (pCtx->retryCnt == 0) return false; + // if (pCtx->retryCnt == 0) return false; if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; return true; } @@ -1365,8 +1365,8 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGDebug("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, - pCtx->retryCnt + 1, pCtx->retryLimit); + tGDebug("%s retry on next node,use:%s, step: %d,timeout:%" PRId64 "", transLabel(pThrd->pTransInst), tbuf, + pCtx->retryStep, pCtx->retryNextInterval); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1406,6 +1406,86 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { *dst = epset; return true; } +bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp) { + bool noDelay = true; + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } else { + SEpSet epset; + if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset) < 0) { + // invalid epset + EPSET_FORWARD_INUSE(&pCtx->epSet); + } else if (!transEpSetIsEqual(&pCtx->epSet, &epset)) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } + return noDelay; +} +bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + STransConnCtx* pCtx = pMsg->ctx; + int32_t code = pResp->code; + + bool retry = pTransInst->retry(code, pResp->msgType - 1); + if (retry == false) { + return false; + } + + bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + noDelay = cliResetEpset(pCtx, pResp); + transFreeMsg(pResp->pCont); + transUnrefCliHandle(pConn); + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + noDelay = cliResetEpset(pCtx, pResp); + transFreeMsg(pResp->pCont); + addConnToPool(pThrd->pool, pConn); + } else { + noDelay = cliResetEpset(pCtx, pResp); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); + } + + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 1; + pCtx->retryInit = true; + } else { + if (noDelay == false) { + pCtx->epsetRetryCnt = 0; + pCtx->retryStep++; + + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; + } + } + + cliSchedMsgToNextNode(pMsg, pThrd); + return false; +} int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -1419,69 +1499,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; + bool retry = cliGenRetryRule(pConn, pResp, pMsg); if (retry == true) { - if (!pCtx->retryInit) { - pCtx->retryMinInterval = pTransInst->retryMinInterval; - pCtx->retryMaxInterval = pTransInst->retryMaxInterval; - pCtx->retryStepFactor = pTransInst->retryStepFactor; - pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; - pCtx->retryInit = true; - pCtx->retryStep = 1; - pCtx->retryInitTimestamp = taosGetTimestampMs(); - pCtx->retryNextInterval = pCtx->retryMinInterval; - } else { - pCtx->retryStep++; - int64_t factor = 1; - for (int i = 0; i < pCtx->retryStep - 1; i++) { - factor *= pCtx->retryStepFactor; - } - - pCtx->retryNextInterval = factor * pCtx->retryMinInterval; - if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { - pCtx->retryNextInterval = pCtx->retryMaxInterval; - } - } - - if (taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - retry = false; - } + return -1; } - - if (retry) { - pMsg->sent = 0; - pCtx->retryCnt += 1; - - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { - cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, EPSET_GET_SIZE(&pCtx->epSet) * 3); - // if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); - EPSET_FORWARD_INUSE(&pCtx->epSet); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - //} - } else { - cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, pTransInst->retryLimit); - if (pCtx->retryCnt < pCtx->retryLimit) { - if (pResp->contLen == 0) { - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else { - if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet) < 0) { - tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn), pConn); - } - } - addConnToPool(pThrd->pool, pConn); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } else { - // change error code for taos client driver if retryCnt exceeds limit - if (0 == strncmp(pTransInst->label, "TSC", strlen("TSC"))) pResp->code = TSDB_CODE_APP_NOT_READY; - } - } - } - STraceId* trace = &pResp->info.traceId; bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); From 963d56cfc19ac7aa8883d512c3f674b62dcf381b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 11:10:28 +0800 Subject: [PATCH 07/89] retry code --- include/util/taoserror.h | 1 + source/libs/transport/src/transCli.c | 45 +++++++++++++++++----------- source/util/src/terror.c | 1 + 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 26f22f398c..f9a6816c5c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -415,6 +415,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_PROPOSE_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0911) #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) +#define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7ffddda777..d3a2d45d88 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1406,24 +1406,33 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { *dst = epset; return true; } -bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp) { +bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; - if (pResp->contLen == 0) { - if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { - noDelay = false; - } else { - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (hasEpSet == false) { + assert(pResp->contLen == 0); + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } } else { - SEpSet epset; - if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset) < 0) { - // invalid epset - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else if (!transEpSetIsEqual(&pCtx->epSet, &epset)) { - noDelay = false; + SEpSet epSet; + + assert(pResp->contLen == sizeof(epSet)); + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + assert(0); + } + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; } else { + tDebug("epset equal, continue"); EPSET_FORWARD_INUSE(&pCtx->epSet); } + noDelay = false; } return noDelay; } @@ -1440,18 +1449,20 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - noDelay = cliResetEpset(pCtx, pResp); + noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { - noDelay = cliResetEpset(pCtx, pResp); + noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); + } else if (code == TSDB_CODE_SYN_RESTORING) { + noDelay = cliResetEpset(pCtx, pResp, false); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); } else { - noDelay = cliResetEpset(pCtx, pResp); - addConnToPool(pThrd->pool, pConn); - transFreeMsg(pResp->pCont); } if (!pCtx->retryInit) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ef1cd6556c..be41a71535 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -407,6 +407,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for re TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq From c84f9f86340b96405fb22e585f2119d802d304ba Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 18:29:31 +0800 Subject: [PATCH 08/89] add debug info --- source/libs/scheduler/src/schTask.c | 30 +++++++++++++++------------- source/libs/transport/src/transCli.c | 8 +++++--- source/libs/transport/src/transSvr.c | 18 ++++++++--------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 0b235e63e9..af78647018 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -346,7 +346,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) pCtx->inRedirect = true; pCtx->periodMs = tsRedirectPeriod; pCtx->startTs = taosGetTimestampMs(); - + if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; @@ -360,7 +360,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) goto _return; } - + pCtx->totalTimes++; if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { @@ -378,8 +378,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) int64_t nowTs = taosGetTimestampMs(); int64_t lastTime = nowTs - pCtx->startTs; if (lastTime > tsMaxRetryWaitTime) { - SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", - nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); } @@ -398,8 +398,9 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) _return: - SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, pCtx->totalTimes, pTask->delayExecMs); - + SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, + pCtx->totalTimes, pTask->delayExecMs); + return TSDB_CODE_SUCCESS; } @@ -415,7 +416,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; - + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); @@ -435,8 +436,9 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; - SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port); + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; + SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, + addr->epSet.numOfEps, pEp->fqdn, pEp->port); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1146,7 +1148,7 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchTask *pTask = NULL; SSchJob *pJob = NULL; int32_t code = 0; - + if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { return; } @@ -1160,15 +1162,15 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { if (pTask->delayExecMs > 0) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { - SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); + SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + param->rId = pJob->refId; param->queryId = pJob->queryId; param->taskId = pTask->taskId; - if (NULL == pTask->delayTimer) { + if (NULL == pTask->delayTimer) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); @@ -1178,7 +1180,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void*)param, schMgmt.timer, &pTask->delayTimer); + taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d3a2d45d88..8b65794475 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1035,12 +1035,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { addr.sin_family = AF_INET; addr.sin_addr.s_addr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, conn->ip); addr.sin_port = (uint16_t)htons((uint16_t)conn->port); - tTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); + + STraceId* trace = &(pMsg->msg.info.traceId); + tGTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); int ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - tTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, - uv_err_name(ret)); + tGTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, + uv_err_name(ret)); uv_timer_stop(conn->timer); conn->timer->data = NULL; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5f36d91023..374d445874 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1233,17 +1233,17 @@ int transReleaseSrvHandle(void* handle) { m->msg = tmsg; m->type = Release; - tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); + tDebug("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to send to release handle", exh); + tDebug("handle %p failed to send to release handle", exh); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to send to release handle", exh); + tDebug("handle %p failed to send to release handle", exh); return -1; } int transSendResponse(const STransMsg* msg) { @@ -1268,19 +1268,19 @@ int transSendResponse(const STransMsg* msg) { m->type = Normal; STraceId* trace = (STraceId*)&msg->info.traceId; - tGTrace("conn %p start to send resp (1/2)", exh->handle); + tGDebug("conn %p start to send resp (1/2)", exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to send resp", exh); + tDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to send resp", exh); + tDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); return -1; } @@ -1302,19 +1302,19 @@ int transRegisterMsg(const STransMsg* msg) { m->type = Register; STrans* pTransInst = pThrd->pTransInst; - tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); + tDebug("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to register brokenlink", exh); + tDebug("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to register brokenlink", exh); + tDebug("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); return -1; } From be410b9903f2380ed55c7af1f377192a12c54016 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 21:33:11 +0800 Subject: [PATCH 09/89] add debug info --- source/client/src/clientEnv.c | 2 +- source/libs/transport/src/transCli.c | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 48ecb5caac..4eb6eb0458 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -151,7 +151,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.retryMinInterval = 100; rpcInit.retryStepFactor = 5; rpcInit.retryMaxInterval = 10240; - rpcInit.retryMaxTimouet = -1; + rpcInit.retryMaxTimouet = 20480; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b65794475..160e4ca8e6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -234,7 +234,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps != 0) +#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps >= 0 && (epSet)->inUse >= 0) #define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) @@ -996,9 +996,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); + + char tbuf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, tbuf); + tDebug("current epset %s", tbuf); + if (!EPSET_IS_VALID(&pCtx->epSet)) { - destroyCmsg(pMsg); tError("invalid epset"); + destroyCmsg(pMsg); return; } @@ -1453,18 +1458,25 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { bool noDelay = false; if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); } else if (code == TSDB_CODE_SYN_RESTORING) { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } else { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); } if (!pCtx->retryInit) { @@ -1495,9 +1507,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->epsetRetryCnt++; } } - + pMsg->sent = 0; cliSchedMsgToNextNode(pMsg, pThrd); - return false; + return true; } int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; From febe918d36db31420644451276c36d953a390873 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 10:28:38 +0800 Subject: [PATCH 10/89] add debug info --- source/libs/transport/src/transCli.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 160e4ca8e6..813a7983e4 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1427,10 +1427,10 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { } else { SEpSet epSet; - assert(pResp->contLen == sizeof(epSet)); + // assert(pResp->contLen == sizeof(epSet)); int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); if (valid < 0) { - assert(0); + // assert(0); } if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); @@ -1454,6 +1454,11 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (retry == false) { return false; } + if (pCtx->retryInit) { + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } bool noDelay = false; From fbba39ad1750082e8d89489fc4ade99397cdc153 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 17:19:15 +0800 Subject: [PATCH 11/89] add parameter to rpc --- source/client/src/clientEnv.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 4eb6eb0458..6f1414b72d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -148,10 +148,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.dfp = destroyAhandle; rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; - rpcInit.retryMinInterval = 100; - rpcInit.retryStepFactor = 5; - rpcInit.retryMaxInterval = 10240; - rpcInit.retryMaxTimouet = 20480; + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { From 3fd6de0c53e2b6b8304f14a5f4f35508d2e7b571 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Nov 2022 14:20:30 +0800 Subject: [PATCH 12/89] change code --- source/libs/transport/src/transCli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 813a7983e4..061ae9cb57 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1467,7 +1467,8 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); - } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || + code == TSDB_CODE_SYN_PROPOSE_NOT_READY) { tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 55af567ccc8e415be2e356eaaf39b6eabfab5b01 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Nov 2022 14:47:06 +0800 Subject: [PATCH 13/89] change code --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 061ae9cb57..8b16c177f7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -367,7 +367,7 @@ void cliHandleResp(SCliConn* conn) { STraceId* trace = &transMsg.info.traceId; tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, - TMSG_INFO(pHead->msgType), conn->dst, conn->src, msgLen, tstrerror(transMsg.code)); + TMSG_INFO(pHead->msgType), conn->dst, conn->src, pHead->msgLen, tstrerror(transMsg.code)); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); From 294af08f7a23724b249468b12168600c01583358 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 27 Nov 2022 16:04:36 +0800 Subject: [PATCH 14/89] fix(query): handle countAlwaysReturnValue behavior when input data is empty --- source/libs/executor/src/executorimpl.c | 76 ++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4319dd379a..a743465755 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1595,6 +1595,62 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) { + if (!tsCountAlwaysReturnValue) { + return TSDB_CODE_SUCCESS; + } + + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + bool hasCountFunc = false; + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0)) { + hasCountFunc = true; + break; + } + } + + if (!hasCountFunc) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = createDataBlock(); + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + pBlock->info.rowSize = 0; + pBlock->info.groupId = 0; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SColumnInfoData pCol = {0}; + pCol.hasNull = true; + pCol.info.type = TSDB_DATA_TYPE_NULL; + + SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; + for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { + SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { + int32_t slotId = pFuncParam->pCol->slotId; + taosArrayPush(pBlock->pDataBlock, &pCol); + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + } + } + } + + *ppBlock = pBlock; + + return TSDB_CODE_SUCCESS; +} + +static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) { + if (!blockAllocated) { + return; + } + + blockDataDestroy(*ppBlock); + *ppBlock = NULL; +} + + // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { @@ -1612,14 +1668,27 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; + bool hasValidBlock = false; + bool blockAllocated = false; + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - break; + if (!hasValidBlock) { + createDataBlockForEmptyInput(pOperator, &pBlock); + if (pBlock == NULL) { + break; + } + blockAllocated = true; + } else { + break; + } } + hasValidBlock = true; int32_t code = getTableScanInfo(pOperator, &order, &scanFlag); if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } @@ -1628,6 +1697,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SExprSupp* pSup1 = &pAggInfo->scalarExprSup; code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } } @@ -1637,8 +1707,12 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { setInputDataBlock(pSup, pBlock, order, scanFlag, true); code = doAggregateImpl(pOperator, pSup->pCtx); if (code != 0) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } + + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + } initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); From c019645193f83fec554614f25540cfc5a3f5a79f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 27 Nov 2022 20:38:39 +0800 Subject: [PATCH 15/89] fix bugs --- source/libs/executor/src/executorimpl.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a743465755..402f267c6d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1615,27 +1615,35 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc } SSDataBlock* pBlock = createDataBlock(); - pBlock->info.rows = 0; + pBlock->info.rows = 1; pBlock->info.capacity = 0; - pBlock->info.rowSize = 0; pBlock->info.groupId = 0; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SColumnInfoData pCol = {0}; - pCol.hasNull = true; - pCol.info.type = TSDB_DATA_TYPE_NULL; + SColumnInfoData colInfo = {0}; + colInfo.hasNull = true; + colInfo.info.type = TSDB_DATA_TYPE_NULL; + colInfo.info.bytes = 1; SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - taosArrayPush(pBlock->pDataBlock, &pCol); + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + if (slotId >= numOfCols) { + taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); + for (int32_t k = numOfCols; k < slotId + 1; ++k) { + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + } } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + // do nothing } } } + blockDataEnsureCapacity(pBlock, pBlock->info.rows); *ppBlock = pBlock; return TSDB_CODE_SUCCESS; From ab86b6ba47410ce27585960a92d69c3e48ed51cb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 00:30:39 +0800 Subject: [PATCH 16/89] fix hyperloglog bugs --- include/libs/function/function.h | 2 +- source/libs/executor/src/executorimpl.c | 4 +++- source/libs/function/src/builtinsimpl.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 6f2a675466..0831e477ff 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -57,7 +57,7 @@ typedef struct SFuncExecFuncs { #define MAX_INTERVAL_TIME_WINDOW 10000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 -#define FUNCTIONS_NAME_MAX_LENGTH 16 +#define FUNCTIONS_NAME_MAX_LENGTH 32 typedef struct SResultRowEntryInfo { bool initialized : 1; // output buffer has been initialized diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 402f267c6d..1f0ccf02ce 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1604,7 +1604,9 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc bool hasCountFunc = false; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0)) { + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) { hasCountFunc = true; break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 35f50cebca..141cd04263 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4787,6 +4787,10 @@ int32_t hllFunction(SqlFunctionCtx* pCtx) { int32_t numOfRows = pInput->numOfRows; int32_t numOfElems = 0; + if (IS_NULL_TYPE(type)) { + goto _hll_over; + } + for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_s(pCol, i)) { continue; @@ -4808,6 +4812,7 @@ int32_t hllFunction(SqlFunctionCtx* pCtx) { } } +_hll_over: pInfo->totalCount += numOfElems; if (pInfo->totalCount == 0 && !tsCountAlwaysReturnValue) { From 4c158238bab5d25548e484abcb0cfaea03ad1bba Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 01:00:34 +0800 Subject: [PATCH 17/89] add test cases --- tests/system-test/2-query/count.py | 144 +++++++++++++++++- .../2-query/countAlwaysReturnValue.py | 68 +++++++++ 2 files changed, 204 insertions(+), 8 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 254a8792af..8962cf305a 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -51,6 +51,136 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],rownum) tdSql.query(f'select count({k}) from {ntbname} where ts <={self.ts+self.rowNum-2}') tdSql.checkEqual(tdSql.queryResult[0][0],rownum-1) + def query_empty_stb(self): + tdSql.query(f'select count(*) from (select distinct tbname from {self.stbname})') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'select count(*) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + function_names = ['count', 'hyperloglog'] + for function_name in function_names: + tdSql.query(f'select {function_name}(tbname) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(ts) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c2),max(1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.query(f'select sum(1),{function_name}(1),max(c2) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, None) + tdSql.query(f'select {function_name}(1),sum(1),max(c2),min(1),min(2),min(3),min(4),min(5),min(6),min(7),min(8) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(11) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 10, None) + tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') + tdSql.checkRows(2) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') + tdSql.checkRows(0) + + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) + tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 2) + def query_empty_ntb(self): + tdSql.query(f'select count(*) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + function_names = ['count', 'hyperloglog'] + for function_name in function_names: + tdSql.query(f'select {function_name}(tbname) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(ts) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c2),max(1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.query(f'select sum(1),{function_name}(1),max(c2) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, None) + tdSql.query(f'select {function_name}(1),sum(1),max(c2),min(1),min(2),min(3),min(4),min(5),min(6),min(7),min(8) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(11) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 10, None) + tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by tbname interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') + tdSql.checkRows(0) + + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) @@ -81,11 +211,11 @@ class TDTestCase: def check_ntb(self): tdSql.prepare() tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) - tdSql.query(f'select count(tbname) from {self.ntbname}') - tdSql.checkRows(0) + self.query_empty_ntb() tdSql.execute('flush database db') tdSql.query(f'select count(tbname) from {self.ntbname}') - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) self.insert_data(self.column_dict,self.ntbname,self.rowNum) self.count_query_ntb(self.column_dict,self.ntbname,self.rowNum) tdSql.execute('flush database db') @@ -96,13 +226,11 @@ class TDTestCase: tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) for i in range(self.tbnum): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') - tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') - tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) - tdSql.query(f'select count(tbname) from {self.stbname}') - tdSql.checkRows(0) + self.query_empty_stb() tdSql.execute('flush database db') tdSql.query(f'select count(tbname) from {self.stbname}') - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) for i in range(self.tbnum): diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 08c5ae104f..244964e486 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -33,6 +33,19 @@ class TDTestCase: f"create table {dbname}.ctb2 using {dbname}.stb tags (2)" ) + tdSql.execute( + f"create table {dbname}.tb_empty (ts timestamp, c0 int)" + ) + tdSql.execute( + f"create table {dbname}.stb_empty (ts timestamp, c0 int) tags (t0 int)" + ) + tdSql.execute( + f"create table {dbname}.ctb1_empty using {dbname}.stb tags (1)" + ) + tdSql.execute( + f"create table {dbname}.ctb2_empty using {dbname}.stb tags (2)" + ) + tdSql.execute( f"insert into {dbname}.tb values (now(), NULL)") @@ -94,6 +107,61 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + # test empty table/input + tdSql.query(f"select count(*) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(ts) from {dbname}.stb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(c0) from {dbname}.ctb1 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(1) from {dbname}.ctb2 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(*) from {dbname}.tb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(ts) from {dbname}.stb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(c0) from {dbname}.ctb1_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(1) from {dbname}.ctb2_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(c0) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(ts) from {dbname}.stb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb1 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb2 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(c0) from {dbname}.tb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(ts) from {dbname}.stb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb1_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb2_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(*), hyperloglog(c0), sum(1), max(c0) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(*), hyperloglog(c0), sum(1), max(c0) from {dbname}.tb_empty") + tdSql.checkRows(0) + def run(self): tdSql.prepare() From f3c1eb829e9f071ebbbde1d131a19b391f059fe8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 10:10:11 +0800 Subject: [PATCH 18/89] refactor retry --- source/libs/transport/src/transCli.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b16c177f7..2f0b9609f3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1425,21 +1425,26 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { } } } else { - SEpSet epSet; - - // assert(pResp->contLen == sizeof(epSet)); + SEpSet epSet; int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); if (valid < 0) { - // assert(0); - } - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { - tDebug("epset not equal, retry new epset"); - pCtx->epSet = epSet; + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + noDelay = true; + } } else { - tDebug("epset equal, continue"); - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + noDelay = false; } - noDelay = false; } return noDelay; } From 60ee8db228df0bfefa81bcbf8d4e6b85d00bffac Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 11:19:56 +0800 Subject: [PATCH 19/89] fix:add log for sml --- source/client/src/clientSml.c | 6 +++++- source/libs/parser/src/parInsertSml.c | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 83bbe392cf..6773cfc638 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -163,6 +163,7 @@ typedef struct { SMLProtocolType protocol; int8_t precision; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; SHashObj *childTables; SHashObj *superTables; @@ -863,6 +864,7 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra } else { ASSERT(0); } + uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP; @@ -2063,7 +2065,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql", info->id); + uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s", info->id, (info->isRawLine ? "rawdata" : sql)); int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { @@ -2575,6 +2577,8 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } + info->isRawLine = (rawLine == NULL); + int32_t perBatch = LINE_BATCH; if (numLines > perBatch) { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index d18b11ad57..7bcfd0646f 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -303,9 +303,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols if (kv) { int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - // uError("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - // uError("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); } if (IS_VAR_DATA_TYPE(kv->type)) { From bd84ecc3618631fc3b45a8b8653d948641d78231 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 16:31:33 +0800 Subject: [PATCH 20/89] fix:[TD-20613] add interface for write raw block with sdfasdf --- include/client/taos.h | 1 + include/os/os.h | 3 +- source/client/src/clientImpl.c | 2 +- source/client/src/clientRawBlockWrite.c | 206 ++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 25887b2879..677c830ff7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -297,6 +297,7 @@ DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); +DLL_EXPORT int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields); DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); diff --git a/include/os/os.h b/include/os/os.h index ab4d0a406e..629b0e27c6 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -43,12 +43,13 @@ extern "C" { #include #include #include -#include #if defined(DARWIN) #else #include #include +#include + #endif #else diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3140371c4..ec94520fb0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1645,7 +1645,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - // version + length + numOfRows + numOfCol + groupId + flag_segment + column_info + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 6a6a5d56d7..ea0b900e68 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1211,6 +1211,211 @@ static void destroyVgHash(void* data) { taosMemoryFreeClear(vgData->data); } +int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){ + int32_t code = TSDB_CODE_SUCCESS; + STableMeta* pTableMeta = NULL; + SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; + + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + if (!pRequest) { + uError("WriteRaw:createRequest error request is null"); + code = terrno; + goto end; + } + + pRequest->syncQuery = true; + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); + + struct SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + SVgroupInfo vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname); + goto end; + } + + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); + goto end; + } + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + int32_t numOfCols = numFields; + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for(int k = 0; k < numOfCols; k++){ + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + SSchema* schema = pTableMeta->schema + i; + if(strcmp(schema->name, fields[k].name) != 0) continue; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if (IS_VAR_DATA_TYPE(schema->type)) { + nVar++; + } + } + } + + fLen -= sizeof(TSKEY); + + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); + int32_t schemaLen = 0; + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + subReq = taosMemoryCalloc(1, totalLen); + SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTableMeta->sversion); + tdSRowSetTpInfo(&rb, numOfCols, fLen); + int32_t dataLen = 0; + + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); + int32_t* colLength = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + pCol[i].offset = (int32_t*)pStart; + pStart += rows * sizeof(int32_t); + } else { + pCol[i].nullbitmap = pStart; + pStart += BitmapLen(rows); + } + + pCol[i].pData = pStart; + pStart += colLength[i]; + } + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + int32_t offset = 0; + for (int32_t k = 0; k < numOfCols; k++) { + const SSchema* pColumn = NULL; + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + if (strcmp((pTableMeta->schema + i)->name, fields[k].name) == 0) { + pColumn = pTableMeta->schema + i; + break; + } + } + if(pColumn == NULL){ + uError("column not exist:%s", fields[k].name); + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } else { + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } + + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + + taosMemoryFree(pCol); + + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->sversion = htonl(pTableMeta->sversion); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htonl(rows); + blk->dataLen = htonl(dataLen); + subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; + subReq->numOfBlocks = 1; + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); + nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vgData; + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + subReq = NULL; // no need free + taosArrayPush(nodeStmt->pDataBlocks, &dst); + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + + end: + taosMemoryFreeClear(pTableMeta); + qDestroyQuery(pQuery); + taosMemoryFree(subReq); + return code; +} + int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; @@ -1293,6 +1498,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; From 9b5d78d23dbb9eaf3a3135a57c2bdc69eb25bbbe Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 16:32:06 +0800 Subject: [PATCH 21/89] fix:[TD-20613] add interface for write raw block with some fields --- source/client/src/clientRawBlockWrite.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ea0b900e68..bcb97b7354 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1294,6 +1294,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SRowBuilder rb = {0}; tdSRowInit(&rb, pTableMeta->sversion); + rb.rowType = TD_ROW_KV; tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; From f81ce17df43575fb70fda83cdbac863d22615045 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:06:02 +0800 Subject: [PATCH 22/89] fix first last --- source/libs/function/src/builtinsimpl.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a2e653469f..ea976c9a79 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2936,6 +2936,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = pInputCol->info.bytes; + if (IS_NULL_TYPE(pInputCol->info.type)) { + return 0; + } + // All null data column, return directly. if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); @@ -3040,6 +3044,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; + if (IS_NULL_TYPE(type)) { + return 0; + } + // All null data column, return directly. if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); From f5db4a8b400a4a5571708a4c0c63defe71554bd0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 22:08:21 +0800 Subject: [PATCH 23/89] change paramter --- source/libs/transport/src/transCli.c | 76 +++++++++++++++------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2f0b9609f3..1f3ee3efaf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1416,13 +1416,19 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; if (hasEpSet == false) { - assert(pResp->contLen == 0); + // assert(pResp->contLen == 0); if (pResp->contLen == 0) { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); } + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } } else { SEpSet epSet; @@ -1433,17 +1439,19 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); - noDelay = true; } } else { if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); pCtx->epSet = epSet; } else { - tDebug("epset equal, continue"); - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } - noDelay = false; } } return noDelay; @@ -1459,14 +1467,22 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (retry == false) { return false; } - if (pCtx->retryInit) { - if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - return false; - } + + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 1; + pCtx->retryInit = true; + } + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; } bool noDelay = false; - if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); @@ -1490,34 +1506,24 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); } - if (!pCtx->retryInit) { - pCtx->retryMinInterval = pTransInst->retryMinInterval; - pCtx->retryMaxInterval = pTransInst->retryMaxInterval; - pCtx->retryStepFactor = pTransInst->retryStepFactor; - pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; - pCtx->retryInitTimestamp = taosGetTimestampMs(); - pCtx->retryNextInterval = pCtx->retryMinInterval; - pCtx->retryStep = 1; - pCtx->retryInit = true; - } else { - if (noDelay == false) { - pCtx->epsetRetryCnt = 0; - pCtx->retryStep++; + if (noDelay == false) { + pCtx->epsetRetryCnt = 1; + pCtx->retryStep++; - int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); - pCtx->retryNextInterval = factor * pCtx->retryMinInterval; - if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { - pCtx->retryNextInterval = pCtx->retryMaxInterval; - } - - if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - return false; - } - } else { - pCtx->retryNextInterval = 0; - pCtx->epsetRetryCnt++; + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; } + + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; } + pMsg->sent = 0; cliSchedMsgToNextNode(pMsg, pThrd); return true; From 91b09893bf61737069c13d8086cfa27dc9439ce3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 22:16:00 +0800 Subject: [PATCH 24/89] change paramter --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1f3ee3efaf..d0b8adeffe 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1489,7 +1489,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT) { tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 8c90a98e2d1371289cf42e87ab1f1f3a5b688ef9 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:25:17 +0800 Subject: [PATCH 25/89] fix last_row --- source/libs/function/src/builtinsimpl.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ea976c9a79..19804bea09 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2937,7 +2937,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = pInputCol->info.bytes; if (IS_NULL_TYPE(pInputCol->info.type)) { - return 0; + return TSDB_CODE_SUCCESS; } // All null data column, return directly. @@ -2945,7 +2945,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { ASSERT(pInputCol->hasNull == true); // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); - return 0; + return TSDB_CODE_SUCCESS; } SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL; @@ -3045,7 +3045,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = bytes; if (IS_NULL_TYPE(type)) { - return 0; + return TSDB_CODE_SUCCESS; } // All null data column, return directly. @@ -3053,7 +3053,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { ASSERT(pInputCol->hasNull == true); // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); - return 0; + return TSDB_CODE_SUCCESS; } SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL; @@ -3318,9 +3318,14 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; + if (IS_NULL_TYPE(type)) { + return TSDB_CODE_SUCCESS; + } + TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); From 05975114607040b9cae3b5f26a9b97d2a2c6164d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:54:27 +0800 Subject: [PATCH 26/89] fix scalar error --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5dbd728cce..7dc95ab26f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1696,7 +1696,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } // there is an scalar expression that needs to be calculated before apply the group aggregation. - if (pAggInfo->scalarExprSup.pExprInfo != NULL) { + if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { SExprSupp* pSup1 = &pAggInfo->scalarExprSup; code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { From 94c8ba64d27a6f8b6ecc760063f14d61c017558f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:55:16 +0800 Subject: [PATCH 27/89] fix test cases --- tests/script/tsim/parser/join.sim | 6 +++--- tests/system-test/2-query/distribute_agg_count.py | 2 +- tests/system-test/2-query/last_row.py | 3 ++- tests/system-test/2-query/mavg.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/script/tsim/parser/join.sim b/tests/script/tsim/parser/join.sim index 8ad5946a54..a2ae25014e 100644 --- a/tests/script/tsim/parser/join.sim +++ b/tests/script/tsim/parser/join.sim @@ -496,18 +496,18 @@ sql create table tm0 using m1 tags('abc', 1); sql create table m2(ts timestamp, k int) tags(a int, b binary(12)); sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; -if $rows != 0 then +if $rows != 1 then return -1 endi sql create table tm2 using m2 tags(2, 'abc'); sql select count(*) from tm0, tm2 where tm0.ts=tm2.ts; -if $rows != 0 then +if $rows != 1 then return -1 endi sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index 7d131cd77d..67d47d7da5 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -183,7 +183,7 @@ class TDTestCase: tdSql.checkRows(20) tdSql.query(f"select count(c1) from {dbname}.stb1 where t1> 4 partition by tbname") - tdSql.checkRows(15) + tdSql.checkRows(16) # union all tdSql.query(f"select count(c1) from {dbname}.stb1 union all select count(c1) from {dbname}.stb1 ") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 01da658989..a6bcc2c5f1 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -402,7 +402,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f"select count(c1) from {dbname}.ct4 where t1 = 1 ") - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0,0,0) tdSql.query(f"select last_row(c1) ,last(c1) from {dbname}.stb1 where c1 is null") tdSql.checkRows(1) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index f76980106d..d659a682b1 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -298,7 +298,7 @@ class TDTestCase: if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): pre_result = np.array(pre_result, dtype = 'int64') - pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k + #pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k tdSql.query(self.mavg_query_form( sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, table_expr=table_expr, condition=condition From daeeacb45032c5abfd01da5cee45889bb60460ac Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 23:54:42 +0800 Subject: [PATCH 28/89] fix compilation error --- source/libs/executor/src/executorimpl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7bddc29530..094060b213 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1443,7 +1443,6 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pBlock = createDataBlock(); pBlock->info.rows = 1; pBlock->info.capacity = 0; - pBlock->info.groupId = 0; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SColumnInfoData colInfo = {0}; From 65aa86edec350b78cedd95cd50282a54b9fe5693 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:11:34 +0800 Subject: [PATCH 29/89] change paramter --- source/common/src/tglobal.c | 2 +- source/libs/transport/src/transCli.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cb82ad300b..02703dd0a1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -88,7 +88,7 @@ int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; int32_t tsRedirectPeriod = 100; -int32_t tsRedirectFactor = 5; +int32_t tsRedirectFactor = 1; int32_t tsRedirectMaxPeriod = 10000; int32_t tsMaxRetryWaitTime = 60000; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d0b8adeffe..429dc80622 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1444,6 +1444,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); pCtx->epSet = epSet; + noDelay = false; } else { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; @@ -1475,7 +1476,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; pCtx->retryInitTimestamp = taosGetTimestampMs(); pCtx->retryNextInterval = pCtx->retryMinInterval; - pCtx->retryStep = 1; + pCtx->retryStep = 0; pCtx->retryInit = true; } if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { From 3b6982e5b164f8293c15044c6750914cc1bbed2d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:23:47 +0800 Subject: [PATCH 30/89] change paramter --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 02703dd0a1..0540776987 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,7 +90,7 @@ bool tsKeepColumnName = false; int32_t tsRedirectPeriod = 100; int32_t tsRedirectFactor = 1; int32_t tsRedirectMaxPeriod = 10000; -int32_t tsMaxRetryWaitTime = 60000; +int32_t tsMaxRetryWaitTime = 60000 * 2; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, From 5bed17233782595c0c554cfbd1422ac96a10109d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 10:31:08 +0800 Subject: [PATCH 31/89] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 55 ++++++++++++++----------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index bcb97b7354..8bae3c23bc 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1294,7 +1294,6 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SRowBuilder rb = {0}; tdSRowInit(&rb, pTableMeta->sversion); - rb.rowType = TD_ROW_KV; tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; @@ -1705,8 +1704,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = &pSW->pSchema[i]; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { @@ -1717,7 +1716,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); + (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); int32_t schemaLen = 0; int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; @@ -1764,14 +1763,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SRowBuilder rb = {0}; tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); + tdSRowSetTpInfo(&rb, pSW->nCols, fLen); int32_t totalLen = 0; - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } +// SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); +// for (int i = 0; i < pSW->nCols; i++) { +// SSchema* schema = &pSW->pSchema[i]; +// taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); +// } for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); @@ -1780,22 +1779,29 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resInfo.current += 1; int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + for (int32_t i = 0; i < pSW->nCols; i++) { + const SSchema* pColumn = NULL; + for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { + if (strcmp(pTableMeta->schema[k].name, pSW->pSchema[i].name) == 0) { + pColumn = &pTableMeta->schema[k]; + break; } } + if (pColumn == NULL){ + uError("column not exist:%s", pSW->pSchema[i].name); + code = TSDB_CODE_INVALID_PARA; + goto end; + } + char* colData = rspObj.resInfo.row[i]; + if (!colData) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + colData -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + } + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { offset += TYPE_BYTES[pColumn->type]; } @@ -1806,7 +1812,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { totalLen += rowLen; } - taosHashCleanup(schemaHash); blk->uid = htobe64(uid); blk->suid = htobe64(suid); blk->sversion = htonl(sver); From 829a7631b38d601ac4a2a475da37546e0624d8de Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 10:49:47 +0800 Subject: [PATCH 32/89] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 8bae3c23bc..a3447dabda 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1794,12 +1794,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { } char* colData = rspObj.resInfo.row[i]; if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, i); } else { if (IS_VAR_DATA_TYPE(pColumn->type)) { colData -= VARSTR_HEADER_SIZE; } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, i); } if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { From 21391ea77b505c683835678ca5fc5f4ed0508d38 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:52:06 +0800 Subject: [PATCH 33/89] change paramter --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5546d762f4..67eff26714 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -258,8 +258,13 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { From 5425d67c9a5ecf7deafef98efcf4386c0e5ddd06 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 11:40:58 +0800 Subject: [PATCH 34/89] add timestamp to msg head --- include/os/osSocket.h | 3 +++ source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/transCli.c | 10 ++++++++++ source/libs/transport/src/transSvr.c | 25 ++++++++++++++++++++----- source/os/src/osSocket.c | 27 +++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 2c7c579401..3ba28d8156 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -169,6 +169,9 @@ void taosSetMaskSIGPIPE(); uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt, char *dstStr, int32_t len); +uint64_t taosHton64(uint64_t val); +uint64_t taosNtoh64(uint64_t val); + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index ac54749ae1..4aa9c6be42 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -161,6 +161,7 @@ typedef struct { char spi : 2; char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset + uint64_t timestamp; char user[TSDB_UNI_LEN]; uint32_t magicNum; STraceId traceId; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 71cc14493f..b4b0b0caf7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -758,6 +758,14 @@ static void cliSendCb(uv_write_t* req, int status) { SCliConn* pConn = transReqQueueRemove(req); if (pConn == NULL) return; + SCliMsg* pMsg = !transQueueEmpty(&pConn->cliMsgs) ? transQueueGet(&pConn->cliMsgs, 0) : NULL; + if (pMsg != NULL) { + int64_t cost = taosGetTimestampUs() - pMsg->st; + if (cost > 1000) { + tWarn("%s conn %p send cost:%dus, send exception", CONN_GET_INST_LABEL(pConn), pConn, (int)cost); + } + } + if (status == 0) { tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); } else { @@ -806,6 +814,7 @@ void cliSend(SCliConn* pConn) { pHead->traceId = pMsg->info.traceId; pHead->magicNum = htonl(TRANS_MAGIC_NUM); } + pHead->timestamp = taosHton64(taosGetTimestampUs()); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -1568,6 +1577,7 @@ int transReleaseCliHandle(void* handle) { SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; + cmsg->st = taosGetTimestampUs(); cmsg->type = Release; cmsg->ctx = pCtx; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 395e28d68f..562e1dc6f6 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -231,14 +231,29 @@ static bool uvHandleReq(SSvrConn* pConn) { } } STraceId* trace = &pHead->traceId; + + int64_t cost = taosGetTimestampUs() - taosNtoh64(pHead->timestamp); + static int64_t EXCEPTION_LIMIT_US = 100 * 1000; + if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - - tGDebug("%s conn %p %s received from %s, local info:%s, len:%d", transLabel(pTransInst), pConn, - TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen); + if (cost >= EXCEPTION_LIMIT_US) { + tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception", transLabel(pTransInst), + pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); + } else { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus", transLabel(pTransInst), pConn, + TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, (int)cost); + } } else { - tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, - TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transMsg.code); + if (cost >= EXCEPTION_LIMIT_US) { + tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus, recv exception", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); + } else { + tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); + } } // pHead->noResp = 1, diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index fd5bde90ba..db2a9937b5 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -1103,3 +1103,30 @@ void taosWinSocketInit() { #else #endif } + +uint64_t taosHton64(uint64_t val) { +#if defined(WINDOWS) || defined(DARWIN) + return ((val & 0x00000000000000ff) << 7 * 8) | ((val & 0x000000000000ff00) << 5 * 8) | + ((val & 0x0000000000ff0000) << 3 * 8) | ((val & 0x00000000ff000000) << 1 * 8) | + ((val & 0x000000ff00000000) >> 1 * 8) | ((val & 0x0000ff0000000000) >> 3 * 8) | + ((val & 0x00ff000000000000) >> 5 * 8) | ((val & 0xff00000000000000) >> 7 * 8); +#else + if (__BYTE_ORDER == __LITTLE_ENDIAN) { + return (((uint64_t)htonl((int)((val << 32) >> 32))) << 32) | (unsigned int)htonl((int)(val >> 32)); + } else if (__BYTE_ORDER == __BIG_ENDIAN) { + return val; + } +#endif +} + +uint64_t taosNtoh64(uint64_t val) { +#if defined(WINDOWS) || defined(DARWIN) + return taosHton64(val); +#else + if (__BYTE_ORDER == __LITTLE_ENDIAN) { + return (((uint64_t)htonl((int)((val << 32) >> 32))) << 32) | (unsigned int)htonl((int)(val >> 32)); + } else if (__BYTE_ORDER == __BIG_ENDIAN) { + return val; + } +#endif +} From 03eaf16d0801271c83c7c966bbd819261985cac7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 11:42:24 +0800 Subject: [PATCH 35/89] add timestamp to msg head --- tests/system-test/0-others/compatibility.py | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 30513ac020..17733f4c15 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -72,9 +72,8 @@ class TDTestCase: def buildTaosd(self,bPath): - # os.system(f"mv {bPath}/build_bak {bPath}/build ") - os.system(f" cd {bPath} && make install ") - + print(f" build path {bPath} ") + os.system(f" cd {bPath} && cmake -DBUILD_TOOLS=on .. && make -j4 && make install") def run(self): bPath=self.getBuildPath() @@ -86,18 +85,18 @@ class TDTestCase: tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 - tdsqlF=tdCom.newTdSql() - print(tdsqlF) - tdsqlF.query(f"SELECT SERVER_VERSION();") - print(tdsqlF.query(f"SELECT SERVER_VERSION();")) - oldServerVersion=tdsqlF.queryResult[0][0] - tdLog.info(f"Base server version is {oldServerVersion}") - tdsqlF.query(f"SELECT CLIENT_VERSION();") + # tdsqlF=tdCom.newTdSql() + # print(tdsqlF) + # tdsqlF.query(f"SELECT SERVER_VERSION();") + # print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + # oldServerVersion=tdsqlF.queryResult[0][0] + # tdLog.info(f"Base server version is {oldServerVersion}") + # tdsqlF.query(f"SELECT CLIENT_VERSION();") - # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison - oldClientVersion=tdsqlF.queryResult[0][0] - tdLog.info(f"Base client version is {oldClientVersion}") - + # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + # oldClientVersion=tdsqlF.queryResult[0][0] + # tdLog.info(f"Base client version is {oldClientVersion}") + oldServerVersion="3.0.1.0" tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") @@ -138,6 +137,9 @@ class TDTestCase: os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers2) + tdsql.query("select count(*) from db4096.stb0") + tdsql.checkData(0,0,50000) + tdsql=tdCom.newTdSql() tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") From b5e4b05752400f732bbd6d777d7c7fadbd3e5c06 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 11:52:40 +0800 Subject: [PATCH 36/89] fix:[TS-2057] add interface with ttl in schemaless --- include/client/taos.h | 10 ++ include/libs/parser/parser.h | 2 +- source/client/src/clientSml.c | 128 ++++++++++--------------- source/libs/parser/inc/parInsertUtil.h | 2 +- source/libs/parser/src/parInsertSml.c | 4 +- source/libs/parser/src/parInsertSql.c | 2 +- source/libs/parser/src/parInsertStmt.c | 2 +- source/libs/parser/src/parInsertUtil.c | 4 +- utils/test/c/sml_test.c | 47 ++++++++- 9 files changed, 108 insertions(+), 93 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 25887b2879..9aaf15e8b0 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -218,6 +218,9 @@ DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_ DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); + +/* --------------------------schemaless INTERFACE------------------------------- */ + DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid); @@ -225,6 +228,13 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, + int precision, int32_t ttl, int64_t reqid); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, + int protocol, int precision, int32_t ttl, int64_t reqid); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 1a7e6dc748..fe471530fa 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -108,7 +108,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery* pQuery); void smlDestroyHandle(void* pHandle); int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen); + char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6773cfc638..f4d8c80e3f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -162,8 +162,9 @@ typedef struct { SMLProtocolType protocol; int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; SHashObj *childTables; SHashObj *superTables; @@ -2325,7 +2326,7 @@ static int32_t smlInsertData(SSmlHandle *info) { code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->msgBuf.buf, info->msgBuf.len); + info->ttl, info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; @@ -2516,7 +2517,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { } TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision) { + int numLines, int protocol, int precision, int32_t ttl) { int batchs = 0; STscObj *pTscObj = request->pTscObj; @@ -2578,6 +2579,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char } info->isRawLine = (rawLine == NULL); + info->ttl = ttl; int32_t perBatch = LINE_BATCH; @@ -2641,88 +2643,43 @@ end: * @return TAOS_RES */ +TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int32_t ttl, int64_t reqid) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); +} + TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, 0); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } -TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, - int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); +TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, 0); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - int numLines = 0; - *totalRows = 0; - char *tmp = lines; - for (int i = 0; i < len; i++) { - if (lines[i] == '\n' || i == len - 1) { - numLines++; - if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) { // ignore comment - (*totalRows)++; - } - tmp = lines + i + 1; - } - } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); +TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); } -TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int64_t reqid) { +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl, int64_t reqid) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2753,5 +2710,16 @@ TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); + return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); +} + +TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +} +TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0); +} + +TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 09d55d369f..5cc72f8692 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -135,7 +135,7 @@ int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, - SArray *tagName, uint8_t tagNum); + SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 7bcfd0646f..e76ca7751d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -208,7 +208,7 @@ end: } int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen) { + char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; @@ -229,7 +229,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags); + pTableMeta->tableInfo.numOfTags, ttl); taosArrayDestroy(tagName); smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9c39954f09..5bba012481 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -562,7 +562,7 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, - pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags); + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index a6ce71211a..e0e191b7c9 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -137,7 +137,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags); + insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); code = insBuildCreateTbMsg(pDataBlock, &tbReq); tdDestroySVCreateTbReq(&tbReq); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5f8120171f..06e32a4415 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -850,7 +850,7 @@ int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchem } void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum) { + SArray* tagName, uint8_t tagNum, int32_t ttl) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; @@ -858,7 +858,7 @@ void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, i if (sname) pTbReq->ctb.stbName = strdup(sname); pTbReq->ctb.pTag = (uint8_t*)pTag; pTbReq->ctb.tagName = taosArrayDup(tagName); - pTbReq->ttl = TSDB_DEFAULT_TABLE_TTL; + pTbReq->ttl = ttl; pTbReq->commentLen = -1; return; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 40d5bb12d2..47b7adbf18 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1084,20 +1084,22 @@ int sml_19221_Test() { return code; } -int sml_time_Test() { +int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase='2022-02-02 10:22:22' 1626006833639000000", + "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1107,8 +1109,43 @@ int sml_time_Test() { return code; } +int sml_ttl_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); + TAOS_ROW row = taos_fetch_row(pRes); + int32_t ttl = *(int32_t*)row[0]; + ASSERT(ttl == 20); + + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int main(int argc, char *argv[]) { int ret = 0; + ret = sml_ttl_Test(); + ASSERT(!ret); + ret = sml_ts2164_Test(); + ASSERT(!ret); ret = smlProcess_influx_Test(); ASSERT(!ret); ret = smlProcess_telnet_Test(); From c79195928fd1d3cfae74ea376303472d2ed07dfc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 13:21:49 +0800 Subject: [PATCH 37/89] chore: re-enable old style build asan config for taos-tools --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e8fb125db7..a3b716d5b7 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2aac500 + GIT_TAG 274d230 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 3b68577563c94a42a5a68ef3fbac79b2ac6403db Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 16:28:34 +0800 Subject: [PATCH 38/89] chore: update taos-tools 274d230 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index a3b716d5b7..8b9104e7e5 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 274d230 + GIT_TAG 9f587e9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From f3681433c66cade5cc9f40e18bf879ff1ea03810 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Nov 2022 16:43:12 +0800 Subject: [PATCH 39/89] fix: stream plan error --- include/libs/nodes/querynodes.h | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5291130e4f..9f980fd0db 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -106,7 +106,7 @@ typedef struct SValueNode { char* p; } datum; int64_t typeData; - char unit; + int8_t unit; } SValueNode; typedef struct SLeftValueNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 462ac513a5..50e410b339 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2952,6 +2952,7 @@ static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; static const char* jkValueNotReserved = "NotReserved"; static const char* jkValueIsNull = "IsNull"; +static const char* jkValueUnit = "Unit"; static const char* jkValueDatum = "Datum"; static int32_t datumToJson(const void* pObj, SJson* pJson) { @@ -3041,6 +3042,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkValueIsNull, pNode->isNull); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkValueUnit, pNode->unit); + } if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = datumToJson(pNode, pJson); } @@ -3191,6 +3195,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkValueIsNull, &pNode->isNull); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkValueUnit, &pNode->unit); + } if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = jsonToDatum(pJson, pNode); } From 8b02719ea579413ed7090360098a24a4a680eed8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 17:01:16 +0800 Subject: [PATCH 40/89] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 122 +++++++++++------------- utils/test/c/tmq_taosx_ci.c | 26 +++++ 2 files changed, 83 insertions(+), 65 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a3447dabda..9db79c6eb6 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1262,22 +1262,19 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch } uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); uint64_t uid = pTableMeta->uid; - int32_t numOfCols = numFields; + int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for(int k = 0; k < numOfCols; k++){ for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { SSchema* schema = pTableMeta->schema + i; - if(strcmp(schema->name, fields[k].name) != 0) continue; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { nVar++; } } - } fLen -= sizeof(TSKEY); @@ -1298,14 +1295,14 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch int32_t dataLen = 0; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); + char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; + pStart += sizeof(int32_t) * numFields; - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + for (int32_t i = 0; i < numFields; ++i) { + if (IS_VAR_DATA_TYPE(fields[i].type)) { pCol[i].offset = (int32_t*)pStart; pStart += rows * sizeof(int32_t); } else { @@ -1317,36 +1314,35 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch pStart += colLength[i]; } + SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int i = 0; i < numFields; i++) { + TAOS_FIELD* schema = &fields[i]; + taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + } + for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); int32_t offset = 0; for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = NULL; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - if (strcmp((pTableMeta->schema + i)->name, fields[k].name) == 0) { - pColumn = pTableMeta->schema + i; - break; - } - } - if(pColumn == NULL){ - uError("column not exist:%s", fields[k].name); - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + const SSchema* pColumn = &pTableMeta->schema[k]; + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { // add none + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); + }else{ + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } } } @@ -1360,6 +1356,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch dataLen += rowLen; } + taosHashCleanup(schemaHash); taosMemoryFree(pCol); blk->uid = htobe64(uid); @@ -1704,8 +1701,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + SSchema* schema = &pTableMeta->schema[i]; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { @@ -1716,7 +1713,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); + (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); int32_t schemaLen = 0; int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; @@ -1763,14 +1760,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SRowBuilder rb = {0}; tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pSW->nCols, fLen); + tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); int32_t totalLen = 0; -// SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); -// for (int i = 0; i < pSW->nCols; i++) { -// SSchema* schema = &pSW->pSchema[i]; -// taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); -// } + SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = &pSW->pSchema[i]; + taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + } for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); @@ -1779,29 +1776,22 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resInfo.current += 1; int32_t offset = 0; - for (int32_t i = 0; i < pSW->nCols; i++) { - const SSchema* pColumn = NULL; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - if (strcmp(pTableMeta->schema[k].name, pSW->pSchema[i].name) == 0) { - pColumn = &pTableMeta->schema[k]; - break; - } - } - if (pColumn == NULL){ - uError("column not exist:%s", pSW->pSchema[i].name); - code = TSDB_CODE_INVALID_PARA; - goto end; - } - char* colData = rspObj.resInfo.row[i]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, i); + for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { + const SSchema* pColumn = &pTableMeta->schema[k]; + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; + char* colData = rspObj.resInfo.row[*index]; + if (!colData) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + colData -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, i); } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { offset += TYPE_BYTES[pColumn->type]; } @@ -1812,6 +1802,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { totalLen += rowLen; } + taosHashCleanup(schemaHash); blk->uid = htobe64(uid); blk->suid = htobe64(suid); blk->sversion = htonl(sver); @@ -1882,6 +1873,7 @@ end: return code; } + static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pVgHash = NULL; @@ -2094,7 +2086,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) const SSchema* pColumn = &pTableMeta->schema[k]; int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); } else { char* colData = rspObj.resInfo.row[*index]; if (!colData) { @@ -2175,7 +2167,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; -end: + end: tDeleteSTaosxRsp(&rspObj.rsp); rspObj.resInfo.pRspMsg = NULL; doFreeReqResultInfo(&rspObj.resInfo); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 6540fdac4c..5af9ba68b2 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -76,6 +76,32 @@ static void msg_process(TAOS_RES* msg) { } int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ + + /* test for TD-20612 start*/ +// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// return 0; + /* test for TD-20612 end*/ + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " "nchar(8), t4 bool)"); From 8a555080f1b6a372b4329da255738c53d03f5201 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:08:53 +0800 Subject: [PATCH 41/89] change parameterr --- source/common/src/tglobal.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4f9089b0cd..b64e3a562c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "tglobal.h" #include "tconfig.h" -#include "tmisce.h" #include "tgrant.h" #include "tlog.h" +#include "tmisce.h" GRANT_CFG_DECLARE; @@ -86,10 +86,10 @@ bool tsQueryPlannerTrace = false; int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; -int32_t tsRedirectPeriod = 100; -int32_t tsRedirectFactor = 1; -int32_t tsRedirectMaxPeriod = 10000; -int32_t tsMaxRetryWaitTime = 60000 * 2; +int32_t tsRedirectPeriod = 10; +int32_t tsRedirectFactor = 2; +int32_t tsRedirectMaxPeriod = 1000; +int32_t tsMaxRetryWaitTime = 10000; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -124,7 +124,7 @@ int32_t tsMinIntervalTime = 1; int32_t tsMaxMemUsedByInsert = 1024; float tsSelectivityRatio = 1.0; -int32_t tsTagFilterResCacheSize = 1024*10; +int32_t tsTagFilterResCacheSize = 1024 * 10; // the maximum allowed query buffer size during query processing for each data node. // -1 no limit (default) From 0b91b5b2b3d41b36754841575552b341de33476a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 29 Nov 2022 14:25:31 +0800 Subject: [PATCH 42/89] fix:calculate the correct deletion range --- source/libs/executor/src/tfill.c | 9 +- .../tsim/stream/fillIntervalDelete1.sim | 167 ++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 42b1d058b5..68228f2689 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -918,8 +918,11 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS return; } + TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->needFill = true; - pFillInfo->start = start; + pFillInfo->start = realStart; pFillInfo->current = pFillInfo->start; pFillInfo->end = end; pFillInfo->pos = FILL_POS_INVALID; @@ -1418,9 +1421,13 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { if (code == TSDB_CODE_SUCCESS) { code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); } + // ts will be deleted later if (delTs != ts) { streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + streamStateFreeCur(pCur); + pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } + endTs = nextKey.ts - 1; if (code != TSDB_CODE_SUCCESS) { break; } diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index e14062e830..ca347f120a 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -344,14 +344,176 @@ endi +sql drop stream if exists streams11; +sql drop stream if exists streams12; +sql drop stream if exists streams13; +sql drop stream if exists streams14; +sql drop stream if exists streams15; +sql drop database if exists test7; +sql create database test7 vgroups 1; +sql use test7; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams11 trigger at_once into streamt11 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(NULL); +sql create stream streams12 trigger at_once into streamt12 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(value,100.0,200); +sql create stream streams13 trigger at_once into streamt13 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(next); +sql create stream streams14 trigger at_once into streamt14 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(prev); +sql create stream streams15 trigger at_once into streamt15 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(linear); +sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791210001,1,1,1,1.0,'aaa'); + +sql insert into t1 values(1648791215000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791220000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791225000,4,4,4,4.0,'fff'); + +sql insert into t1 values(1648791230000,5,5,5,5.0,'ddd'); +sql insert into t1 values(1648791230001,6,6,6,6.0,'eee'); +sql insert into t1 values(1648791230002,7,7,7,7.0,'fff'); + +$loop_count = 0 + +loop7: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt11 order by ts; + +if $rows != 21 then + print ====streamt11=rows3=$rows + goto loop7 +endi + +sql select * from streamt12 order by ts; + +if $rows != 21 then + print ====streamt12=rows3=$rows + goto loop7 +endi + +sql select * from streamt13 order by ts; + +if $rows != 21 then + print ====streamt13=rows3=$rows + goto loop7 +endi + +sql select * from streamt14 order by ts; + +if $rows != 21 then + print ====streamt14=rows3=$rows + goto loop7 +endi + +sql select * from streamt15 order by ts; + +if $rows != 21 then + print ====streamt15=rows3=$rows + goto loop7 +endi + +sql delete from t1 where ts > 1648791210001 and ts < 1648791230000; + +$loop_count = 0 + +loop8: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt11 order by ts; + +if $rows != 21 then + print ====streamt11=rows3=$rows + goto loop8 +endi + +if $data12 != NULL then + print ====streamt11=3=data01=$data01 + goto loop8 +endi + +if $data[19][2] != NULL then + print ====streamt11=3=data[19][2]=$data[19][2] + goto loop8 +endi + +sql select * from streamt12 order by ts; + +if $rows != 21 then + print ====streamt12=rows3=$rows + goto loop8 +endi + +if $data12 != 200 then + print ====streamt12=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 200 then + print ====streamt12=3=data[19][2]=$data[19][2] + goto loop8 +endi + +sql select * from streamt13 order by ts; + +if $rows != 21 then + print ====streamt13=rows3=$rows + goto loop8 +endi + +if $data12 != 3 then + print ====streamt13=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 3 then + print ====streamt13=3=data[19][2]=$data[19][2] + goto loop8 +endi +sql select * from streamt14 order by ts; + +if $rows != 21 then + print ====streamt14=rows3=$rows + goto loop8 +endi + +if $data12 != 2 then + print ====streamt14=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 2 then + print ====streamt14=3=data[19][2]=$data[19][2] + goto loop8 +endi +sql select * from streamt15 order by ts; +if $rows != 21 then + print ====streamt15=rows3=$rows + goto loop8 +endi +if $data12 != 2 then + print ====streamt15=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 2 then + print ====streamt15=3=data[19][2]=$data[19][2] + goto loop8 +endi @@ -384,6 +546,11 @@ sql drop stream if exists streams7; sql drop stream if exists streams8; sql drop stream if exists streams9; sql drop stream if exists streams10; +sql drop stream if exists streams11; +sql drop stream if exists streams12; +sql drop stream if exists streams13; +sql drop stream if exists streams14; +sql drop stream if exists streams15; sql use test1; sql select * from t1; From 20187bacc6bfd22c65632c8fa84ae99855b2bd70 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:37:43 +0800 Subject: [PATCH 43/89] change parameterr --- tests/system-test/0-others/compatibility.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 17733f4c15..e1ce9e13af 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -98,8 +98,8 @@ class TDTestCase: # tdLog.info(f"Base client version is {oldClientVersion}") oldServerVersion="3.0.1.0" tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") - tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + tdLog.info(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") sleep(3) # tdsqlF.query(f"select count(*) from {stb}") @@ -108,10 +108,10 @@ class TDTestCase: sleep(2) print(f"start taosd: nohup taosd -c {cPath} & ") - os.system(f" nohup taosd -c {cPath} & " ) + os.system(f" nohup /usr/bin/taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") - os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") + os.system("/usr/local/bin/taosBenchmark -f 0-others/compa4096.json -y") os.system("pkill -9 taosd") @@ -134,7 +134,7 @@ class TDTestCase: tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers1) - os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers2) tdsql.query("select count(*) from db4096.stb0") From dcd16684a066a2c6e96463102c2d328846676e91 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:53:24 +0800 Subject: [PATCH 44/89] fix invalid param --- source/libs/transport/src/trans.c | 3 +++ source/libs/transport/src/transCli.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 415a8766e3..c6a5cfdc95 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -63,6 +63,9 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + if (pRpc->numOfThreads <= 0) { + pRpc->numOfThreads = 1; + } uint32_t ip = 0; if (pInit->connType == TAOS_CONN_SERVER) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c967701930..725f352e5c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1476,7 +1476,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - bool retry = pTransInst->retry(code, pResp->msgType - 1); + bool retry = pTransInst->retry != NULL ? pTransInst->retry(code, pResp->msgType - 1) : false; if (retry == false) { return false; } From 47c0d4cbbd6dbe3f1fe257d91792c2c0b7d3b6cd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:58:16 +0800 Subject: [PATCH 45/89] fix invalid param --- source/libs/scheduler/src/schTask.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index af78647018..ed052be784 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -785,6 +785,9 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + taosMemoryFree(origEpset); + taosMemoryFree(newEpset); + memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); return TSDB_CODE_SUCCESS; From 8c3ff153428caaf3d823cb1f34f9b499a26028c1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 18:00:44 +0800 Subject: [PATCH 46/89] fix partition by column --- source/libs/executor/src/executorimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 094060b213..6f6097d258 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1507,7 +1507,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock) { + if (!hasValidBlock && + downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; From 2956eec51ff8863238738bcfe8721320321e59c0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:01:09 +0800 Subject: [PATCH 47/89] fix partition/group by tag count should not have output --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 7 +++++++ source/libs/executor/src/scanoperator.c | 1 + 3 files changed, 9 insertions(+) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8163217039..db642c05c8 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -332,6 +332,7 @@ typedef struct STableScanInfo { int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; + bool hasGroupByTag; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6f6097d258..6df464b06e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1424,6 +1424,13 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION || + (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && + ((STableScanInfo *)downstream->info)->hasGroupByTag == true)) { + return TSDB_CODE_SUCCESS; + } + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; bool hasCountFunc = false; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84b7678b9f..a4e609ae81 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -895,6 +895,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; + pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); From 900fe01f0ed103ed224fd959c54fde3e4f2d8e08 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:32:51 +0800 Subject: [PATCH 48/89] fix error --- source/libs/executor/src/executorimpl.c | 5 ++--- source/libs/function/src/builtinsimpl.c | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6df464b06e..14be6326e0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1425,7 +1425,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc } SOperatorInfo* downstream = pOperator->pDownstream[0]; - if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION || + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && ((STableScanInfo *)downstream->info)->hasGroupByTag == true)) { return TSDB_CODE_SUCCESS; @@ -1514,8 +1514,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock && - downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION) { + if (!hasValidBlock) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index acbddc7128..1881372e34 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3930,12 +3930,16 @@ static void hllTransferInfo(SHLLInfo* pInput, SHLLInfo* pOutput) { int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_SUCCESS; + } SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t start = pInput->startRowIndex; + for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data); From 50c8392dfc09ddb099e73b54251207eed3aa10f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 19:39:09 +0800 Subject: [PATCH 49/89] change parameterr --- tests/system-test/0-others/sysinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 4ddae42ac5..2601c200d9 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -51,7 +51,7 @@ class TDTestCase: sleep(self.delaytime) if platform.system().lower() == 'windows': sleep(10) - tdSql.error('select server_status()') + #tdSql.error('select server_status()') def run(self): self.get_database_info() From f87b9fa7fec257c80203f0c02bef8ee35a21144a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 19:41:03 +0800 Subject: [PATCH 50/89] refactor(sync): add interface syncSnapshotSending, syncSnapshotRecving --- include/libs/sync/sync.h | 5 ++++ source/libs/sync/inc/syncInt.h | 4 +++ source/libs/sync/src/syncMain.c | 46 +++++++++++++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 34 ++++++++++++++++++++---- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index b6d0aecfd1..1b65d95cb1 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -43,6 +43,9 @@ extern "C" { #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 #define SYNC_HEART_TIMEOUT_MS 1000 * 8 +#define SYNC_HEARTBEAT_SLOW_MS 1500 +#define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 + #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 @@ -226,6 +229,8 @@ int32_t syncEndSnapshot(int64_t rid); int32_t syncLeaderTransfer(int64_t rid); int32_t syncStepDown(int64_t rid, SyncTerm newTerm); bool syncIsReadyForRead(int64_t rid); +bool syncSnapshotSending(int64_t rid); +bool syncSnapshotRecving(int64_t rid); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 93ea138c84..d764901ab3 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -195,6 +195,8 @@ typedef struct SSyncNode { int32_t electNum; int32_t becomeLeaderNum; int32_t configChangeNum; + int32_t hbSlowNum; + int32_t hbrSlowNum; bool isStart; @@ -239,6 +241,8 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); +bool syncNodeSnapshotSending(SSyncNode* pSyncNode); +bool syncNodeSnapshotRecving(SSyncNode* pSyncNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 62eba3208e..19a3bd06cf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -447,6 +447,28 @@ bool syncIsReadyForRead(int64_t rid) { return ready; } +bool syncSnapshotSending(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + return false; + } + + bool b = syncNodeSnapshotSending(pSyncNode); + syncNodeRelease(pSyncNode); + return b; +} + +bool syncSnapshotRecving(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + return false; + } + + bool b = syncNodeSnapshotRecving(pSyncNode); + syncNodeRelease(pSyncNode); + return b; +} + int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { if (pSyncNode->peersNum == 0) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); @@ -1013,6 +1035,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->electNum = 0; pSyncNode->becomeLeaderNum = 0; pSyncNode->configChangeNum = 0; + pSyncNode->hbSlowNum = 0; + pSyncNode->hbrSlowNum = 0; sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); @@ -1563,6 +1587,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->leaderCache = EMPTY_RAFT_ID; } + pSyncNode->hbSlowNum = 0; + // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); @@ -1607,6 +1633,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->leaderTime = taosGetTimestampMs(); pSyncNode->becomeLeaderNum++; + pSyncNode->hbrSlowNum = 0; // reset restoreFinish pSyncNode->restoreFinish = false; @@ -2167,6 +2194,25 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { return b; } +bool syncNodeSnapshotSending(SSyncNode* pSyncNode) { + if (pSyncNode == NULL) return false; + bool b = false; + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + if (pSyncNode->senders[i] != NULL && pSyncNode->senders[i]->start) { + b = true; + break; + } + } + return b; +} + +bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) { + if (pSyncNode == NULL) return false; + if (pSyncNode->pNewNodeReceiver == NULL) return false; + if (pSyncNode->pNewNodeReceiver->start) return true; + return false; +} + static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index c022a3e75b..552cf9e7eb 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -283,13 +283,15 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", elt-num:%d, bl-num:%d, cc-num:%d, hit:%d, mis:%d, aq:%d, snaping:%" PRId64 + ", snap-tm:%" PRIu64 + ", elt-num:%d, bl-num:%d, cc-num:%d, hit:%d, mis:%d, hb-slow:%d, hbr-slow:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, pNode->becomeLeaderNum, - pNode->configChangeNum, cacheHit, cacheMiss, aqItems, pNode->snapshottingIndex, pNode->replicaNum, - pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, - pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->configChangeNum, cacheHit, cacheMiss, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, + pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, + pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, + hbTimeStr, hbrTimeStr); } } @@ -463,12 +465,23 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool } void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff) { + if (timeDiff > SYNC_HEARTBEAT_SLOW_MS) { + pSyncNode->hbSlowNum++; + + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sNInfo(pSyncNode, + "recv sync-heartbeat from %s:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 + "}, net elapsed:%" PRId64, + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timeDiff); + } + if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, net elapsed:%" PRId64, @@ -487,6 +500,17 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p } void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff) { + if (timeDiff > SYNC_HEARTBEAT_REPLY_SLOW_MS) { + pSyncNode->hbrSlowNum++; + + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sNTrace(pSyncNode, + "recv sync-heartbeat-reply from %s:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, net elapsed:%" PRId64, host, + port, pMsg->term, pMsg->timeStamp, timeDiff); + } + if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; From ca51b86359acef64d78a7e309571e6dc5044d044 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:45:18 +0800 Subject: [PATCH 51/89] fix test cases --- tests/script/tsim/parser/union.sim | 8 ++-- .../system-test/1-insert/delete_childtable.py | 4 +- tests/system-test/2-query/count.py | 43 ++++++++----------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 8dc19912c8..9e7c6f77cc 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -245,12 +245,12 @@ endi # first subclause are empty sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (select sum(c1) as c from union_tb1); -if $rows != 1 then - return -1 -endi -if $data00 != 495000 then +if $rows != 2 then return -1 endi +#if $data00 != 495000 then +# return -1 +#endi # all subclause are empty sql (select c1 from union_tb0 limit 0) union all (select c1 from union_tb1 where ts>'2021-1-1 0:0:0') diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py index 584e88330c..e3144edb45 100644 --- a/tests/system-test/1-insert/delete_childtable.py +++ b/tests/system-test/1-insert/delete_childtable.py @@ -121,7 +121,7 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(*) from {stbname}') if tb_num <= 1: - if len(tdSql.queryResult) != 0: + if len(tdSql.queryResult) != 1 and tdSql.queryResult[0][0] != 0: tdLog.exit('delete case failure!') else: tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num) @@ -229,4 +229,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 8962cf305a..40d9b3ff8b 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -93,13 +93,17 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') - tdSql.checkRows(2) - tdSql.checkData(0, 0, 0) - tdSql.checkData(1, 0, 0) - tdSql.checkData(0, 1, None) - tdSql.checkData(1, 1, None) + tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by t0') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by t0') + tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -109,15 +113,11 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.stbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 2) - tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.stbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 2) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') tdSql.checkRows(1) @@ -158,9 +158,7 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') @@ -172,15 +170,10 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') tdSql.checkRows(0) - - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 1) - tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.ntbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 1) + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) From b2ed72aeca780e1f156a2cbf87f62a4667ff3ae0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 19:54:06 +0800 Subject: [PATCH 52/89] fix:error in index --- source/client/src/clientRawBlockWrite.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 9db79c6eb6..5583c8ae00 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1330,15 +1330,15 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); }else{ if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; + if (pCol[*index].offset[j] != -1) { + char* data = pCol[*index].pData + pCol[*index].offset[j]; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; + if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { + char* data = pCol[*index].pData + pColumn->bytes * j; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); From 8ec0b424567e731a3f1a5d81eee8e3984e1ac883 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 20:04:04 +0800 Subject: [PATCH 53/89] refactor(sync): modify some trace log to info --- source/libs/sync/src/syncUtil.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 552cf9e7eb..4cd3a53283 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -639,7 +639,7 @@ void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs } void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, int32_t voteGranted, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char logBuf[256]; char host[64]; @@ -647,42 +647,42 @@ void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, i syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); if (voteGranted == -1) { - sNTrace(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + sNInfo(pSyncNode, + "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } else { - sNTrace(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d", - host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); + sNInfo(pSyncNode, + "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); } } void syncLogSendRequestVote(SSyncNode* pNode, const SyncRequestVote* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + sNInfo(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, + pMsg->voteGranted, s); } void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, + pMsg->voteGranted, s); } From 6eb032f3e9fa8a771844f97de6a6417b1a9cf659 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 20:07:26 +0800 Subject: [PATCH 54/89] change test case --- tests/system-test/0-others/sysinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 2601c200d9..9ffa6ca9e6 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -48,10 +48,10 @@ class TDTestCase: tdSql.checkData(0,0,1) #!for bug tdDnodes.stoptaosd(1) - sleep(self.delaytime) + sleep(self.delaytime * 5) if platform.system().lower() == 'windows': sleep(10) - #tdSql.error('select server_status()') + tdSql.error('select server_status()') def run(self): self.get_database_info() From 81e6a9d906e34088ac54274aa30a77c9a3669cf6 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 29 Nov 2022 21:41:07 +0800 Subject: [PATCH 55/89] fix:concurrency conflicts occur when obtaining windows system time --- source/os/src/osTime.c | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 58a09565f9..4634fae24a 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -551,17 +551,26 @@ int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { static SYSTEMTIME ss; static LARGE_INTEGER offset; - ss.wYear = 1970; - ss.wMonth = 1; - ss.wDay = 1; - ss.wHour = 0; - ss.wMinute = 0; - ss.wSecond = 0; - ss.wMilliseconds = 0; - SystemTimeToFileTime(&ss, &ff); - offset.QuadPart = ff.dwHighDateTime; - offset.QuadPart <<= 32; - offset.QuadPart |= ff.dwLowDateTime; + static int8_t offsetInit = 0; + static volatile bool offsetInitFinished = false; + int8_t old = atomic_val_compare_exchange_8(&offsetInit, 0, 1); + if (0 == old) { + ss.wYear = 1970; + ss.wMonth = 1; + ss.wDay = 1; + ss.wHour = 0; + ss.wMinute = 0; + ss.wSecond = 0; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + offsetInitFinished = true; + } else { + while (!offsetInitFinished) + ; // Ensure initialization is completed. + } GetSystemTimeAsFileTime(&f); t.QuadPart = f.dwHighDateTime; From 9278ed2ab5b608cd8b497d7f24517ee91a2c403e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 21:59:49 +0800 Subject: [PATCH 56/89] fix mem leak --- source/libs/scheduler/inc/schInt.h | 8 ++++---- source/libs/scheduler/src/schTask.c | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 83b58a77d2..dfc48e7d9f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -146,7 +146,7 @@ typedef struct SSchedulerMgmt { bool exit; int32_t jobRef; int32_t jobNum; - SSchStat stat; + SSchStat stat; void *timer; SRWLatch hbLock; SHashObj *hbConnections; @@ -214,9 +214,9 @@ typedef struct SSchRedirectCtx { } SSchRedirectCtx; typedef struct SSchTimerParam { - int64_t rId; - uint64_t queryId; - uint64_t taskId; + int64_t rId; + uint64_t queryId; + uint64_t taskId; } SSchTimerParam; typedef struct SSchTask { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ed052be784..9a7f3332b3 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1152,7 +1152,12 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchJob *pJob = NULL; int32_t code = 0; - if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { + int64_t rId = pTimerParam->rId; + uint64_t queryId = pTimerParam->queryId; + uint64_t taskId = pTimerParam->taskId; + taosMemoryFree(pTimerParam); + + if (schProcessOnCbBegin(&pJob, &pTask, queryId, rId, taskId)) { return; } From f913fbdaf656d44f95d0bb9e97f0119c151594a5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 22:14:46 +0800 Subject: [PATCH 57/89] fix errors --- source/libs/function/src/builtinsimpl.c | 3 ++- tests/script/tsim/parser/lastrow_query.sim | 2 +- tests/system-test/2-query/distribute_agg_count.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1881372e34..985f3cad34 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -526,7 +526,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumOfElems(pCtx); + int32_t numOfElem = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -539,6 +539,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { numOfElem = 1; *((int64_t*)buf) = 0; } else { + numOfElem = getNumOfElems(pCtx); *((int64_t*)buf) += numOfElem; } diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index 5f557fd7bd..a1b14c7a0e 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -55,7 +55,7 @@ endi # regression test case 1 sql select count(*) from lr_tb1 where ts>'2018-09-18 08:45:00.1' and ts<'2018-09-18 08:45:00.2' -if $row != 0 then +if $row != 1 then return -1 endi diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index 67d47d7da5..7d131cd77d 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -183,7 +183,7 @@ class TDTestCase: tdSql.checkRows(20) tdSql.query(f"select count(c1) from {dbname}.stb1 where t1> 4 partition by tbname") - tdSql.checkRows(16) + tdSql.checkRows(15) # union all tdSql.query(f"select count(c1) from {dbname}.stb1 union all select count(c1) from {dbname}.stb1 ") From 30fa44ea385cf3dbde06f4d2887dcbaa25845dad Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 22:19:23 +0800 Subject: [PATCH 58/89] chore: update taos-tools and add x86 definition --- cmake/cmake.platform | 2 ++ cmake/taostools_CMakeLists.txt.in | 2 +- include/os/os.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e822cc5e2c..711d74fa4c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -82,10 +82,12 @@ IF ("${CPUTYPE}" STREQUAL "") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(TD_INTEL_64 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") SET(TD_INTEL_32 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8b9104e7e5..79b447e20d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 9f587e9 + GIT_TAG d19e82c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/os/os.h b/include/os/os.h index f4592fc575..0688eeb9ad 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -48,8 +48,10 @@ extern "C" { #else #include #include +#if defined(_TD_X86_) #include #endif +#endif #else #ifndef __func__ From 95677c4ac42788fb824674994f0bb2c161aa6998 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 22:31:21 +0800 Subject: [PATCH 59/89] fix: get cpu id on x86 only --- source/os/src/osSysinfo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f336b84e1e..7ec1da0530 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -481,6 +481,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { #elif defined(_TD_DARWIN_64) #else +#ifdef _TD_X86_ // Since the compiler is not support avx/avx2 instructions, the global variables always need to be // set to be false #if __AVX__ || __AVX2__ @@ -504,6 +505,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { // Ref to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 __cpuid_fix(7u, eax, ebx, ecx, edx); *avx2 = (char) ((ebx & bit_AVX2) == bit_AVX2); +#endif // _TD_X86_ #endif return 0; From cb09f2be939c366b552e2520f4a933bd53363981 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 29 Nov 2022 22:43:34 +0800 Subject: [PATCH 60/89] fix compile --- tests/system-test/0-others/compatibility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index e1ce9e13af..7926812dff 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -132,8 +132,8 @@ class TDTestCase: tdLog.info(f"New client version is {nowClientVersion}") tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") - tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers1) + #tdsql.query(f"select count(*) from {stb}") + #tdsql.checkData(0,0,tableNumbers*recordNumbers1) os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers2) From af30786e4fe8699f895979134ff8514a14769e77 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 22:47:55 +0800 Subject: [PATCH 61/89] fix avg function error --- source/libs/function/src/detail/tavgfunction.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 7d018a8dc7..4cafbd4e6e 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -471,7 +471,6 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - pAvgRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -483,6 +482,8 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { goto _over; } + pAvgRes->type = type; + if (pInput->colDataSMAIsSet) { // try to use SMA if available numOfElem = calculateAvgBySMAInfo(pAvgRes, numOfRows, type, pAgg); } else if (!pCol->hasNull) { // try to employ the simd instructions to speed up the loop @@ -592,6 +593,10 @@ _over: } static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { + if (IS_NULL_TYPE(pInput->type)) { + return; + } + pOutput->type = pInput->type; if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) { pOutput->sum.isum += pInput->sum.isum; @@ -748,4 +753,4 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { taosMemoryFree(res); return pResInfo->numOfRes; -} \ No newline at end of file +} From 548684e5ac06b7e6e0ec9a5046953baa5ac53334 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 29 Nov 2022 23:39:58 +0800 Subject: [PATCH 62/89] fix(stream): delete multiple row --- include/common/tmsg.h | 55 +++++++++---------- include/common/tmsgdef.h | 4 +- source/common/src/tmsg.c | 37 ++++++------- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 13 +++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +-- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 4 +- source/libs/stream/src/streamExec.c | 9 ++- source/libs/stream/src/streamRecover.c | 2 +- source/os/src/osSemaphore.c | 3 +- 11 files changed, 70 insertions(+), 69 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 86967bb579..80590a25c0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1592,14 +1592,14 @@ typedef struct SSubQueryMsg { int8_t explain; int8_t needFetch; uint32_t sqlLen; - char *sql; + char* sql; uint32_t msgLen; - char *msg; + char* msg; } SSubQueryMsg; -int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); -int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); -void tFreeSSubQueryMsg(SSubQueryMsg *pReq); +int32_t tSerializeSSubQueryMsg(void* buf, int32_t bufLen, SSubQueryMsg* pReq); +int32_t tDeserializeSSubQueryMsg(void* buf, int32_t bufLen, SSubQueryMsg* pReq); +void tFreeSSubQueryMsg(SSubQueryMsg* pReq); typedef struct { SMsgHead header; @@ -1638,9 +1638,8 @@ typedef struct { int32_t execId; } SResFetchReq; -int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); -int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); - +int32_t tSerializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq); +int32_t tDeserializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq); typedef struct { SMsgHead header; @@ -1713,12 +1712,11 @@ typedef struct { int32_t execId; } STaskDropReq; -int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); -int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); - -int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); -int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); +int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); typedef struct { int32_t code; @@ -2923,9 +2921,8 @@ typedef struct { STqOffsetVal reqOffset; } SMqPollReq; -int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); -int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); - +int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); +int32_t tDeserializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); typedef struct { int32_t vgId; @@ -3138,7 +3135,8 @@ int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); typedef struct { // int64_t uid; char tbname[TSDB_TABLE_NAME_LEN]; - int64_t ts; + int64_t startTs; + int64_t endTs; } SSingleDeleteReq; int32_t tEncodeSSingleDeleteReq(SEncoder* pCoder, const SSingleDeleteReq* pReq); @@ -3160,8 +3158,8 @@ typedef struct { } SBatchMsg; typedef struct { - SMsgHead header; - SArray* pMsgs; //SArray + SMsgHead header; + SArray* pMsgs; // SArray } SBatchReq; typedef struct { @@ -3173,11 +3171,11 @@ typedef struct { } SBatchRspMsg; typedef struct { - SArray* pRsps; //SArray + SArray* pRsps; // SArray } SBatchRsp; -int32_t tSerializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq); -int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq); +int32_t tSerializeSBatchReq(void* buf, int32_t bufLen, SBatchReq* pReq); +int32_t tDeserializeSBatchReq(void* buf, int32_t bufLen, SBatchReq* pReq); static FORCE_INLINE void tFreeSBatchReqMsg(void* msg) { if (NULL == msg) { return; @@ -3186,8 +3184,8 @@ static FORCE_INLINE void tFreeSBatchReqMsg(void* msg) { taosMemoryFree(pMsg->msg); } -int32_t tSerializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp); -int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp); +int32_t tSerializeSBatchRsp(void* buf, int32_t bufLen, SBatchRsp* pRsp); +int32_t tDeserializeSBatchRsp(void* buf, int32_t bufLen, SBatchRsp* pRsp); static FORCE_INLINE void tFreeSBatchRspMsg(void* p) { if (NULL == p) { @@ -3198,11 +3196,10 @@ static FORCE_INLINE void tFreeSBatchRspMsg(void* p) { taosMemoryFree(pRsp->msg); } -int32_t tSerializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq); -int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq); -int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq); -int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq); - +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); #pragma pack(pop) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 7ca552f179..e80766d249 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -280,8 +280,8 @@ enum { TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cd97ceaae1..c7e98415d1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4496,7 +4496,7 @@ int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq) { if (num <= 0) { pReq->pMsgs = NULL; tEndDecode(&decoder); - + tDecoderClear(&decoder); return 0; } @@ -4511,7 +4511,7 @@ int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq) { if (tDecodeBinaryAlloc(&decoder, &msg.msg, NULL) < 0) return -1; if (NULL == taosArrayPush(pReq->pMsgs, &msg)) return -1; } - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4553,7 +4553,7 @@ int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp) { if (num <= 0) { pRsp->pRsps = NULL; tEndDecode(&decoder); - + tDecoderClear(&decoder); return 0; } @@ -4569,14 +4569,13 @@ int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp) { if (tDecodeBinaryAlloc(&decoder, &msg.msg, NULL) < 0) return -1; if (NULL == taosArrayPush(pRsp->pRsps, &msg)) return -1; } - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4603,7 +4602,7 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->cgroup) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4634,7 +4633,7 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4664,7 +4663,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; - if (tEncodeBinary(&encoder, (uint8_t*)pReq->msg, pReq->msgLen) < 0) return -1; + if (tEncodeBinary(&encoder, (uint8_t *)pReq->msg, pReq->msgLen) < 0) return -1; tEndEncode(&encoder); @@ -4704,8 +4703,8 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; - if (tDecodeBinaryAlloc(&decoder, (void**)&pReq->msg, NULL) < 0) return -1; - + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, NULL) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4721,7 +4720,6 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { taosMemoryFreeClear(pReq->msg); } - int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -4768,14 +4766,13 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) { if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; @@ -4846,14 +4843,13 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -4902,7 +4898,7 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4939,14 +4935,13 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR if (tDecodeI32(&decoder, &pRsp->sversion) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->tversion) < 0) return -1; if (tDecodeI64(&decoder, &pRsp->affectedRows) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -6645,13 +6640,15 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) { int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { if (tEncodeCStr(pEncoder, pReq->tbname) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->startTs) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->endTs) < 0) return -1; return 0; } int32_t tDecodeSSingleDeleteReq(SDecoder *pDecoder, SSingleDeleteReq *pReq) { if (tDecodeCStrTo(pDecoder, pReq->tbname) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->ts) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->startTs) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->endTs) < 0) return -1; return 0; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 61f027039d..3d9ebec4c9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1127,7 +1127,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { SRpcMsg rpcMsg = { .code = 0, .contLen = len, - .msgType = TDMT_VND_STREAM_RECOVER_STEP2, + .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq, }; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index a4bfb6c876..5907be576a 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -21,14 +21,16 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl SBatchDeleteReq* deleteReq) { ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); int32_t totRow = pDataBlock->info.rows; - SColumnInfoData* pTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); tqDebug("stream delete msg: row %d", totRow); for (int32_t row = 0; row < totRow; row++) { - int64_t ts = *(int64_t*)colDataGetData(pTsCol, row); + int64_t startTs = *(int64_t*)colDataGetData(pStartTsCol, row); + int64_t endTs = *(int64_t*)colDataGetData(pEndTsCol, row); int64_t groupId = *(int64_t*)colDataGetData(pGidCol, row); char* name; void* varTbName = NULL; @@ -42,8 +44,8 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl } else { name = buildCtbNameByGroupId(stbFullName, groupId); } - tqDebug("stream delete msg: vgId:%d, groupId :%" PRId64 ", name: %s, ts:%" PRId64, pVnode->config.vgId, groupId, - name, ts); + tqDebug("stream delete msg: vgId:%d, groupId :%" PRId64 ", name: %s, start ts:%" PRId64 "end ts:%" PRId64, + pVnode->config.vgId, groupId, name, startTs, endTs); #if 0 SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); @@ -59,7 +61,8 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl taosMemoryFree(name); #endif SSingleDeleteReq req = { - .ts = ts, + .startTs = startTs, + .endTs = endTs, }; strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); taosMemoryFree(name); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c75d1ffded..2d87bb3b07 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -263,7 +263,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; - case TDMT_VND_STREAM_RECOVER_STEP2: { + case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } @@ -402,7 +402,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); - case TDMT_VND_STREAM_RECOVER_STEP1: + case TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE: return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); case TDMT_STREAM_RECOVER_FINISH: return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); @@ -1184,11 +1184,11 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void int64_t uid = mr.me.uid; - int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->ts, pOneReq->ts); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); if (code < 0) { terrno = code; vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64, - TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->ts, pOneReq->ts); + TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); } tDecoderClear(&mr.coder); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 68228f2689..474a067f9e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1313,8 +1313,8 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + tdbFree(tbname); } - tdbFree(tbname); pBlock->info.rows++; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a6a477a9e3..d598fd09ca 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3117,7 +3117,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (!winInfo.pOutputBuf) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -3242,8 +3242,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + tdbFree(tbname); } - tdbFree(tbname); pBlock->info.rows += 1; } if ((*Ite) == NULL) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 009f7eec9a..6a83a9a4da 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,7 +16,8 @@ #include "streamInc.h" static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - void* exec = pTask->exec.executor; + int32_t code; + void* exec = pTask->exec.executor; // set input const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; @@ -49,8 +50,10 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); + if ((code = qExecTask(exec, &output, &ts)) < 0) { + /*ASSERT(false);*/ + qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, + terrstr()); } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 2a2784afea..7eee95a580 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -36,7 +36,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_STEP1, + .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, }; if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 53d8dad226..bfce8b3151 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,7 +76,8 @@ int32_t tsem_wait(tsem_t* sem) { } int32_t tsem_timewait(tsem_t* sem, int64_t milis) { - return tsem_wait(sem); + return 0; + /*return tsem_wait(sem);*/ #if 0 struct timespec ts; timespec_get(&ts); From f565ff3769644504732f00fa123f8c041911fac1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 23:42:20 +0800 Subject: [PATCH 63/89] fix: get cpuid on x86 only (#18552) * chore: taostools sanitizer flag align with tdengine * chore: re-enable old style build asan config for taos-tools * chore: update taos-tools 274d230 * chore: update taos-tools and add x86 definition * fix: get cpu id on x86 only --- cmake/cmake.platform | 2 ++ cmake/taostools_CMakeLists.txt.in | 2 +- include/os/os.h | 2 ++ source/os/src/osSysinfo.c | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e822cc5e2c..711d74fa4c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -82,10 +82,12 @@ IF ("${CPUTYPE}" STREQUAL "") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(TD_INTEL_64 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") SET(TD_INTEL_32 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8b9104e7e5..79b447e20d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 9f587e9 + GIT_TAG d19e82c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/os/os.h b/include/os/os.h index f4592fc575..0688eeb9ad 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -48,8 +48,10 @@ extern "C" { #else #include #include +#if defined(_TD_X86_) #include #endif +#endif #else #ifndef __func__ diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f336b84e1e..7ec1da0530 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -481,6 +481,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { #elif defined(_TD_DARWIN_64) #else +#ifdef _TD_X86_ // Since the compiler is not support avx/avx2 instructions, the global variables always need to be // set to be false #if __AVX__ || __AVX2__ @@ -504,6 +505,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { // Ref to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 __cpuid_fix(7u, eax, ebx, ecx, edx); *avx2 = (char) ((ebx & bit_AVX2) == bit_AVX2); +#endif // _TD_X86_ #endif return 0; From df75d474d1bf8dbb62c2c5b1428b92007405d104 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Nov 2022 01:32:12 +0800 Subject: [PATCH 64/89] chore: taos-tools support sanitizer workflow --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 79b447e20d..c2ef22c338 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d19e82c + GIT_TAG cf30c86 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From b7cba5d68a3ea03202965f56f3055d5d8f77d89f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 30 Nov 2022 09:35:07 +0800 Subject: [PATCH 65/89] change test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0abec6b340..1c11853864 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -113,7 +113,7 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -,,y,script,./test.sh -f tsim/parser/limit2.sim +#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim From 1bca95bdadd1483d6eebe03a8a2d341cdc81f597 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 30 Nov 2022 09:47:15 +0800 Subject: [PATCH 66/89] fix:modify end ts --- source/libs/executor/src/tfill.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 68228f2689..dd00903ae7 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1427,7 +1427,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { streamStateFreeCur(pCur); pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } - endTs = nextKey.ts - 1; + endTs = TMAX(ts, nextKey.ts - 1); if (code != TSDB_CODE_SUCCESS) { break; } From 2888574be92e4622c99c5158b90044e42e60160a Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 30 Nov 2022 10:51:57 +0800 Subject: [PATCH 67/89] fix compile --- tests/system-test/0-others/compatibility.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 7926812dff..635046e37a 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -110,7 +110,7 @@ class TDTestCase: print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup /usr/bin/taosd -c {cPath} & " ) sleep(10) - tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") + #tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("/usr/local/bin/taosBenchmark -f 0-others/compa4096.json -y") os.system("pkill -9 taosd") @@ -134,7 +134,7 @@ class TDTestCase: tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") #tdsql.query(f"select count(*) from {stb}") #tdsql.checkData(0,0,tableNumbers*recordNumbers1) - os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers2) tdsql.query("select count(*) from db4096.stb0") From b6d3d2322ccee0bef8d9c32d78a7c63cbf33be77 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 11:02:07 +0800 Subject: [PATCH 68/89] fix: cannot close vnode while transferring snapshot --- source/dnode/vnode/src/vnd/vnodeSync.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6c205b5522..ad6f153aca 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -515,6 +515,16 @@ void vnodeSyncPreClose(SVnode *pVnode) { vInfo("vgId:%d, pre close sync", pVnode->config.vgId); syncLeaderTransfer(pVnode->sync); syncPreStop(pVnode->sync); + + while (!syncSnapshotRecving(pVnode->sync)) { + vInfo("vgId:%s, snapshot is recving"); + taosMsleep(300); + } + while (!syncSnapshotSending(pVnode->sync)) { + vInfo("vgId:%s, snapshot is sending"); + taosMsleep(300); + } + taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { vInfo("vgId:%d, post block after close sync", pVnode->config.vgId); From 82bc543ed7d08ae28955a32216cc62c534e09811 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 11:05:25 +0800 Subject: [PATCH 69/89] fix: cannot close mnode while transferring snapshot --- source/dnode/mnode/impl/src/mndMain.c | 9 +++++++++ source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 2d888d2ff7..1b74b00dac 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -491,6 +491,15 @@ void mndPreClose(SMnode *pMnode) { if (pMnode != NULL) { syncLeaderTransfer(pMnode->syncMgmt.sync); syncPreStop(pMnode->syncMgmt.sync); + + while (!syncSnapshotRecving(pMnode->syncMgmt.sync)) { + mInfo("vgId:1, snapshot is recving"); + taosMsleep(300); + } + while (!syncSnapshotSending(pMnode->syncMgmt.sync)) { + mInfo("vgId:1, snapshot is sending"); + taosMsleep(300); + } } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ad6f153aca..5ae9813bef 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -517,11 +517,11 @@ void vnodeSyncPreClose(SVnode *pVnode) { syncPreStop(pVnode->sync); while (!syncSnapshotRecving(pVnode->sync)) { - vInfo("vgId:%s, snapshot is recving"); + vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId); taosMsleep(300); } while (!syncSnapshotSending(pVnode->sync)) { - vInfo("vgId:%s, snapshot is sending"); + vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId); taosMsleep(300); } From 5ecbcb10544ae59b9d279cfa63f918e1ce421620 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 11:07:14 +0800 Subject: [PATCH 70/89] fix: cannot close mnode while transferring snapshot --- source/dnode/mnode/impl/src/mndMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1b74b00dac..a88fb61e0c 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -492,11 +492,11 @@ void mndPreClose(SMnode *pMnode) { syncLeaderTransfer(pMnode->syncMgmt.sync); syncPreStop(pMnode->syncMgmt.sync); - while (!syncSnapshotRecving(pMnode->syncMgmt.sync)) { + while (syncSnapshotRecving(pMnode->syncMgmt.sync)) { mInfo("vgId:1, snapshot is recving"); taosMsleep(300); } - while (!syncSnapshotSending(pMnode->syncMgmt.sync)) { + while (syncSnapshotSending(pMnode->syncMgmt.sync)) { mInfo("vgId:1, snapshot is sending"); taosMsleep(300); } From 3850b58a3d9a783a45b55ff1d4faa9fa2ddc2ac8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 11:08:30 +0800 Subject: [PATCH 71/89] fix: cannot close vnode while transferring snapshot --- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5ae9813bef..dd382411c1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -516,11 +516,11 @@ void vnodeSyncPreClose(SVnode *pVnode) { syncLeaderTransfer(pVnode->sync); syncPreStop(pVnode->sync); - while (!syncSnapshotRecving(pVnode->sync)) { + while (syncSnapshotRecving(pVnode->sync)) { vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId); taosMsleep(300); } - while (!syncSnapshotSending(pVnode->sync)) { + while (syncSnapshotSending(pVnode->sync)) { vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId); taosMsleep(300); } From a8832e881498eaa1a6f81ef42600ad8718aa48c1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 30 Nov 2022 11:20:03 +0800 Subject: [PATCH 72/89] refactor(sync): modify info log --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncMain.c | 1 + source/libs/sync/src/syncTimeout.c | 8 +++++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index d764901ab3..145880fa3e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -197,6 +197,7 @@ typedef struct SSyncNode { int32_t configChangeNum; int32_t hbSlowNum; int32_t hbrSlowNum; + int32_t tmrRoutineNum; bool isStart; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 19a3bd06cf..3117f6e4c9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1037,6 +1037,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->configChangeNum = 0; pSyncNode->hbSlowNum = 0; pSyncNode->hbrSlowNum = 0; + pSyncNode->tmrRoutineNum = 0; sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index ca67eb1880..0b58f1080f 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -52,7 +52,13 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { } static int32_t syncNodeTimerRoutine(SSyncNode* ths) { - sNInfo(ths, "timer routines"); + ths->tmrRoutineNum++; + + if (ths->tmrRoutineNum % 60 == 0 && ths->replicaNum > 1) { + sNInfo(ths, "timer routines"); + } else { + sNTrace(ths, "timer routines"); + } // timer replicate syncNodeReplicate(ths); From 3b33b4dadbbfb5ba42962151d2204521da513398 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 30 Nov 2022 11:37:50 +0800 Subject: [PATCH 73/89] change parameterr --- tests/system-test/0-others/compatibility.py | 44 ++++++++++----------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 7926812dff..30513ac020 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -72,8 +72,9 @@ class TDTestCase: def buildTaosd(self,bPath): - print(f" build path {bPath} ") - os.system(f" cd {bPath} && cmake -DBUILD_TOOLS=on .. && make -j4 && make install") + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} && make install ") + def run(self): bPath=self.getBuildPath() @@ -85,21 +86,21 @@ class TDTestCase: tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 - # tdsqlF=tdCom.newTdSql() - # print(tdsqlF) - # tdsqlF.query(f"SELECT SERVER_VERSION();") - # print(tdsqlF.query(f"SELECT SERVER_VERSION();")) - # oldServerVersion=tdsqlF.queryResult[0][0] - # tdLog.info(f"Base server version is {oldServerVersion}") - # tdsqlF.query(f"SELECT CLIENT_VERSION();") + tdsqlF=tdCom.newTdSql() + print(tdsqlF) + tdsqlF.query(f"SELECT SERVER_VERSION();") + print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + oldServerVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base server version is {oldServerVersion}") + tdsqlF.query(f"SELECT CLIENT_VERSION();") - # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison - # oldClientVersion=tdsqlF.queryResult[0][0] - # tdLog.info(f"Base client version is {oldClientVersion}") - oldServerVersion="3.0.1.0" + # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + oldClientVersion=tdsqlF.queryResult[0][0] + tdLog.info(f"Base client version is {oldClientVersion}") + tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") - tdLog.info(f"taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") sleep(3) # tdsqlF.query(f"select count(*) from {stb}") @@ -108,10 +109,10 @@ class TDTestCase: sleep(2) print(f"start taosd: nohup taosd -c {cPath} & ") - os.system(f" nohup /usr/bin/taosd -c {cPath} & " ) + os.system(f" nohup taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") - os.system("/usr/local/bin/taosBenchmark -f 0-others/compa4096.json -y") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("pkill -9 taosd") @@ -132,14 +133,11 @@ class TDTestCase: tdLog.info(f"New client version is {nowClientVersion}") tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") - #tdsql.query(f"select count(*) from {stb}") - #tdsql.checkData(0,0,tableNumbers*recordNumbers1) - os.system(f"/usr/local/bin/taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") tdsql.query(f"select count(*) from {stb}") tdsql.checkData(0,0,tableNumbers*recordNumbers2) - tdsql.query("select count(*) from db4096.stb0") - tdsql.checkData(0,0,50000) - tdsql=tdCom.newTdSql() tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") From 5556fd0c45194cc755e4fda8e47df7d62c3f32c6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 12:02:54 +0800 Subject: [PATCH 74/89] enh: add sync offline state --- include/libs/sync/sync.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 6 +++--- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- source/libs/sync/src/syncMain.c | 6 +++++- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1b65d95cb1..604e1479d5 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -67,6 +67,7 @@ typedef struct SWal SWal; typedef struct SSyncRaftEntry SSyncRaftEntry; typedef enum { + TAOS_SYNC_STATE_OFFLINE = 0, TAOS_SYNC_STATE_FOLLOWER = 100, TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 8a8561161b..a7ad983b0c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -150,7 +150,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); - if (minfo.isMnode && minfo.load.syncState == TAOS_SYNC_STATE_ERROR) { + if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; @@ -160,7 +160,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { (*pMgmt->getVnodeLoadsFp)(&vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); - if (pLoad->syncState == TAOS_SYNC_STATE_ERROR) { + if (pLoad->syncState == TAOS_SYNC_STATE_ERROR || pLoad->syncState == TAOS_SYNC_STATE_OFFLINE) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "vnode:%d sync state is %s", pLoad->vgId, syncStr(pLoad->syncState)); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a88fb61e0c..f533fafeee 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -151,10 +151,10 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) bool roleChanged = false; for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) { - if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_ERROR) { + if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_OFFLINE) { mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0", pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore); - pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_ERROR; + pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_OFFLINE; pVgroup->vnodeGid[vg].syncRestore = 0; roleChanged = true; } @@ -756,7 +756,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(desc.status, "ready", sizeof(desc.status)); pClusterInfo->vgroups_alive++; } - if (pVgid->syncState != TAOS_SYNC_STATE_ERROR) { + if (pVgid->syncState != TAOS_SYNC_STATE_ERROR && pVgid->syncState != TAOS_SYNC_STATE_OFFLINE) { pClusterInfo->vnodes_alive++; } pClusterInfo->vnodes_total++; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b02ca4c773..7c86a5be22 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -185,7 +185,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { return -1; } - pObj->syncState = TAOS_SYNC_STATE_ERROR; + pObj->syncState = TAOS_SYNC_STATE_OFFLINE; mndReloadSyncConfig(pSdb->pMnode); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4fa00dcf7c..dda0d7d26a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -887,7 +887,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { } pVgid->dnodeId = pDnode->id; - pVgid->syncState = TAOS_SYNC_STATE_ERROR; + pVgid->syncState = TAOS_SYNC_STATE_OFFLINE; mInfo("db:%s, vgId:%d, vn:%d is added, memory:%" PRId64 ", dnode:%d avail:%" PRId64 " used:%" PRId64, pVgroup->dbName, pVgroup->vgId, pVgroup->replica, vgMem, pVgid->dnodeId, pDnode->memAvail, pDnode->memUsed); @@ -1193,7 +1193,7 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica]; pVgroup->replica++; pGid->dnodeId = newDnodeId; - pGid->syncState = TAOS_SYNC_STATE_ERROR; + pGid->syncState = TAOS_SYNC_STATE_OFFLINE; for (int32_t i = 0; i < pVgroup->replica - 1; ++i) { if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 19a3bd06cf..7a72b89078 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2457,8 +2457,12 @@ const char* syncStr(ESyncState state) { return "candidate"; case TAOS_SYNC_STATE_LEADER: return "leader"; - default: + case TAOS_SYNC_STATE_ERROR: return "error"; + case TAOS_SYNC_STATE_OFFLINE: + return "offline"; + default: + return "unknown"; } } From 12191a46f7a2906c98c739cd506f6b12cfea0a0c Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 30 Nov 2022 12:05:53 +0800 Subject: [PATCH 75/89] test: modify limit2.sim to limit.py --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/limit.py | 353 +++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 tests/system-test/2-query/limit.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1df0d5ef71..6c5c1718b2 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -529,6 +529,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/limit.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py new file mode 100644 index 0000000000..91719aa21b --- /dev/null +++ b/tests/system-test/2-query/limit.py @@ -0,0 +1,353 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 2 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + # tdLog.debug(colString) + # tdLog.debug(tagString) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)"%(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + # t = time.time() + # startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + # print("===sql: %s"%(sql)) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + # print("===sql: %s"%(sql)) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'lm2_db0', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'lm2_stb0', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 'lm2_tb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'lm2_db0', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'lm2_stb0', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'FLOAT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'smallint', 'count':1},{'type': 'tinyint', 'count':1},{'type': 'bool', 'count':1},{'type': 'binary', 'len':10, 'count':1},{'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 'lm2_tb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + val1 = 1 + val2 = paraDict["ctbNum"] - 1 + # select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 + sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(1) + tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) + tdSql.checkData(0, 1, 2) + tdSql.checkData(0, 2, "tb2") + tdSql.checkData(0, 3, "tb2") + tdSql.checkData(0, 4, 2) + tdSql.checkData(0, 5, 2) + + # select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0 + sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"]) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(2) + tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) + tdSql.checkData(0, 1, "tb4") + tdSql.checkData(0, 2, 4) + + tdSql.checkData(1, 1, "tb3") + tdSql.checkData(1, 2, 3) + + # select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1 + sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"]) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(1) + + ## TBASE-348 + tdSql.error(f"select count(*) from %s where t1 like 1"%(paraDict["stbName"])) + + ts0 = paraDict["startTs"] + delta = paraDict["tsStep"] + tsu = paraDict["rowsPerTbl"] * delta + tsu = tsu - delta + tsu = tsu + ts0 + tb = paraDict["ctbPrefix"] + '0' + # select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1 + sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(10) + tdSql.checkData(0, 0, "18-09-17 09:05:00.000") + tdSql.checkData(0, 1, -1) + tdSql.checkData(1, 1, 1) + tdSql.checkData(9, 0, "18-09-17 09:50:00.000") + tdSql.checkData(9, 1, 5) + + tb5 = paraDict["ctbPrefix"] + '5' + sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(10) + tdSql.checkData(0, 0, -1) + tdSql.checkData(0, 1, -2) + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, -2) + tdSql.checkData(9, 0, 5) + tdSql.checkData(9, 1, -2) + + ### [TBASE-350] + ## tb + interval + fill(value) + limit offset + tb = paraDict["ctbPrefix"] + '0' + limit = paraDict["rowsPerTbl"] + offset = limit / 2 + sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(limit) + tdSql.checkData(0, 1, 0) + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(8200) + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(10) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, -1000) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, -1000) + + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(10) + tdSql.checkData(0, 0, -1000) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, -1000) + tdSql.checkData(3, 0, 2) + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(9998) + + + sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(0) + + # tb + interval + fill(linear) + limit offset + limit = paraDict["rowsPerTbl"] + offset = limit / 2 + sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(limit) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 0) + tdSql.checkData(1, 3, 0.500000000) + tdSql.checkData(3, 5, 0.000000000) + tdSql.checkData(4, 6, 0.000000000) + tdSql.checkData(4, 7, 1) + tdSql.checkData(5, 7, None) + tdSql.checkData(6, 8, "binary3") + tdSql.checkData(7, 9, None) + + + + limit = paraDict["rowsPerTbl"] + offset = limit / 2 + sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(limit) + + + limit = paraDict["rowsPerTbl"] + offset = limit / 2 + 10 + sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(limit) + tdSql.checkData(0, 1, 5) + tdSql.checkData(0, 2, 5) + tdSql.checkData(0, 3, 5.000000000) + tdSql.checkData(0, 4, 5.000000000) + tdSql.checkData(0, 5, 0.000000000) + tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 8, "binary5") + tdSql.checkData(0, 9, "nchar5") + tdSql.checkData(1, 8, None) + tdSql.checkData(1, 9, None) + + + limit = paraDict["rowsPerTbl"] + offset = limit * 2 - 11 + sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(10) + tdSql.checkData(0, 1, -1) + tdSql.checkData(0, 2, -2) + tdSql.checkData(1, 1, 5) + tdSql.checkData(1, 2, 5) + tdSql.checkData(1, 3, 5.000000000) + tdSql.checkData(1, 5, 0.000000000) + tdSql.checkData(1, 6, 0.000000000) + tdSql.checkData(1, 8, "binary5") + tdSql.checkData(1, 9, "nchar5") + + ### [TBASE-350] + ## stb + interval + fill + group by + limit offset + sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10" + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(2) + + limit = 5 + offset = paraDict["rowsPerTbl"] * 2 + offset = offset - 2 + sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset) + print("====sql:%s"%(sqlStr)) + tdSql.query(sqlStr) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9) + tdSql.checkData(0, 1, 9) + tdSql.checkData(0, 2, 9.000000000) + tdSql.checkData(0, 3, 9.000000000) + tdSql.checkData(0, 4, 1) + tdSql.checkData(0, 5, "binary9") + tdSql.checkData(0, 6, "nchar9") + + #add one more test case + sqlStr = f"select max(c1), last(c8) from lm2_db0.lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(linear) limit 10 offset 4089;" + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From e45023f3e98c869c1e6dc1525b573552ad03281f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 12:23:08 +0800 Subject: [PATCH 76/89] fix(query): add an null ptr checking before copying memory. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4ba311212a..26c9ce6810 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -775,7 +775,10 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); - memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 + memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); + } + colDataAppend(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false); } } else { From b29f1634c032634190e8168c820a65246d90dd69 Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Wed, 30 Nov 2022 12:48:36 +0800 Subject: [PATCH 77/89] docs:Update 02-intro.md Update 02-intro.md 19k star >> 20k star --- docs/zh/02-intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 47bfd3f96b..525c8aa5c9 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -69,7 +69,7 @@ TDengine 的主要功能如下： - **[分析能力](https://www.taosdata.com/tdengine/easy_data_analytics)**：通过超级表、存储计算分离、分区分片、预计算和其它技术，TDengine 能够高效地浏览、格式化和访问数据。 -- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 19k，且拥有一个活跃的开发者社区。 +- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 20k，且拥有一个活跃的开发者社区。 采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面： From 76cc7a89fb3db1132ebb4448a2204b8c10af59ec Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 30 Nov 2022 13:44:22 +0800 Subject: [PATCH 78/89] refacotr(sync), force stop snapshot sender/receiver when stop node --- source/libs/sync/src/syncMain.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3117f6e4c9..44abf3c5e9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1142,12 +1142,21 @@ void syncNodeClose(SSyncNode* pSyncNode) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] != NULL) { sSTrace((pSyncNode->senders)[i], "snapshot sender destroy while close, data:%p", (pSyncNode->senders)[i]); + + if (snapshotSenderIsStart((pSyncNode->senders)[i])) { + snapshotSenderStop((pSyncNode->senders)[i], false); + } + snapshotSenderDestroy((pSyncNode->senders)[i]); (pSyncNode->senders)[i] = NULL; } } if (pSyncNode->pNewNodeReceiver != NULL) { + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + } + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); pSyncNode->pNewNodeReceiver = NULL; } From 7e9b7f4e150caee6c58bbdf4320e3d482c90f82a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 14:06:22 +0800 Subject: [PATCH 79/89] enh: show more info while drop dnode --- source/common/src/systable.c | 2 + source/dnode/mnode/impl/inc/mndVgroup.h | 2 - source/dnode/mnode/impl/src/mndVgroup.c | 172 +++++++++++++++--------- 3 files changed, 108 insertions(+), 68 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index c3a1f9f67e..1829101639 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -212,6 +212,8 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v4_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "v4_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 94469899fc..765d4fd33d 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -36,8 +36,6 @@ int64_t mndGetVgroupMemory(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId); int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups); -int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray); -int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid); int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid); int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index dda0d7d26a..dd2b985c14 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -179,6 +179,16 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { pOld->hashEnd = pNew->hashEnd; pOld->replica = pNew->replica; pOld->isTsma = pNew->isTsma; + for (int32_t i = 0; i < pNew->replica; ++i) { + SVnodeGid *pNewGid = &pNew->vnodeGid[i]; + for (int32_t j = 0; j < pOld->replica; ++j) { + SVnodeGid *pOldGid = &pOld->vnodeGid[j]; + if (pNewGid->dnodeId == pOldGid->dnodeId) { + pNewGid->syncState = pOldGid->syncState; + pNewGid->syncRestore = pOldGid->syncRestore; + } + } + } memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } @@ -659,8 +669,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false); - // default 3 replica - for (int32_t i = 0; i < 3; ++i) { + // default 3 replica, add 1 replica if move vnode + for (int32_t i = 0; i < 4; ++i) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); @@ -851,7 +861,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { +static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); @@ -893,6 +903,15 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { pVgroup->replica++; pDnode->numOfVnodes++; + + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + return 0; } @@ -901,7 +920,8 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { +static int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, SArray *pArray, + SVnodeGid *pDelVgid) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); @@ -941,6 +961,15 @@ _OVER: SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + return 0; } @@ -1088,7 +1117,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (!force) { mInfo("vgId:%d, will add 1 vnode", pVgroup->vgId); - if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; for (int32_t i = 0; i < newVg.replica - 1; ++i) { if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; } @@ -1100,6 +1129,16 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVnodeGid del = newVg.vnodeGid[vnIndex]; newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; for (int32_t i = 0; i < newVg.replica; ++i) { if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; @@ -1107,11 +1146,20 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; } else { mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId); - if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1; + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; newVg.replica--; SVnodeGid del = newVg.vnodeGid[vnIndex]; newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } for (int32_t i = 0; i < newVg.replica; ++i) { if (i != vnIndex) { @@ -1128,16 +1176,12 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1; + if (pRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; - } - - { - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; } mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); @@ -1195,6 +1239,14 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb pGid->dnodeId = newDnodeId; pGid->syncState = TAOS_SYNC_STATE_OFFLINE; + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + for (int32_t i = 0; i < pVgroup->replica - 1; ++i) { if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; } @@ -1224,6 +1276,14 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid)); memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + sdbFreeRaw(pVgRaw); + return -1; + } + (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; for (int32_t i = 0; i < pVgroup->replica; ++i) { if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, pVgroup, pVgroup->vnodeGid[i].dnodeId) != 0) return -1; @@ -1236,9 +1296,8 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1, SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3, SDnodeObj *pOld3) { - int32_t code = -1; - SSdbRaw *pRaw = NULL; - STrans *pTrans = NULL; + int32_t code = -1; + STrans *pTrans = NULL; pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "red-vgroup"); if (pTrans == NULL) goto _OVER; @@ -1319,17 +1378,13 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, } { - pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; - } - - { - pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; } mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica); @@ -1342,7 +1397,6 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, _OVER: mndTransDrop(pTrans); - sdbFreeRaw(pRaw); mndReleaseDb(pMnode, pDb); return code; } @@ -1593,13 +1647,13 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) @@ -1612,7 +1666,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb SVnodeGid del1 = {0}; SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; + if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del1) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; @@ -1620,7 +1674,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVgroup, pArray, &del2) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &del2, true) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; @@ -1629,16 +1683,6 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb return -1; } - { - SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); - if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { - sdbFreeRaw(pVgRaw); - return -1; - } - (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); - } - { SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); if (pVgRaw == NULL) return -1; @@ -1657,10 +1701,9 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, } static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { - int32_t code = -1; - SSdbRaw *pRaw = NULL; - STrans *pTrans = NULL; - SArray *pArray = mndBuildDnodesArray(pMnode, 0); + int32_t code = -1; + STrans *pTrans = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode, 0); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; @@ -1676,13 +1719,13 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj } if (newVg1.replica == 1) { - if (mndAddVnodeToVgroup(pMnode, &newVg1, pArray) != 0) goto _OVER; + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER; + if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; @@ -1727,17 +1770,23 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj #endif { - pRaw = mndVgroupActionEncode(&newVg1); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg1); + if (pRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; } { - pRaw = mndVgroupActionEncode(&newVg2); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg2); + if (pRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - pRaw = NULL; } mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId, @@ -1757,7 +1806,6 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj _OVER: taosArrayDestroy(pArray); mndTransDrop(pTrans); - sdbFreeRaw(pRaw); return code; } @@ -1802,16 +1850,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD { SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; - } - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - } - - { - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) { + if (pRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { sdbFreeRaw(pRaw); return -1; } From 4c21f20118e0a1767df71247d25976e14aac04e3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Wed, 30 Nov 2022 14:42:31 +0800 Subject: [PATCH 80/89] fix compile --- tests/system-test/0-others/compatibility.py | 254 ++++++++++---------- 1 file changed, 129 insertions(+), 125 deletions(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 30513ac020..377cdbd7e6 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -15,155 +15,159 @@ from util.cluster import * class TDTestCase: - def caseDescription(self): - ''' - 3.0 data compatibility test - case1: basedata version is 3.0.1.0 - ''' - return + def caseDescription(self): + ''' + 3.0 data compatibility test + case1: basedata version is 3.0.1.0 + ''' + return - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] - for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath - def getCfgPath(self): - buildPath = self.getBuildPath() - selfPath = os.path.dirname(os.path.realpath(__file__)) + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - cfgPath = buildPath + "/../sim/dnode1/cfg/" - else: - cfgPath = buildPath + "/../sim/dnode1/cfg/" + if ("community" in selfPath): + cfgPath = buildPath + "/../sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" - return cfgPath + return cfgPath - def installTaosd(self,bPath,cPath): - # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") - # os.system(f" mv {bPath}/build {bPath}/build_bak ") - # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") - # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") - packagePath="/usr/local/src/" - packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" - os.system(f"cd {packagePath} && tar xvf TDengine-server-3.0.1.0-Linux-x64.tar.gz && cd TDengine-server-3.0.1.0 && ./install.sh -e no " ) - tdDnodes.stop(1) - print(f"start taosd: nohup taosd -c {cPath} & ") - os.system(f" nohup taosd -c {cPath} & " ) - sleep(1) + packagePath="/usr/local/src/" + packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz" + os.system(f"cd {packagePath} && tar xvf TDengine-server-3.0.1.0-Linux-x64.tar.gz && cd TDengine-server-3.0.1.0 && ./install.sh -e no " ) + tdDnodes.stop(1) + print(f"start taosd: nohup taosd -c {cPath} & ") + os.system(f" nohup taosd -c {cPath} & " ) + sleep(1) - - def buildTaosd(self,bPath): - # os.system(f"mv {bPath}/build_bak {bPath}/build ") - os.system(f" cd {bPath} && make install ") + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} && make install ") - def run(self): - bPath=self.getBuildPath() - cPath=self.getCfgPath() - dbname = "test" - stb = f"{dbname}.meters" - self.installTaosd(bPath,cPath) - os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") - tableNumbers=100 - recordNumbers1=100 - recordNumbers2=1000 - tdsqlF=tdCom.newTdSql() - print(tdsqlF) - tdsqlF.query(f"SELECT SERVER_VERSION();") - print(tdsqlF.query(f"SELECT SERVER_VERSION();")) - oldServerVersion=tdsqlF.queryResult[0][0] - tdLog.info(f"Base server version is {oldServerVersion}") - tdsqlF.query(f"SELECT CLIENT_VERSION();") - - # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison - oldClientVersion=tdsqlF.queryResult[0][0] - tdLog.info(f"Base client version is {oldClientVersion}") + def run(self): + print(f"start taosd run") + bPath=self.getBuildPath() + cPath=self.getCfgPath() + dbname = "test" + stb = f"{dbname}.meters" + self.installTaosd(bPath,cPath) + os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + #tdsqlF=tdCom.newTdSql() + #print(tdsqlF) - tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") - tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") - sleep(3) + oldServerVersion = '3.0.1.0' + #tdsqlF.query(f"SELECT SERVER_VERSION();") + #print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + #oldServerVersion=tdsqlF.queryResult[0][0] + #tdLog.info(f"Base server version is {oldServerVersion}") + #tdsqlF.query(f"SELECT CLIENT_VERSION();") + # + ## the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + #oldClientVersion=tdsqlF.queryResult[0][0] - # tdsqlF.query(f"select count(*) from {stb}") - # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) - os.system("pkill taosd") - sleep(2) + #tdLog.info(f"Base client version is {oldClientVersion}") - print(f"start taosd: nohup taosd -c {cPath} & ") - os.system(f" nohup taosd -c {cPath} & " ) - sleep(10) - tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") - os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("pkill -9 taosd") + tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{oldServerVersion}") + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") + sleep(3) + + # tdsqlF.query(f"select count(*) from {stb}") + # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) + os.system("pkill taosd") + sleep(2) + + print(f"start taosd: nohup taosd -c {cPath} & ") + os.system(f" nohup taosd -c {cPath} & " ) + sleep(10) + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") + os.system("pkill -9 taosd") - tdLog.printNoPrefix("==========step2:update new version ") - self.buildTaosd(bPath) - tdDnodes.start(1) - sleep(1) - tdsql=tdCom.newTdSql() - print(tdsql) + tdLog.printNoPrefix("==========step2:update new version ") + self.buildTaosd(bPath) + tdDnodes.start(1) + sleep(1) + tdsql=tdCom.newTdSql() + print(tdsql) - tdsql.query(f"SELECT SERVER_VERSION();") - nowServerVersion=tdsql.queryResult[0][0] - tdLog.info(f"New server version is {nowServerVersion}") - tdsql.query(f"SELECT CLIENT_VERSION();") - nowClientVersion=tdsql.queryResult[0][0] - tdLog.info(f"New client version is {nowClientVersion}") + tdsql.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql.queryResult[0][0] + tdLog.info(f"New server version is {nowServerVersion}") + tdsql.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql.queryResult[0][0] + tdLog.info(f"New client version is {nowClientVersion}") - tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") - tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers1) - os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") - tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers2) + tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) + os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers2) - tdsql=tdCom.newTdSql() - tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") - tdsql.execute("drop database if exists db") - tdsql.execute("create database db") - tdsql.execute("use db") - tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") - tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") - tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") - tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") - tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);") - tdsql.query("select * from db.ct3") - tdsql.checkData(0,1,13) - tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") - tdsql.query("select * from db.ct4") - tdsql.checkData(0,1,14) - tdsql.query("describe information_schema.ins_databases;") - qRows=tdsql.queryRows - for i in range(qRows) : - if tdsql.queryResult[i][0]=="retentions" : - return True - else: - return False - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") + tdsql=tdCom.newTdSql() + tdLog.printNoPrefix(f"==========step4:verify backticks in taos Sql-TD18542") + tdsql.execute("drop database if exists db") + tdsql.execute("create database db") + tdsql.execute("use db") + tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") + tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") + tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);") + tdsql.query("select * from db.ct3") + tdsql.checkData(0,1,13) + tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") + tdsql.query("select * from db.ct4") + tdsql.checkData(0,1,14) + tdsql.query("describe information_schema.ins_databases;") + qRows=tdsql.queryRows + for i in range(qRows) : + if tdsql.queryResult[i][0]=="retentions" : + return True + else: + return False + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) From 4ea7d139ba22b506d7c302d0ce1b6f841b23f545 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Nov 2022 14:48:39 +0800 Subject: [PATCH 81/89] fix(query): fix avg calculation error after SIMD optimize TD-20803 --- .../libs/function/src/detail/tavgfunction.c | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 4cafbd4e6e..e8bd5f8d3c 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -502,7 +502,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { i8VectorSumAVX2(plist, numOfRows, type, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.usum += plist[i]; + if (type == TSDB_DATA_TYPE_TINYINT) { + pAvgRes->sum.isum += plist[i]; + } else { + pAvgRes->sum.usum += (uint8_t)plist[i]; + } } } break; @@ -517,7 +521,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { i16VectorSumAVX2(plist, numOfRows, type, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.isum += plist[i]; + if (type == TSDB_DATA_TYPE_SMALLINT) { + pAvgRes->sum.isum += plist[i]; + } else { + pAvgRes->sum.usum += (uint16_t)plist[i]; + } } } break; @@ -532,7 +540,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { i32VectorSumAVX2(plist, numOfRows, type, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.isum += plist[i]; + if (type == TSDB_DATA_TYPE_INT) { + pAvgRes->sum.isum += plist[i]; + } else { + pAvgRes->sum.usum += (uint32_t)plist[i]; + } } } break; @@ -547,7 +559,11 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { i64VectorSumAVX2(plist, numOfRows, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - pAvgRes->sum.isum += plist[i]; + if (type == TSDB_DATA_TYPE_BIGINT) { + pAvgRes->sum.isum += plist[i]; + } else { + pAvgRes->sum.isum += (uint64_t)plist[i]; + } } } break; From b30c56bf7c27874f6a3d7507e39f28d7248ba02c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 30 Nov 2022 14:00:30 +0800 Subject: [PATCH 82/89] fix:stream scan handles empty datablcok --- source/libs/executor/src/scanoperator.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a4e609ae81..3c4d888528 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1052,6 +1052,9 @@ static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, } static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { + if (pBlock->info.rows == 0) { + return false; + } if ((*pRowIndex) == pBlock->info.rows) { return false; } @@ -1184,10 +1187,10 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 } static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + blockDataCleanup(pDestBlock); if (pSrcBlock->info.rows == 0) { return TSDB_CODE_SUCCESS; } - blockDataCleanup(pDestBlock); int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1837,6 +1840,12 @@ FETCH_NEXT_BLOCK: } setBlockGroupIdByUid(pInfo, pDelBlock); printDataBlock(pDelBlock, "stream scan delete recv filtered"); + if (pDelBlock->info.rows == 0) { + if (pInfo->tqReader) { + blockDataDestroy(pDelBlock); + } + goto FETCH_NEXT_BLOCK; + } if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes); pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; From 4048988908b906ccbdd5dd324d5eac005112a051 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Nov 2022 14:48:39 +0800 Subject: [PATCH 83/89] fix(query): fix avg calculation error after SIMD optimize TD-20803 --- .../libs/function/src/detail/tavgfunction.c | 93 +++++++++++++------ 1 file changed, 63 insertions(+), 30 deletions(-) diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index e8bd5f8d3c..f06bafafe3 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -133,6 +133,14 @@ static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; + } } else { const uint8_t* p = (const uint8_t*)plist; @@ -142,16 +150,16 @@ static void i8VectorSumAVX2(const int8_t* plist, int32_t numOfRows, int32_t type sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const uint64_t* q = (const uint64_t*)∑ + pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.usum += (uint8_t)plist[j + rounds * width]; + } } - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + startIndex]; - } #endif } @@ -176,8 +184,16 @@ static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t ty sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; + } } else { - const uint8_t* p = (const uint8_t*)plist; + const uint16_t* p = (const uint16_t*)plist; for(int32_t i = 0; i < rounds; ++i) { __m128i val = _mm_lddqu_si128((__m128i*)p); @@ -185,16 +201,16 @@ static void i16VectorSumAVX2(const int16_t* plist, int32_t numOfRows, int32_t ty sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const uint64_t* q = (const uint64_t*)∑ + pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.usum += (uint16_t)plist[j + rounds * width]; + } } - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + startIndex]; - } #endif } @@ -219,6 +235,14 @@ static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t ty sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; + } } else { const uint32_t* p = (const uint32_t*)plist; @@ -228,16 +252,16 @@ static void i32VectorSumAVX2(const int32_t* plist, int32_t numOfRows, int32_t ty sum = _mm256_add_epi64(sum, extVal); p += width; } + + // let sum up the final results + const uint64_t* q = (const uint64_t*)∑ + pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.usum += (uint32_t)plist[j + rounds * width]; + } } - // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + startIndex]; - } #endif } @@ -262,13 +286,22 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p } // let sum up the final results - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + if (type == TSDB_DATA_TYPE_BIGINT) { + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - int32_t startIndex = rounds * width; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + startIndex]; + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; + } + } else { + const uint64_t* q = (const uint64_t*)∑ + pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; + + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.usum += (uint64_t)plist[j + rounds * width]; + } } + #endif } From d58640dd5318ed406e5a1194ecd62e00edf17f4f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 30 Nov 2022 17:15:23 +0800 Subject: [PATCH 84/89] enh: show more info while drop dnode --- source/common/src/systable.c | 19 ++++++++----------- source/dnode/mnode/impl/src/mndVgroup.c | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 1829101639..1255116d0c 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -206,18 +206,15 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "v4_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "v4_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "v1_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "v2_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v3_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "v3_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, }; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index dd2b985c14..d06853e470 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -189,6 +189,12 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { } } } + pNew->numOfTables = pOld->numOfTables; + pNew->numOfTimeSeries = pOld->numOfTimeSeries; + pNew->totalStorage = pOld->totalStorage; + pNew->compStorage = pOld->compStorage; + pNew->pointsWritten = pOld->pointsWritten; + pNew->compact = pOld->compact; memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } @@ -673,7 +679,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p for (int32_t i = 0; i < 4; ++i) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); + int16_t dnodeId = (int16_t)pVgroup->vnodeGid[i].dnodeId; + colDataAppend(pColInfo, numOfRows, (const char *)&dnodeId, false); bool exist = false; bool online = false; @@ -705,16 +712,8 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppendNULL(pColInfo, numOfRows); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppendNULL(pColInfo, numOfRows); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppendNULL(pColInfo, numOfRows); + int32_t cacheUsage = (int32_t)pVgroup->cacheUsage; + colDataAppend(pColInfo, numOfRows, (const char *)&cacheUsage, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); From 00b6ad284c604790b882040639eaddc912056bc3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Nov 2022 17:26:14 +0800 Subject: [PATCH 85/89] fix: taosbenchmark printf format (#18569) * feat: taosbenchmark supports retry * fix: taosbenchmark printf format --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index c2ef22c338..47f2d20828 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG cf30c86 + GIT_TAG b103d9b SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 124fb5fc5076163d091f5036b3e610d8cf8f6dd5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 17:52:12 +0800 Subject: [PATCH 86/89] refactor: do some internal refactor. --- include/util/tarray.h | 15 ++------------- source/dnode/mnode/impl/src/mndConsumer.c | 5 +++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + source/libs/stream/src/streamUpdate.c | 7 ++++++- source/util/src/tarray.c | 9 +-------- 5 files changed, 13 insertions(+), 24 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index beebd70dea..0e78397ecb 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -220,15 +220,10 @@ void taosArrayClear(SArray* pArray); */ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); -/** - * clear the array (remove all element) - * @param pArray - * @param fp - */ -void taosArrayClearP(SArray* pArray, FDelete fp); - void* taosArrayDestroy(SArray* pArray); + void taosArrayDestroyP(SArray* pArray, FDelete fp); + void taosArrayDestroyEx(SArray* pArray, FDelete fp); /** @@ -238,12 +233,6 @@ void taosArrayDestroyEx(SArray* pArray, FDelete fp); */ void taosArraySort(SArray* pArray, __compar_fn_t comparFn); -/** - * sort string array - * @param pArray - */ -void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); - /** * search the array * @param pArray diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 58f8172282..4397ea0751 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -538,7 +538,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *newSub = subscribe.topicNames; - taosArraySortString(newSub, taosArrayCompareString); + taosArraySort(newSub, taosArrayCompareString); taosArrayRemoveDuplicateP(newSub, taosArrayCompareString, taosMemoryFree); int32_t newTopicNum = taosArrayGetSize(newSub); @@ -850,7 +850,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // add to current topic taosArrayPush(pOldConsumer->currentTopics, &addedTopic); - taosArraySortString(pOldConsumer->currentTopics, taosArrayCompareString); + taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); + // set status if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { if (pOldConsumer->status == MQ_CONSUMER_STATUS__MODIFY || diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 26c9ce6810..83532c2a20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2543,6 +2543,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* goto _err; } + // TODO: opt the perf of read del index code = tsdbReadDelIdx(pDelFReader, aDelIdx); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(aDelIdx); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 1589fddda4..1ce4a35dff 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -44,6 +44,11 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } } +static void clearItemHelper(void* p) { + SScalableBf** pBf = p; + tScalableBfDestroy(*pBf); +} + static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { if (count < pInfo->numSBFs) { for (uint64_t i = 0; i < count; ++i) { @@ -52,7 +57,7 @@ static void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count) { taosArrayRemove(pInfo->pTsSBFs, 0); } } else { - taosArrayClearP(pInfo->pTsSBFs, (FDelete)tScalableBfDestroy); + taosArrayClearEx(pInfo->pTsSBFs, clearItemHelper); } pInfo->minTS += pInfo->interval * count; } diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index f53b1cfd7f..ae25786375 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -399,9 +399,7 @@ void taosArrayDestroyEx(SArray* pArray, FDelete fp) { } void taosArraySort(SArray* pArray, __compar_fn_t compar) { - assert(pArray != NULL); - assert(compar != NULL); - + ASSERT(pArray != NULL && compar != NULL); taosSort(pArray->pData, pArray->size, pArray->elemSize, compar); } @@ -417,11 +415,6 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t return item == NULL ? -1 : (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; } -void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { - assert(pArray != NULL); - taosSort(pArray->pData, pArray->size, pArray->elemSize, comparFn); -} - static int32_t taosArrayPartition(SArray* pArray, int32_t i, int32_t j, __ext_compar_fn_t fn, const void* userData) { void* key = taosArrayGetP(pArray, i); while (i < j) { From 26122eb51a224808bcbf6ad011c64d7a1179318a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Nov 2022 19:47:39 +0800 Subject: [PATCH 87/89] fix: benchmark query result precision (#18581) * fix: taosbenchmark query result precision * test: update to f5da335 * test: update taos-tools 96192d4 * test: update taos-tools c6df1eb * test: update taostools 2378cdf * fix: taosbenchmark query result precision --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 47f2d20828..e0585f8d13 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG b103d9b + GIT_TAG 5445810 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7497eab926e054b0058f39ac49a21481d7c9c4cc Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 30 Nov 2022 20:32:43 +0800 Subject: [PATCH 88/89] refactor(sync): if already start, force close snapshot receiver --- source/libs/sync/src/syncSnapshot.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 222b7c4e1e..3722b0f6f5 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -541,6 +541,10 @@ _START_RECEIVER: taosMsleep(10); } + if (snapshotReceiverIsStart(pReceiver)) { + snapshotReceiverForceStop(pReceiver); + } + snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender } From 2c8b962b5a1391d450afdb6d3d45d2e5591804fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 21:04:58 +0800 Subject: [PATCH 89/89] refactor: do some internal refactor. --- include/libs/executor/executor.h | 7 - source/libs/executor/inc/executil.h | 2 + source/libs/executor/inc/executorimpl.h | 24 +- source/libs/executor/inc/tfill.h | 16 +- source/libs/executor/src/exchangeoperator.c | 11 - source/libs/executor/src/executil.c | 10 + source/libs/executor/src/executor.c | 14 + source/libs/executor/src/executorimpl.c | 349 ---- source/libs/executor/src/filloperator.c | 1494 +++++++++++++++++ source/libs/executor/src/scanoperator.c | 24 - source/libs/executor/src/tfill.c | 1113 +----------- source/libs/executor/src/timewindowoperator.c | 12 +- 12 files changed, 1535 insertions(+), 1541 deletions(-) create mode 100644 source/libs/executor/src/filloperator.c diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0bca254e14..d210004760 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -160,13 +160,6 @@ int32_t qAsyncKillTask(qTaskInfo_t tinfo); */ void qDestroyTask(qTaskInfo_t tinfo); -/** - * Get the queried table uid - * @param qHandle - * @return - */ -int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); - /** * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. * diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 875528576d..51150ede3c 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -161,4 +161,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +void printDataBlock(SSDataBlock* pBlock, const char* flag); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8163217039..8fc3401813 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -537,23 +537,6 @@ typedef struct SStreamIntervalOperatorInfo { SWinKey delKey; } SStreamIntervalOperatorInfo; -typedef struct SFillOperatorInfo { - struct SFillInfo* pFillInfo; - SSDataBlock* pRes; - SSDataBlock* pFinalRes; - int64_t totalInputRows; - void** p; - SSDataBlock* existNewGroupBlock; - STimeWindow win; - SColMatchInfo matchInfo; - int32_t primaryTsCol; - int32_t primarySrcSlotId; - uint64_t curGroupId; // current handled group id - SExprInfo* pExprInfo; - int32_t numOfExpr; - SExprSupp noFillExprSupp; -} SFillOperatorInfo; - typedef struct SDataGroupInfo { uint64_t groupId; int64_t numOfRows; @@ -805,8 +788,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo); - int32_t getMaximumIdleDurationSec(); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, @@ -824,9 +805,8 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); -void printDataBlock(SSDataBlock* pBlock, const char* flag); -uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); +void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index b0017fef50..fad4059515 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -25,6 +25,8 @@ extern "C" { #include "tcommon.h" #include "tsimplehash.h" +#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) + struct SSDataBlock; typedef struct SFillColInfo { @@ -113,12 +115,12 @@ typedef struct SStreamFillInfo { int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); -void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, - int32_t numOfNotFillCols, const struct SNodeListNode* val); -bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); +void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, + int32_t numOfNotFillCols, const struct SNodeListNode* val); +bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNotFillCols, int32_t capacity, SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, int32_t slotId, @@ -128,6 +130,8 @@ void* taosDestroyFillInfo(struct SFillInfo* pFillInfo); int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t getFillInfoStart(struct SFillInfo* pFillInfo); +bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, + int32_t rowIndex); #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index b2ddff45a4..963a273290 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -15,26 +15,15 @@ #include "filter.h" #include "function.h" -#include "functionMgt.h" #include "os.h" -#include "querynodes.h" -#include "tfill.h" #include "tname.h" #include "tref.h" - #include "tdatablock.h" -#include "tglobal.h" #include "tmsg.h" -#include "tsort.h" -#include "ttime.h" - #include "executorimpl.h" #include "index.h" #include "query.h" -#include "tcompare.h" #include "thash.h" -#include "ttypes.h" -#include "vnode.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b135566caa..8a85885d98 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2002,3 +2002,13 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return TSDB_CODE_SUCCESS; } + +void printDataBlock(SSDataBlock* pBlock, const char* flag) { + if (!pBlock || pBlock->info.rows == 0) { + qDebug("===stream===printDataBlock: Block is Null or Empty"); + return; + } + char* pBuf = NULL; + qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); + taosMemoryFree(pBuf); +} \ No newline at end of file diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 34bd9cf8ca..10ceb9ccee 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -704,6 +704,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) { return TSDB_CODE_SUCCESS; } +static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { + STaskCostInfo* pSummary = &pTaskInfo->cost; + + SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; + if (pSummary->pRecoder != NULL) { + qDebug( + "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, + pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, + pRecorder->totalCheckedRows); + } +} + void qDestroyTask(qTaskInfo_t qTaskHandle) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; if (pTaskInfo == NULL) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 300ba52934..f85287b0ae 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -91,7 +91,6 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* static void releaseQueryBuf(size_t numOfTables); -static void destroyFillOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -1157,20 +1156,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { - STaskCostInfo* pSummary = &pTaskInfo->cost; - - SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; - if (pSummary->pRecoder != NULL) { - qDebug( - "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " - "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, - pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, - pRecorder->totalCheckedRows); - } -} - // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // @@ -1513,179 +1498,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pInfo->pRes; } -static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { - pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - SSDataBlock* pResBlock = pInfo->pFinalRes; - - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; - taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); - - blockDataCleanup(pInfo->pRes); - doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); - taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); - - int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; - taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows); - - pInfo->curGroupId = pInfo->existNewGroupBlock->info.id.groupId; - pInfo->existNewGroupBlock = NULL; -} - -static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { - if (taosFillHasMoreResults(pInfo->pFillInfo)) { - int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; - taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); - pInfo->pRes->info.id.groupId = pInfo->curGroupId; - return; - } - - // handle the cached new group data block - if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); - } -} - -static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { - SFillOperatorInfo* pInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - setInputDataBlock(pSup, pBlock, order, scanFlag, false); - projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL); - - // reset the row value before applying the no-fill functions to the input data block, which is "pBlock" in this case. - pInfo->pRes->info.rows = 0; - SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; - setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false); - - projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL); - pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; -} - -static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { - SFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SResultInfo* pResultInfo = &pOperator->resultInfo; - SSDataBlock* pResBlock = pInfo->pFinalRes; - - blockDataCleanup(pResBlock); - - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag); - - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows > 0) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; - while (1) { - SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); - if (pBlock == NULL) { - if (pInfo->totalInputRows == 0) { - setOperatorCompleted(pOperator); - return NULL; - } - - taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); - } else { - blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); - - blockDataCleanup(pInfo->pRes); - blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); - blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); - doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); - - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { - pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block - pInfo->totalInputRows += pInfo->pRes->info.rows; - - if (order == pInfo->pFillInfo->order) { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } - taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); - } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block - pInfo->existNewGroupBlock = pBlock; - - // Fill the previous group data block, before handle the data block of new group. - // Close the fill operation for previous group data block - taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); - } - } - - int32_t numOfResultRows = pOperator->resultInfo.capacity - pResBlock->info.rows; - taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows); - - // current group has no more result to return - if (pResBlock->info.rows > 0) { - // 1. The result in current group not reach the threshold of output result, continue - // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately - if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || pInfo->existNewGroupBlock != NULL) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - } else if (pInfo->existNewGroupBlock) { // try next group - assert(pBlock != NULL); - - blockDataCleanup(pResBlock); - - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows > pResultInfo->threshold) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - } else { - return NULL; - } - } -} - -static SSDataBlock* doFill(SOperatorInfo* pOperator) { - SFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSDataBlock* fillResult = NULL; - while (true) { - fillResult = doFillImpl(pOperator); - if (fillResult == NULL) { - setOperatorCompleted(pOperator); - break; - } - - doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); - if (fillResult->info.rows > 0) { - break; - } - } - - if (fillResult != NULL) { - pOperator->resultInfo.totalRows += fillResult->info.rows; - } - - return fillResult; -} - void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { for (int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExprInfo = &pExpr[i]; @@ -1955,167 +1767,6 @@ void destroyAggOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void destroyFillOperatorInfo(void* param) { - SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; - pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes); - - cleanupExprSupp(&pInfo->noFillExprSupp); - - taosMemoryFreeClear(pInfo->p); - taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFreeClear(param); -} - -static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, - int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, - const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); - - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, - pInfo->primaryTsCol, order, id); - - if (order == TSDB_ORDER_ASC) { - pInfo->win.skey = win.skey; - pInfo->win.ekey = win.ekey; - } else { - pInfo->win.skey = win.ekey; - pInfo->win.ekey = win.skey; - } - pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - - if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { - taosMemoryFree(pInfo->pFillInfo); - taosMemoryFree(pInfo->p); - return TSDB_CODE_OUT_OF_MEMORY; - } else { - return TSDB_CODE_SUCCESS; - } -} - -static bool isWstartColumnExist(SFillOperatorInfo* pInfo) { - if (pInfo->noFillExprSupp.numOfExprs == 0) { - return false; - } - - for (int32_t i = 0; i < pInfo->noFillExprSupp.numOfExprs; ++i) { - SExprInfo* exprInfo = pInfo->noFillExprSupp.pExprInfo + i; - if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && exprInfo->base.numOfParams == 1 && - exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { - return true; - } - } - return false; -} - -static int32_t createPrimaryTsExprIfNeeded(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode, SExprSupp* pExprSupp, - const char* idStr) { - bool wstartExist = isWstartColumnExist(pInfo); - - if (wstartExist == false) { - if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) { - qError("pWStartTs of fill physical node is not a target node, %s", idStr); - return TSDB_CODE_QRY_SYS_ERROR; - } - - SExprInfo* pExpr = taosMemoryRealloc(pExprSupp->pExprInfo, (pExprSupp->numOfExprs + 1) * sizeof(SExprInfo)); - if (pExpr == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - createExprFromTargetNode(&pExpr[pExprSupp->numOfExprs], (STargetNode*)pPhyFillNode->pWStartTs); - pExprSupp->numOfExprs += 1; - pExprSupp->pExprInfo = pExpr; - } - - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, - SExecTaskInfo* pTaskInfo) { - SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); - pOperator->exprSupp.pExprInfo = pExprInfo; - - SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; - pNoFillSupp->pExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->numOfExprs); - int32_t code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - SInterval* pInterval = - QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType - ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval - : &((SIntervalAggOperatorInfo*)downstream->info)->interval; - - int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - int32_t type = convertFillType(pPhyFillNode->mode); - - SResultInfo* pResultInfo = &pOperator->resultInfo; - - initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; - pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; - - int32_t numOfOutputCols = 0; - code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, - COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - - code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, - (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, - pTaskInfo->id.str, pInterval, type, order); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false); - blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity); - - code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - if (pInfo != NULL) { - destroyFillOperatorInfo(pInfo); - } - - pTaskInfo->code = code; - taosMemoryFreeClear(pOperator); - return NULL; -} - static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c new file mode 100644 index 0000000000..e26bfe9889 --- /dev/null +++ b/source/libs/executor/src/filloperator.c @@ -0,0 +1,1494 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "os.h" +#include "query.h" +#include "taosdef.h" +#include "tmsg.h" +#include "ttypes.h" + +#include "executorimpl.h" +#include "tcommon.h" +#include "thash.h" +#include "ttime.h" + +#include "executorInt.h" +#include "function.h" +#include "querynodes.h" +#include "tdatablock.h" +#include "tfill.h" + +#define FILL_POS_INVALID 0 +#define FILL_POS_START 1 +#define FILL_POS_MID 2 +#define FILL_POS_END 3 + +typedef struct STimeRange { + TSKEY skey; + TSKEY ekey; + uint64_t groupId; +} STimeRange; + +typedef struct SFillOperatorInfo { + struct SFillInfo* pFillInfo; + SSDataBlock* pRes; + SSDataBlock* pFinalRes; + int64_t totalInputRows; + void** p; + SSDataBlock* existNewGroupBlock; + STimeWindow win; + SColMatchInfo matchInfo; + int32_t primaryTsCol; + int32_t primarySrcSlotId; + uint64_t curGroupId; // current handled group id + SExprInfo* pExprInfo; + int32_t numOfExpr; + SExprSupp noFillExprSupp; +} SFillOperatorInfo; + +static void destroyFillOperatorInfo(void* param); +static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); + +static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, + SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; + SSDataBlock* pResBlock = pInfo->pFinalRes; + + int32_t order = TSDB_ORDER_ASC; + int32_t scanFlag = MAIN_SCAN; + getTableScanInfo(pOperator, &order, &scanFlag); + + int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; + taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); + + blockDataCleanup(pInfo->pRes); + doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); + + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); + + int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; + taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows); + + pInfo->curGroupId = pInfo->existNewGroupBlock->info.id.groupId; + pInfo->existNewGroupBlock = NULL; +} + +static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, + SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + if (taosFillHasMoreResults(pInfo->pFillInfo)) { + int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; + taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); + pInfo->pRes->info.id.groupId = pInfo->curGroupId; + return; + } + + // handle the cached new group data block + if (pInfo->existNewGroupBlock) { + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + } +} + +void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { + SFillOperatorInfo* pInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + setInputDataBlock(pSup, pBlock, order, scanFlag, false); + projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, NULL); + + // reset the row value before applying the no-fill functions to the input data block, which is "pBlock" in this case. + pInfo->pRes->info.rows = 0; + SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; + setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false); + + projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL); + pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; +} + +static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { + SFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SResultInfo* pResultInfo = &pOperator->resultInfo; + SSDataBlock* pResBlock = pInfo->pFinalRes; + + blockDataCleanup(pResBlock); + + int32_t order = TSDB_ORDER_ASC; + int32_t scanFlag = MAIN_SCAN; + getTableScanInfo(pOperator, &order, &scanFlag); + + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + if (pResBlock->info.rows > 0) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + + SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + while (1) { + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); + if (pBlock == NULL) { + if (pInfo->totalInputRows == 0) { + setOperatorCompleted(pOperator); + return NULL; + } + + taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); + } else { + blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); + + blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); + doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); + + if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { + pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block + pInfo->totalInputRows += pInfo->pRes->info.rows; + + if (order == pInfo->pFillInfo->order) { + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); + } else { + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); + } + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); + } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block + pInfo->existNewGroupBlock = pBlock; + + // Fill the previous group data block, before handle the data block of new group. + // Close the fill operation for previous group data block + taosFillSetStartInfo(pInfo->pFillInfo, 0, pInfo->win.ekey); + } + } + + int32_t numOfResultRows = pOperator->resultInfo.capacity - pResBlock->info.rows; + taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows); + + // current group has no more result to return + if (pResBlock->info.rows > 0) { + // 1. The result in current group not reach the threshold of output result, continue + // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately + if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || pInfo->existNewGroupBlock != NULL) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + } else if (pInfo->existNewGroupBlock) { // try next group + assert(pBlock != NULL); + + blockDataCleanup(pResBlock); + + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + if (pResBlock->info.rows > pResultInfo->threshold) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + } else { + return NULL; + } + } +} + +static SSDataBlock* doFill(SOperatorInfo* pOperator) { + SFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* fillResult = NULL; + while (true) { + fillResult = doFillImpl(pOperator); + if (fillResult == NULL) { + setOperatorCompleted(pOperator); + break; + } + + doFilter(fillResult, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + if (fillResult->info.rows > 0) { + break; + } + } + + if (fillResult != NULL) { + pOperator->resultInfo.totalRows += fillResult->info.rows; + } + + return fillResult; +} + +void destroyFillOperatorInfo(void* param) { + SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; + pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); + pInfo->pRes = blockDataDestroy(pInfo->pRes); + pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes); + + cleanupExprSupp(&pInfo->noFillExprSupp); + + taosMemoryFreeClear(pInfo->p); + taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFreeClear(param); +} + +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, + int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, + const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); + + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); + w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + + pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, + pInfo->primaryTsCol, order, id); + + if (order == TSDB_ORDER_ASC) { + pInfo->win.skey = win.skey; + pInfo->win.ekey = win.ekey; + } else { + pInfo->win.skey = win.ekey; + pInfo->win.ekey = win.skey; + } + pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); + + if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { + taosMemoryFree(pInfo->pFillInfo); + taosMemoryFree(pInfo->p); + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; + } +} + +static bool isWstartColumnExist(SFillOperatorInfo* pInfo) { + if (pInfo->noFillExprSupp.numOfExprs == 0) { + return false; + } + + for (int32_t i = 0; i < pInfo->noFillExprSupp.numOfExprs; ++i) { + SExprInfo* exprInfo = pInfo->noFillExprSupp.pExprInfo + i; + if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && exprInfo->base.numOfParams == 1 && + exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { + return true; + } + } + return false; +} + +static int32_t createPrimaryTsExprIfNeeded(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode, SExprSupp* pExprSupp, + const char* idStr) { + bool wstartExist = isWstartColumnExist(pInfo); + + if (wstartExist == false) { + if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) { + qError("pWStartTs of fill physical node is not a target node, %s", idStr); + return TSDB_CODE_QRY_SYS_ERROR; + } + + SExprInfo* pExpr = taosMemoryRealloc(pExprSupp->pExprInfo, (pExprSupp->numOfExprs + 1) * sizeof(SExprInfo)); + if (pExpr == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + createExprFromTargetNode(&pExpr[pExprSupp->numOfExprs], (STargetNode*)pPhyFillNode->pWStartTs); + pExprSupp->numOfExprs += 1; + pExprSupp->pExprInfo = pExpr; + } + + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo) { + SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); + pOperator->exprSupp.pExprInfo = pExprInfo; + + SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; + pNoFillSupp->pExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->numOfExprs); + int32_t code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SInterval* pInterval = + QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType + ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval + : &((SIntervalAggOperatorInfo*)downstream->info)->interval; + + int32_t order = (pPhyFillNode->inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + int32_t type = convertFillType(pPhyFillNode->mode); + + SResultInfo* pResultInfo = &pOperator->resultInfo; + + initResultSizeInfo(&pOperator->resultInfo, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; + pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; + + int32_t numOfOutputCols = 0; + code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, + COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + + code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, + (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, + pTaskInfo->id.str, pInterval, type, order); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false); + blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity); + + code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + if (pInfo != NULL) { + destroyFillOperatorInfo(pInfo); + } + + pTaskInfo->code = code; + taosMemoryFreeClear(pOperator); + return NULL; +} + +TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + return win.skey; +} + +TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { + STimeWindow win = {.skey = ts, .ekey = ts}; + getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + return win.skey; +} + +void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + colDataAppend(pCol, rowId, pCell->pData, pCell->isNull); +} + +SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { + if (!pRaw || !pRaw->pRowVal) { + return NULL; + } + char* pData = (char*)pRaw->pRowVal; + SResultCellData* pCell = pRaw->pRowVal; + for (int32_t i = 0; i < index; i++) { + pData += (pCell->bytes + sizeof(SResultCellData)); + pCell = (SResultCellData*)pData; + } + return pCell; +} + +void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { + for (int32_t i = start; i < end; i++) { + destroyExprInfo(pFillCol[i].pExpr, 1); + taosVariantDestroy(&pFillCol[i].fillVal); + } + taosMemoryFreeClear(pFillCol[start].pExpr); + taosMemoryFree(pFillCol); + return NULL; +} + +void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { + pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); + tSimpleHashCleanup(pFillSup->pResMap); + pFillSup->pResMap = NULL; + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + pFillSup->cur.pRowVal = NULL; + cleanupExprSupp(&pFillSup->notFillExprSup); + + taosMemoryFree(pFillSup); + return NULL; +} + +void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { + taosArrayDestroy(pFillLinear->pDeltaVal); + taosArrayDestroy(pFillLinear->pNextDeltaVal); + taosMemoryFree(pFillLinear); + return NULL; +} +void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_NULL) { + taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); + taosMemoryFreeClear(pFillInfo->pResRow); + } + pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + taosArrayDestroy(pFillInfo->delRanges); + taosMemoryFree(pFillInfo); + return NULL; +} + +static void destroyStreamFillOperatorInfo(void* param) { + SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; + pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); + pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + pInfo->pRes = blockDataDestroy(pInfo->pRes); + pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); + pInfo->pPrevSrcBlock = blockDataDestroy(pInfo->pPrevSrcBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFree(pInfo); +} + +static void resetFillWindow(SResultRowData* pRowData) { + pRowData->key = INT64_MIN; + pRowData->pRowVal = NULL; +} + +void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { + resetFillWindow(&pFillSup->prev); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); + resetFillWindow(&pFillSup->cur); + resetFillWindow(&pFillSup->next); + resetFillWindow(&pFillSup->nextNext); +} + +void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState); + + SWinKey key = {.ts = ts, .groupId = groupId}; + int32_t curVLen = 0; + int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; +} + +void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { + SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; + resetPrevAndNextWindow(pFillSup, pState); + + SWinKey key = {.ts = ts, .groupId = groupId}; + void* curVal = NULL; + int32_t curVLen = 0; + int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + ASSERT(code == TSDB_CODE_SUCCESS); + pFillSup->cur.key = key.ts; + pFillSup->cur.pRowVal = curVal; + + SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); + SWinKey preKey = {.groupId = groupId}; + void* preVal = NULL; + int32_t preVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); + + if (code == TSDB_CODE_SUCCESS) { + pFillSup->prev.key = preKey.ts; + pFillSup->prev.pRowVal = preVal; + + code = streamStateCurNext(pState, pCur); + ASSERT(code == TSDB_CODE_SUCCESS); + + code = streamStateCurNext(pState, pCur); + if (code != TSDB_CODE_SUCCESS) { + streamStateFreeCur(pCur); + pCur = NULL; + } + } else { + streamStateFreeCur(pCur); + pCur = streamStateFillSeekKeyNext(pState, &key); + } + + SWinKey nextKey = {.groupId = groupId}; + void* nextVal = NULL; + int32_t nextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->next.key = nextKey.ts; + pFillSup->next.pRowVal = nextVal; + if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { + code = streamStateCurNext(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; + } + } + } + } + streamStateFreeCur(pCur); +} + +static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } +static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } +static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { + return pFillSup->nextNext.key != INT64_MIN; + return false; +} + +static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); + SResultCellData* pCell = getResultCell(pRowVal, i); + if (!colDataIsNull_s(pColData, rowId)) { + pCell->isNull = false; + pCell->type = pColData->info.type; + pCell->bytes = pColData->info.bytes; + char* val = colDataGetData(pColData, rowId); + if (IS_VAR_DATA_TYPE(pCell->type)) { + memcpy(pCell->pData, val, varDataTLen(val)); + } else { + memcpy(pCell->pData, val, pCell->bytes); + } + } else { + pCell->isNull = true; + } + } + pRowVal->key = ts; +} + +static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, + SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + char* var = colDataGetData(pCol, rowId); + double start = 0; + GET_TYPED_DATA(start, double, pCol->info.type, var); + SResultCellData* pCell = getResultCell(pRowVal, slotId); + double end = 0; + GET_TYPED_DATA(end, double, pCell->type, pCell->pData); + double delta = 0; + if (order == TSDB_ORDER_ASC) { + delta = (end - start) / winCount; + } else { + delta = (start - end) / winCount; + } + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void calcRowDeltaData(SResultRowData* pStartRow, SResultRowData* pEndRow, SArray* pDelta, SFillColInfo* pFillCol, + int32_t numOfCol, int32_t winCount) { + for (int32_t i = 0; i < numOfCol; i++) { + if (!pFillCol[i].notFillCol) { + int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); + SResultCellData* pSCell = getResultCell(pStartRow, slotId); + double start = 0.0; + GET_TYPED_DATA(start, double, pSCell->type, pSCell->pData); + SResultCellData* pECell = getResultCell(pEndRow, slotId); + double end = 0.0; + GET_TYPED_DATA(end, double, pECell->type, pECell->pData); + double delta = (end - start) / winCount; + taosArraySet(pDelta, slotId, &delta); + } + } +} + +static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + pFillInfo->start = ts; +} + +static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); + pFillInfo->end = ts; +} + +static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { + setFillInfoStart(start, pInterval, pFillInfo); + pFillInfo->current = pFillInfo->start; + setFillInfoEnd(end, pInterval, pFillInfo); +} + +void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + return; + } + + TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + + pFillInfo->needFill = true; + pFillInfo->start = realStart; + pFillInfo->current = pFillInfo->start; + pFillInfo->end = end; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_SET_VALUE: + break; + case TSDB_FILL_PREV: + pFillInfo->pResRow = &pFillSup->prev; + break; + case TSDB_FILL_NEXT: + pFillInfo->pResRow = &pFillSup->next; + break; + case TSDB_FILL_LINEAR: { + setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(pFillSup->prev.key, pFillSup->next.key, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->winIndex = 0; + } break; + default: + ASSERT(0); + break; + } +} + +void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { + for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); + pCell->isNull = pCurCell->isNull; + if (!pCurCell->isNull) { + memcpy(pCell->pData, pCurCell->pData, pCell->bytes); + } + } +} + +void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, + SStreamFillInfo* pFillInfo) { + pFillInfo->preRowKey = pFillSup->cur.key; + if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + return; + } + TSKEY prevWKey = INT64_MIN; + TSKEY nextWKey = INT64_MIN; + if (hasPrevWindow(pFillSup)) { + prevWKey = pFillSup->prev.key; + } + if (hasNextWindow(pFillSup)) { + nextWKey = pFillSup->next.key; + } + + pFillInfo->needFill = true; + pFillInfo->pos = FILL_POS_INVALID; + switch (pFillInfo->type) { + case TSDB_FILL_NULL: + case TSDB_FILL_SET_VALUE: { + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + if (pFillSup->next.key == pFillInfo->nextRowKey) { + pFillInfo->preRowKey = INT64_MIN; + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + pFillInfo->needFill = false; + pFillInfo->pos = FILL_POS_START; + } + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + } else { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + copyNotFillExpData(pFillSup, pFillInfo); + } break; + case TSDB_FILL_PREV: { + if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || + (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || + (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillSup->prev.key = pFillSup->cur.key; + pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->preRowKey = INT64_MIN; + } + pFillInfo->pResRow = &pFillSup->prev; + } break; + case TSDB_FILL_NEXT: { + if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillSup->next.key = pFillSup->cur.key; + pFillSup->next.pRowVal = pFillSup->cur.pRowVal; + pFillInfo->preRowKey = INT64_MIN; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + } + pFillInfo->pResRow = &pFillSup->next; + } break; + case TSDB_FILL_LINEAR: { + pFillInfo->pLinearInfo->winIndex = 0; + if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_MID; + pFillInfo->pLinearInfo->nextEnd = nextWKey; + int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + + numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pNextDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pLinearInfo->hasNext = true; + } else if (hasPrevWindow(pFillSup)) { + setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_END; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->prev; + pFillInfo->pLinearInfo->hasNext = false; + } else { + ASSERT(hasNextWindow(pFillSup)); + setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); + pFillInfo->pos = FILL_POS_START; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + int32_t numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, + pFillSup->interval.slidingUnit, pFillSup->interval.precision); + calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, + pFillSup->numOfAllCols, numOfWins); + pFillInfo->pResRow = &pFillSup->cur; + pFillInfo->pLinearInfo->hasNext = false; + } + } break; + default: + ASSERT(0); + break; + } + ASSERT(pFillInfo->pos != FILL_POS_INVALID); +} + +static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { + SWinKey key = {.groupId = groupId, .ts = ts}; + if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { + return false; + } + tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + return true; +} + +static void buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { + uint64_t groupId = pBlock->info.id.groupId; + if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { + return; + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + SFillInfo tmpInfo = { + .currentKey = ts, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); + if (!filled) { + SResultCellData* pCell = getResultCell(pResRow, slotId); + setRowCell(pColData, pBlock->info.rows, pCell); + } + } + pBlock->info.rows++; +} + +static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { + if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { + return true; + } + return false; +} + +static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + } +} + +static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { + uint64_t groupId = pBlock->info.id.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) { + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->pLinearInfo->winIndex++; + continue; + } + pFillInfo->pLinearInfo->winIndex++; + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; + SFillInfo tmp = { + .currentKey = pFillInfo->current, + .order = TSDB_ORDER_ASC, + .interval = pFillSup->interval, + }; + + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); + int16_t type = pColData->info.type; + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); + int32_t index = pBlock->info.rows; + if (pFillCol->notFillCol) { + bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); + if (!filled) { + setRowCell(pColData, index, pCell); + } + } else { + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { + colDataAppendNULL(pColData, index); + continue; + } + double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId); + double vCell = 0; + GET_TYPED_DATA(vCell, double, pCell->type, pCell->pData); + vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex; + int64_t result = 0; + SET_TYPED_DATA(&result, pCell->type, vCell); + colDataAppend(pColData, index, (const char*)&result, false); + } + } + pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pBlock->info.rows++; + } +} + +static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { + SWinKey key = {.groupId = groupId, .ts = pRow->key}; + int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); + ASSERT(code == TSDB_CODE_SUCCESS); +} + +static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + if (pFillInfo->needFill == false) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + return; + } + + if (pFillInfo->pos == FILL_POS_START) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } + if (pFillInfo->type != TSDB_FILL_LINEAR) { + doStreamFillNormal(pFillSup, pFillInfo, pRes); + } else { + doStreamFillLinear(pFillSup, pFillInfo, pRes); + + if (pFillInfo->pos == FILL_POS_MID) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } + + if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->winIndex = 0; + taosArrayClear(pFillInfo->pLinearInfo->pDeltaVal); + taosArrayAddAll(pFillInfo->pLinearInfo->pDeltaVal, pFillInfo->pLinearInfo->pNextDeltaVal); + pFillInfo->pResRow = &pFillSup->cur; + setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); + doStreamFillLinear(pFillSup, pFillInfo, pRes); + } + } + if (pFillInfo->pos == FILL_POS_END) { + buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + } +} + +void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, + int32_t rowId, uint64_t groupId, int32_t rowSize) { + TSKEY ts = tsCol[rowId]; + pFillInfo->nextRowKey = ts; + SResultRowData tmpNextRow = {.key = ts}; + tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); + transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); + keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); + taosMemoryFreeClear(tmpNextRow.pRowVal); +} + +static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, + SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { + uint64_t groupId = pBlock->info.id.groupId; + getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); + if (pFillSup->prev.key == pFillInfo->preRowKey) { + resetFillWindow(&pFillSup->prev); + } + setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); + doStreamFillRange(pFillInfo, pFillSup, pRes); +} + +static void doStreamFillImpl(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcBlock; + uint64_t groupId = pBlock->info.id.groupId; + SSDataBlock* pRes = pInfo->pRes; + pRes->info.id.groupId = groupId; + if (hasRemainCalc(pFillInfo)) { + doStreamFillRange(pFillInfo, pFillSup, pRes); + } + + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; + + if (pInfo->srcRowIndex == 0) { + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + SSDataBlock* preBlock = pInfo->pPrevSrcBlock; + if (preBlock->info.rows > 0) { + int preRowId = preBlock->info.rows - 1; + SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); + doFillResults(pOperator, pFillSup, pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, pRes); + } + pInfo->srcRowIndex++; + } + + while (pInfo->srcRowIndex < pBlock->info.rows) { + TSKEY ts = tsCol[pInfo->srcRowIndex]; + keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); + doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); + if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + return; + } + pInfo->srcRowIndex++; + } + blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + blockDataCleanup(pInfo->pPrevSrcBlock); + copyDataBlock(pInfo->pPrevSrcBlock, pInfo->pSrcBlock); + blockDataCleanup(pInfo->pSrcBlock); +} + +static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; + + SSDataBlock* pBlock = delRes; + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + colDataAppend(pStartCol, pBlock->info.rows, (const char*)&start, false); + colDataAppend(pEndCol, pBlock->info.rows, (const char*)&end, false); + colDataAppendNULL(pUidCol, pBlock->info.rows); + colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + colDataAppendNULL(pCalStartCol, pBlock->info.rows); + colDataAppendNULL(pCalEndCol, pBlock->info.rows); + + SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + + void* tbname = NULL; + streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); + if (tbname == NULL) { + colDataAppendNULL(pTableCol, pBlock->info.rows); + } else { + char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; + STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); + colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + tdbFree(tbname); + } + + pBlock->info.rows++; +} + +static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + if (hasPrevWindow(pFillSup)) { + TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); + buildDeleteRange(pOperator, start, endTs, groupId, delRes); + } else if (hasNextWindow(pFillSup)) { + TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); + buildDeleteRange(pOperator, startTs, end, groupId, delRes); + } else { + buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + } +} + +static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); + setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); + SWinKey key = {.ts = startTs, .groupId = groupId}; + if (!pInfo->pFillInfo->needFill) { + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + } else { + STimeRange tw = { + .skey = startTs, + .ekey = endTs, + .groupId = groupId, + }; + taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + while (key.ts <= endTs) { + key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, + pInfo->pFillSup->interval.precision); + tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + } + } +} + +static void doDeleteFillFinalize(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + int32_t size = taosArrayGetSize(pFillInfo->delRanges); + tSimpleHashClear(pInfo->pFillSup->pResMap); + for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { + STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); + if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { + return; + } + getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); + setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); + if (pInfo->pFillInfo->needFill) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + pInfo->pRes->info.id.groupId = range->groupId; + } + SWinKey key = {.ts = range->skey, .groupId = range->groupId}; + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); + } +} + +static void doDeleteFillResult(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SStreamFillSupporter* pFillSup = pInfo->pFillSup; + SStreamFillInfo* pFillInfo = pInfo->pFillInfo; + SSDataBlock* pBlock = pInfo->pSrcDelBlock; + SSDataBlock* pRes = pInfo->pRes; + SSDataBlock* pDelRes = pInfo->pDelRes; + + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; + TSKEY endTs = ts; + uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; + SWinKey key = {.ts = ts, .groupId = groupId}; + SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); + if (!pCur) { + pInfo->srcDelRowIndex++; + continue; + } + + SWinKey nextKey = {.groupId = groupId, .ts = ts}; + while (pInfo->srcDelRowIndex < pBlock->info.rows) { + void* nextVal = NULL; + int32_t nextLen = 0; + TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; + uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; + int32_t code = TSDB_CODE_SUCCESS; + if (groupId != delGroupId) { + break; + } + if (delTs > nextKey.ts) { + break; + } + endTs = delTs; + SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; + if (delTs == nextKey.ts) { + code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); + } + // ts will be deleted later + if (delTs != ts) { + streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + streamStateFreeCur(pCur); + pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); + } + endTs = TMAX(ts, nextKey.ts - 1); + if (code != TSDB_CODE_SUCCESS) { + break; + } + } + pInfo->srcDelRowIndex++; + } + streamStateFreeCur(pCur); + doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + } + pFillInfo->current = pFillInfo->end + 1; +} + +static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { + blockDataCleanup(pInfo->pPrevSrcBlock); + tSimpleHashClear(pInfo->pFillSup->pResMap); + pInfo->pFillSup->hasDelete = false; + taosArrayClear(pInfo->pFillInfo->delRanges); + pInfo->pFillInfo->delIndex = 0; +} + +static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExprSupp* pSup = &pOperator->exprSupp; + + blockDataCleanup(pDstBlock); + blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + + pDstBlock->info.rows = 0; + pSup = &pInfo->pFillSup->notFillExprSup; + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); + projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; + + blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); +} + +static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + SStreamFillOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + blockDataCleanup(pInfo->pRes); + if (pOperator->status == OP_RES_TO_RETURN) { + if (hasRemainCalc(pInfo->pFillInfo)) { + doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } + } + doDeleteFillFinalize(pOperator); + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; + } + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + SSDataBlock* fillResult = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { + // If there are delete datablocks, we receive them first. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + pOperator->status = OP_RES_TO_RETURN; + SSDataBlock* preBlock = pInfo->pPrevSrcBlock; + if (preBlock->info.rows > 0) { + int preRowId = preBlock->info.rows - 1; + SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); + doFillResults(pOperator, pInfo->pFillSup, pInfo->pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, + pInfo->pRes); + } + pInfo->pFillInfo->preRowKey = INT64_MIN; + if (pInfo->pRes->info.rows > 0) { + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; + } + break; + } + printDataBlock(pBlock, "stream fill recv"); + + switch (pBlock->info.type) { + case STREAM_RETRIEVE: + return pBlock; + case STREAM_DELETE_RESULT: { + pInfo->pSrcDelBlock = pBlock; + pInfo->srcDelRowIndex = 0; + blockDataCleanup(pInfo->pDelRes); + pInfo->pFillSup->hasDelete = true; + doDeleteFillResult(pOperator); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "stream fill delete"); + return pInfo->pDelRes; + } + continue; + } break; + case STREAM_NORMAL: + case STREAM_INVALID: { + doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pInfo->srcRowIndex = 0; + } break; + default: + ASSERT(0); + break; + } + } + + doStreamFillImpl(pOperator); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + if (pInfo->pRes->info.rows > 0) { + break; + } + } + if (pOperator->status == OP_RES_TO_RETURN) { + doDeleteFillFinalize(pOperator); + } + + if (pInfo->pRes->info.rows == 0) { + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; + } + + pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; + printDataBlock(pInfo->pRes, "stream fill"); + return pInfo->pRes; +} + +static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { + pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; + for (int i = 0; i < pFillSup->numOfAllCols; i++) { + SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + pFillSup->rowSize += pSchema->bytes; + } + pFillSup->next.key = INT64_MIN; + pFillSup->nextNext.key = INT64_MIN; + pFillSup->prev.key = INT64_MIN; + pFillSup->cur.key = INT64_MIN; + pFillSup->next.pRowVal = NULL; + pFillSup->nextNext.pRowVal = NULL; + pFillSup->prev.pRowVal = NULL; + pFillSup->cur.pRowVal = NULL; + + return TSDB_CODE_SUCCESS; +} + +static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, + SExprInfo* pFillExprInfo, int32_t numOfFillCols) { + SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); + if (!pFillSup) { + return NULL; + } + pFillSup->numOfFillCols = numOfFillCols; + int32_t numOfNotFillCols = 0; + SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, + (const SNodeListNode*)(pPhyFillNode->pValues)); + pFillSup->type = convertFillType(pPhyFillNode->mode); + pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; + pFillSup->interval = *pInterval; + + int32_t code = initResultBuf(pFillSup); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); + code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + return NULL; + } + + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pFillSup->pResMap = tSimpleHashInit(16, hashFn); + pFillSup->hasDelete = false; + return pFillSup; +} + +SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + pFillInfo->start = INT64_MIN; + pFillInfo->current = INT64_MIN; + pFillInfo->end = INT64_MIN; + pFillInfo->preRowKey = INT64_MIN; + pFillInfo->needFill = false; + pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + pFillInfo->pLinearInfo->hasNext = false; + pFillInfo->pLinearInfo->nextEnd = INT64_MIN; + pFillInfo->pLinearInfo->pDeltaVal = NULL; + pFillInfo->pLinearInfo->pNextDeltaVal = NULL; + if (pFillSup->type == TSDB_FILL_LINEAR) { + pFillInfo->pLinearInfo->pDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); + pFillInfo->pLinearInfo->pNextDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { + double value = 0.0; + taosArrayPush(pFillInfo->pLinearInfo->pDeltaVal, &value); + taosArrayPush(pFillInfo->pLinearInfo->pNextDeltaVal, &value); + } + } + pFillInfo->pLinearInfo->winIndex = 0; + + pFillInfo->pResRow = NULL; + if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_NULL) { + pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + pFillInfo->pResRow->key = INT64_MIN; + pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); + pCell->bytes = pColData->info.bytes; + pCell->type = pColData->info.type; + } + } + + pFillInfo->type = pFillSup->type; + pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + pFillInfo->delIndex = 0; + return pFillInfo; +} + +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo) { + SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + int32_t numOfFillCols = 0; + SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); + pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); + if (!pInfo->pFillSup) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + pInfo->pPrevSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pPrevSrcBlock, pOperator->resultInfo.capacity); + + pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); + if (!pInfo->pFillInfo) { + goto _error; + } + + if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + SVariant* pVar = &(pFillCol->fillVal); + if (pCell->type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (pCell->type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pCell->type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + SET_TYPED_DATA(pCell->pData, pCell->type, v); + } else { + pCell->isNull = true; + } + } + } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL) { + for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { + SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; + int32_t slotId = GET_DEST_SLOT_ID(pFillCol); + SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); + pCell->isNull = true; + } + } + + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + + pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; + pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; + + int32_t numOfOutputCols = 0; + int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->srcRowIndex = 0; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + + _error: + destroyStreamFillOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84b7678b9f..c5d86ed653 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1335,30 +1335,6 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, return code; } -static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* pResBlock) { - if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return; - if (pBlock == NULL || pBlock->info.rows == 0) return; - - SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0); - ASSERT(pSrcBlock->info.rows == 1); - - blockDataEnsureCapacity(pResBlock, 1); - - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - - // build tagArray - /*SArray* tagArray = taosArrayInit(0, sizeof(void*));*/ - /*STagVal tagVal = {*/ - /*.cid = 0,*/ - /*.type = 0,*/ - /*};*/ - // build STag - // set STag - - blockDataDestroy(pSrcBlock); -} - void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 2300fc3359..a66a5e7089 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -35,22 +35,7 @@ #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) \ ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) - -#define FILL_POS_INVALID 0 -#define FILL_POS_START 1 -#define FILL_POS_MID 2 -#define FILL_POS_END 3 - -typedef struct STimeRange { - TSKEY skey; - TSKEY ekey; - uint64_t groupId; -} STimeRange; - static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); -static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, - int32_t rowIndex); static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowIndex) { for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { @@ -91,8 +76,8 @@ static void doSetUserSpecifiedValue(SColumnInfoData* pDst, SVariant* pVar, int32 } // fill windows pseudo column, _wstart, _wend, _wduration and return true, otherwise return false -static bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, - int32_t rowIndex) { +bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, + int32_t rowIndex) { if (!pCol->notFillCol) { return false; } @@ -663,1097 +648,3 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn return pFillCol; } - -TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); - return win.skey; -} - -TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { - STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); - return win.skey; -} - -void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataAppend(pCol, rowId, pCell->pData, pCell->isNull); -} - -SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { - if (!pRaw || !pRaw->pRowVal) { - return NULL; - } - char* pData = (char*)pRaw->pRowVal; - SResultCellData* pCell = pRaw->pRowVal; - for (int32_t i = 0; i < index; i++) { - pData += (pCell->bytes + sizeof(SResultCellData)); - pCell = (SResultCellData*)pData; - } - return pCell; -} - -void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { - for (int32_t i = start; i < end; i++) { - destroyExprInfo(pFillCol[i].pExpr, 1); - taosVariantDestroy(&pFillCol[i].fillVal); - } - taosMemoryFreeClear(pFillCol[start].pExpr); - taosMemoryFree(pFillCol); - return NULL; -} - -void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { - pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); - tSimpleHashCleanup(pFillSup->pResMap); - pFillSup->pResMap = NULL; - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); - pFillSup->cur.pRowVal = NULL; - cleanupExprSupp(&pFillSup->notFillExprSup); - - taosMemoryFree(pFillSup); - return NULL; -} - -void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { - taosArrayDestroy(pFillLinear->pDeltaVal); - taosArrayDestroy(pFillLinear->pNextDeltaVal); - taosMemoryFree(pFillLinear); - return NULL; -} -void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { - if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_NULL) { - taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); - taosMemoryFreeClear(pFillInfo->pResRow); - } - pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); - taosArrayDestroy(pFillInfo->delRanges); - taosMemoryFree(pFillInfo); - return NULL; -} - -void destroyStreamFillOperatorInfo(void* param) { - SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); - pInfo->pPrevSrcBlock = blockDataDestroy(pInfo->pPrevSrcBlock); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFree(pInfo); -} - -static void resetFillWindow(SResultRowData* pRowData) { - pRowData->key = INT64_MIN; - pRowData->pRowVal = NULL; -} - -void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { - resetFillWindow(&pFillSup->prev); - releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); - resetFillWindow(&pFillSup->cur); - resetFillWindow(&pFillSup->next); - resetFillWindow(&pFillSup->nextNext); -} - -void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); - - SWinKey key = {.ts = ts, .groupId = groupId}; - int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; -} - -void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStreamState* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState); - - SWinKey key = {.ts = ts, .groupId = groupId}; - void* curVal = NULL; - int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); - pFillSup->cur.key = key.ts; - pFillSup->cur.pRowVal = curVal; - - SStreamStateCur* pCur = streamStateFillSeekKeyPrev(pState, &key); - SWinKey preKey = {.groupId = groupId}; - void* preVal = NULL; - int32_t preVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); - - if (code == TSDB_CODE_SUCCESS) { - pFillSup->prev.key = preKey.ts; - pFillSup->prev.pRowVal = preVal; - - code = streamStateCurNext(pState, pCur); - ASSERT(code == TSDB_CODE_SUCCESS); - - code = streamStateCurNext(pState, pCur); - if (code != TSDB_CODE_SUCCESS) { - streamStateFreeCur(pCur); - pCur = NULL; - } - } else { - streamStateFreeCur(pCur); - pCur = streamStateFillSeekKeyNext(pState, &key); - } - - SWinKey nextKey = {.groupId = groupId}; - void* nextVal = NULL; - int32_t nextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->next.key = nextKey.ts; - pFillSup->next.pRowVal = nextVal; - if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = streamStateCurNext(pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } - } - } - } - streamStateFreeCur(pCur); -} - -static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } -static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; } -static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) { - return pFillSup->nextNext.key != INT64_MIN; - return false; -} - -static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SResultCellData* pCell = getResultCell(pRowVal, i); - if (!colDataIsNull_s(pColData, rowId)) { - pCell->isNull = false; - pCell->type = pColData->info.type; - pCell->bytes = pColData->info.bytes; - char* val = colDataGetData(pColData, rowId); - if (IS_VAR_DATA_TYPE(pCell->type)) { - memcpy(pCell->pData, val, varDataTLen(val)); - } else { - memcpy(pCell->pData, val, pCell->bytes); - } - } else { - pCell->isNull = true; - } - } - pRowVal->key = ts; -} - -static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta, - SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - char* var = colDataGetData(pCol, rowId); - double start = 0; - GET_TYPED_DATA(start, double, pCol->info.type, var); - SResultCellData* pCell = getResultCell(pRowVal, slotId); - double end = 0; - GET_TYPED_DATA(end, double, pCell->type, pCell->pData); - double delta = 0; - if (order == TSDB_ORDER_ASC) { - delta = (end - start) / winCount; - } else { - delta = (start - end) / winCount; - } - taosArraySet(pDelta, slotId, &delta); - } - } -} - -static void calcRowDeltaData(SResultRowData* pStartRow, SResultRowData* pEndRow, SArray* pDelta, SFillColInfo* pFillCol, - int32_t numOfCol, int32_t winCount) { - for (int32_t i = 0; i < numOfCol; i++) { - if (!pFillCol[i].notFillCol) { - int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SResultCellData* pSCell = getResultCell(pStartRow, slotId); - double start = 0.0; - GET_TYPED_DATA(start, double, pSCell->type, pSCell->pData); - SResultCellData* pECell = getResultCell(pEndRow, slotId); - double end = 0.0; - GET_TYPED_DATA(end, double, pECell->type, pECell->pData); - double delta = (end - start) / winCount; - taosArraySet(pDelta, slotId, &delta); - } - } -} - -static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - pFillInfo->start = ts; -} - -static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision); - pFillInfo->end = ts; -} - -static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) { - setFillInfoStart(start, pInterval, pFillInfo); - pFillInfo->current = pFillInfo->start; - setFillInfoEnd(end, pInterval, pFillInfo); -} - -void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - return; - } - - TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - - pFillInfo->needFill = true; - pFillInfo->start = realStart; - pFillInfo->current = pFillInfo->start; - pFillInfo->end = end; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_SET_VALUE: - break; - case TSDB_FILL_PREV: - pFillInfo->pResRow = &pFillSup->prev; - break; - case TSDB_FILL_NEXT: - pFillInfo->pResRow = &pFillSup->next; - break; - case TSDB_FILL_LINEAR: { - setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(pFillSup->prev.key, pFillSup->next.key, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->winIndex = 0; - } break; - default: - ASSERT(0); - break; - } -} - -void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) { - for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId); - pCell->isNull = pCurCell->isNull; - if (!pCurCell->isNull) { - memcpy(pCell->pData, pCurCell->pData, pCell->bytes); - } - } -} - -void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup, - SStreamFillInfo* pFillInfo) { - pFillInfo->preRowKey = pFillSup->cur.key; - if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - return; - } - TSKEY prevWKey = INT64_MIN; - TSKEY nextWKey = INT64_MIN; - if (hasPrevWindow(pFillSup)) { - prevWKey = pFillSup->prev.key; - } - if (hasNextWindow(pFillSup)) { - nextWKey = pFillSup->next.key; - } - - pFillInfo->needFill = true; - pFillInfo->pos = FILL_POS_INVALID; - switch (pFillInfo->type) { - case TSDB_FILL_NULL: - case TSDB_FILL_SET_VALUE: { - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - if (pFillSup->next.key == pFillInfo->nextRowKey) { - pFillInfo->preRowKey = INT64_MIN; - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - pFillInfo->needFill = false; - pFillInfo->pos = FILL_POS_START; - } - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - } else { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - copyNotFillExpData(pFillSup, pFillInfo); - } break; - case TSDB_FILL_PREV: { - if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) || - (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) || - (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) { - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - pFillSup->prev.key = pFillSup->cur.key; - pFillSup->prev.pRowVal = pFillSup->cur.pRowVal; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->preRowKey = INT64_MIN; - } - pFillInfo->pResRow = &pFillSup->prev; - } break; - case TSDB_FILL_NEXT: { - if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillSup->next.key = pFillSup->cur.key; - pFillSup->next.pRowVal = pFillSup->cur.pRowVal; - pFillInfo->preRowKey = INT64_MIN; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - } - pFillInfo->pResRow = &pFillSup->next; - } break; - case TSDB_FILL_LINEAR: { - pFillInfo->pLinearInfo->winIndex = 0; - if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_MID; - pFillInfo->pLinearInfo->nextEnd = nextWKey; - int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); - pFillInfo->pResRow = &pFillSup->prev; - - numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pNextDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); - pFillInfo->pLinearInfo->hasNext = true; - } else if (hasPrevWindow(pFillSup)) { - setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_END; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); - pFillInfo->pResRow = &pFillSup->prev; - pFillInfo->pLinearInfo->hasNext = false; - } else { - ASSERT(hasNextWindow(pFillSup)); - setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); - pFillInfo->pos = FILL_POS_START; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); - pFillInfo->pResRow = &pFillSup->cur; - pFillInfo->pLinearInfo->hasNext = false; - } - } break; - default: - ASSERT(0); - break; - } - ASSERT(pFillInfo->pos != FILL_POS_INVALID); -} - -static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { - SWinKey key = {.groupId = groupId, .ts = ts}; - if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { - return false; - } - tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - return true; -} - -static void buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { - uint64_t groupId = pBlock->info.id.groupId; - if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { - return; - } - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - SFillInfo tmpInfo = { - .currentKey = ts, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); - if (!filled) { - SResultCellData* pCell = getResultCell(pResRow, slotId); - setRowCell(pColData, pBlock->info.rows, pCell); - } - } - pBlock->info.rows++; -} - -static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { - if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) { - return true; - } - return false; -} - -static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - } -} - -static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { - while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - uint64_t groupId = pBlock->info.id.groupId; - SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; - if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) { - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pFillInfo->pLinearInfo->winIndex++; - continue; - } - pFillInfo->pLinearInfo->winIndex++; - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; - SFillInfo tmp = { - .currentKey = pFillInfo->current, - .order = TSDB_ORDER_ASC, - .interval = pFillSup->interval, - }; - - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId); - int16_t type = pColData->info.type; - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId); - int32_t index = pBlock->info.rows; - if (pFillCol->notFillCol) { - bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); - if (!filled) { - setRowCell(pColData, index, pCell); - } - } else { - if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { - colDataAppendNULL(pColData, index); - continue; - } - double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId); - double vCell = 0; - GET_TYPED_DATA(vCell, double, pCell->type, pCell->pData); - vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex; - int64_t result = 0; - SET_TYPED_DATA(&result, pCell->type, vCell); - colDataAppend(pColData, index, (const char*)&result, false); - } - } - pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - pBlock->info.rows++; - } -} - -static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { - SWinKey key = {.groupId = groupId, .ts = pRow->key}; - int32_t code = streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len); - ASSERT(code == TSDB_CODE_SUCCESS); -} - -static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - if (pFillInfo->needFill == false) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - return; - } - - if (pFillInfo->pos == FILL_POS_START) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - } - if (pFillInfo->type != TSDB_FILL_LINEAR) { - doStreamFillNormal(pFillSup, pFillInfo, pRes); - } else { - doStreamFillLinear(pFillSup, pFillInfo, pRes); - - if (pFillInfo->pos == FILL_POS_MID) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - } - - if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->winIndex = 0; - taosArrayClear(pFillInfo->pLinearInfo->pDeltaVal); - taosArrayAddAll(pFillInfo->pLinearInfo->pDeltaVal, pFillInfo->pLinearInfo->pNextDeltaVal); - pFillInfo->pResRow = &pFillSup->cur; - setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); - doStreamFillLinear(pFillSup, pFillInfo, pRes); - } - } - if (pFillInfo->pos == FILL_POS_END) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - } -} - -void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, - int32_t rowId, uint64_t groupId, int32_t rowSize) { - TSKEY ts = tsCol[rowId]; - pFillInfo->nextRowKey = ts; - SResultRowData tmpNextRow = {.key = ts}; - tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize); - transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow); - keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize); - taosMemoryFreeClear(tmpNextRow.pRowVal); -} - -static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, - SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) { - uint64_t groupId = pBlock->info.id.groupId; - getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup); - if (pFillSup->prev.key == pFillInfo->preRowKey) { - resetFillWindow(&pFillSup->prev); - } - setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo); - doStreamFillRange(pFillInfo, pFillSup, pRes); -} - -static void doStreamFillImpl(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcBlock; - uint64_t groupId = pBlock->info.id.groupId; - SSDataBlock* pRes = pInfo->pRes; - pRes->info.id.groupId = groupId; - if (hasRemainCalc(pFillInfo)) { - doStreamFillRange(pFillInfo, pFillSup, pRes); - } - - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; - - if (pInfo->srcRowIndex == 0) { - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - SSDataBlock* preBlock = pInfo->pPrevSrcBlock; - if (preBlock->info.rows > 0) { - int preRowId = preBlock->info.rows - 1; - SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); - doFillResults(pOperator, pFillSup, pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, pRes); - } - pInfo->srcRowIndex++; - } - - while (pInfo->srcRowIndex < pBlock->info.rows) { - TSKEY ts = tsCol[pInfo->srcRowIndex]; - keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); - doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - return; - } - pInfo->srcRowIndex++; - } - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); - blockDataCleanup(pInfo->pPrevSrcBlock); - copyDataBlock(pInfo->pPrevSrcBlock, pInfo->pSrcBlock); - blockDataCleanup(pInfo->pSrcBlock); -} - -static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStreamState* pState = pOp->pTaskInfo->streamInfo.pState; - - SSDataBlock* pBlock = delRes; - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataAppend(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataAppend(pEndCol, pBlock->info.rows, (const char*)&end, false); - colDataAppendNULL(pUidCol, pBlock->info.rows); - colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); - colDataAppendNULL(pCalStartCol, pBlock->info.rows); - colDataAppendNULL(pCalEndCol, pBlock->info.rows); - - SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - - void* tbname = NULL; - streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname); - if (tbname == NULL) { - colDataAppendNULL(pTableCol, pBlock->info.rows); - } else { - char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; - STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); - tdbFree(tbname); - } - - pBlock->info.rows++; -} - -static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSDataBlock* delRes) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - if (hasPrevWindow(pFillSup)) { - TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); - buildDeleteRange(pOperator, start, endTs, groupId, delRes); - } else if (hasNextWindow(pFillSup)) { - TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); - buildDeleteRange(pOperator, startTs, end, groupId, delRes); - } else { - buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); - } -} - -static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); - setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); - SWinKey key = {.ts = startTs, .groupId = groupId}; - if (!pInfo->pFillInfo->needFill) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); - } else { - STimeRange tw = { - .skey = startTs, - .ekey = endTs, - .groupId = groupId, - }; - taosArrayPush(pInfo->pFillInfo->delRanges, &tw); - while (key.ts <= endTs) { - key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit, - pInfo->pFillSup->interval.precision); - tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); - } - } -} - -static void doDeleteFillFinalize(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - int32_t size = taosArrayGetSize(pFillInfo->delRanges); - tSimpleHashClear(pInfo->pFillSup->pResMap); - for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) { - STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex); - if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) { - return; - } - getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); - setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); - if (pInfo->pFillInfo->needFill) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - pInfo->pRes->info.id.groupId = range->groupId; - } - SWinKey key = {.ts = range->skey, .groupId = range->groupId}; - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); - } -} - -static void doDeleteFillResult(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SStreamFillSupporter* pFillSup = pInfo->pFillSup; - SStreamFillInfo* pFillInfo = pInfo->pFillInfo; - SSDataBlock* pBlock = pInfo->pSrcDelBlock; - SSDataBlock* pRes = pInfo->pRes; - SSDataBlock* pDelRes = pInfo->pDelRes; - - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - TSKEY ts = tsStarts[pInfo->srcDelRowIndex]; - TSKEY endTs = ts; - uint64_t groupId = groupIds[pInfo->srcDelRowIndex]; - SWinKey key = {.ts = ts, .groupId = groupId}; - SStreamStateCur* pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key); - if (!pCur) { - pInfo->srcDelRowIndex++; - continue; - } - - SWinKey nextKey = {.groupId = groupId, .ts = ts}; - while (pInfo->srcDelRowIndex < pBlock->info.rows) { - void* nextVal = NULL; - int32_t nextLen = 0; - TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; - uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; - int32_t code = TSDB_CODE_SUCCESS; - if (groupId != delGroupId) { - break; - } - if (delTs > nextKey.ts) { - break; - } - endTs = delTs; - SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; - if (delTs == nextKey.ts) { - code = streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); - } - // ts will be deleted later - if (delTs != ts) { - streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); - streamStateFreeCur(pCur); - pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); - } - endTs = TMAX(ts, nextKey.ts - 1); - if (code != TSDB_CODE_SUCCESS) { - break; - } - } - pInfo->srcDelRowIndex++; - } - streamStateFreeCur(pCur); - doDeleteFillResultImpl(pOperator, ts, endTs, groupId); - } - pFillInfo->current = pFillInfo->end + 1; -} - -static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { - blockDataCleanup(pInfo->pPrevSrcBlock); - tSimpleHashClear(pInfo->pFillSup->pResMap); - pInfo->pFillSup->hasDelete = false; - taosArrayClear(pInfo->pFillInfo->delRanges); - pInfo->pFillInfo->delIndex = 0; -} - -static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - - blockDataCleanup(pDstBlock); - blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - - pDstBlock->info.rows = 0; - pSup = &pInfo->pFillSup->notFillExprSup; - setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); - pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; - - blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); -} - -static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { - SStreamFillOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - blockDataCleanup(pInfo->pRes); - if (pOperator->status == OP_RES_TO_RETURN) { - if (hasRemainCalc(pInfo->pFillInfo)) { - doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } - } - doDeleteFillFinalize(pOperator); - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, "stream fill"); - return pInfo->pRes; - } - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - SSDataBlock* fillResult = NULL; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { - // If there are delete datablocks, we receive them first. - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - pOperator->status = OP_RES_TO_RETURN; - SSDataBlock* preBlock = pInfo->pPrevSrcBlock; - if (preBlock->info.rows > 0) { - int preRowId = preBlock->info.rows - 1; - SColumnInfoData* pPreTsCol = taosArrayGet(preBlock->pDataBlock, pInfo->primaryTsCol); - doFillResults(pOperator, pInfo->pFillSup, pInfo->pFillInfo, preBlock, (TSKEY*)pPreTsCol->pData, preRowId, - pInfo->pRes); - } - pInfo->pFillInfo->preRowKey = INT64_MIN; - if (pInfo->pRes->info.rows > 0) { - printDataBlock(pInfo->pRes, "stream fill"); - return pInfo->pRes; - } - break; - } - printDataBlock(pBlock, "stream fill recv"); - - switch (pBlock->info.type) { - case STREAM_RETRIEVE: - return pBlock; - case STREAM_DELETE_RESULT: { - pInfo->pSrcDelBlock = pBlock; - pInfo->srcDelRowIndex = 0; - blockDataCleanup(pInfo->pDelRes); - pInfo->pFillSup->hasDelete = true; - doDeleteFillResult(pOperator); - if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "stream fill delete"); - return pInfo->pDelRes; - } - continue; - } break; - case STREAM_NORMAL: - case STREAM_INVALID: { - doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); - memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = 0; - } break; - default: - ASSERT(0); - break; - } - } - - doStreamFillImpl(pOperator); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); - memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - if (pInfo->pRes->info.rows > 0) { - break; - } - } - if (pOperator->status == OP_RES_TO_RETURN) { - doDeleteFillFinalize(pOperator); - } - - if (pInfo->pRes->info.rows == 0) { - setOperatorCompleted(pOperator); - resetStreamFillInfo(pInfo); - return NULL; - } - - pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; - printDataBlock(pInfo->pRes, "stream fill"); - return pInfo->pRes; -} - -static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { - pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols; - for (int i = 0; i < pFillSup->numOfAllCols; i++) { - SFillColInfo* pCol = &pFillSup->pAllColInfo[i]; - SResSchema* pSchema = &pCol->pExpr->base.resSchema; - pFillSup->rowSize += pSchema->bytes; - } - pFillSup->next.key = INT64_MIN; - pFillSup->nextNext.key = INT64_MIN; - pFillSup->prev.key = INT64_MIN; - pFillSup->cur.key = INT64_MIN; - pFillSup->next.pRowVal = NULL; - pFillSup->nextNext.pRowVal = NULL; - pFillSup->prev.pRowVal = NULL; - pFillSup->cur.pRowVal = NULL; - - return TSDB_CODE_SUCCESS; -} - -static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, - SExprInfo* pFillExprInfo, int32_t numOfFillCols) { - SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); - if (!pFillSup) { - return NULL; - } - pFillSup->numOfFillCols = numOfFillCols; - int32_t numOfNotFillCols = 0; - SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, - (const SNodeListNode*)(pPhyFillNode->pValues)); - pFillSup->type = convertFillType(pPhyFillNode->mode); - pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols; - pFillSup->interval = *pInterval; - - int32_t code = initResultBuf(pFillSup); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); - code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } - - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pFillSup->pResMap = tSimpleHashInit(16, hashFn); - pFillSup->hasDelete = false; - return pFillSup; -} - -SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { - SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); - pFillInfo->start = INT64_MIN; - pFillInfo->current = INT64_MIN; - pFillInfo->end = INT64_MIN; - pFillInfo->preRowKey = INT64_MIN; - pFillInfo->needFill = false; - pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); - pFillInfo->pLinearInfo->hasNext = false; - pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - pFillInfo->pLinearInfo->pDeltaVal = NULL; - pFillInfo->pLinearInfo->pNextDeltaVal = NULL; - if (pFillSup->type == TSDB_FILL_LINEAR) { - pFillInfo->pLinearInfo->pDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); - pFillInfo->pLinearInfo->pNextDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); - for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { - double value = 0.0; - taosArrayPush(pFillInfo->pLinearInfo->pDeltaVal, &value); - taosArrayPush(pFillInfo->pLinearInfo->pNextDeltaVal, &value); - } - } - pFillInfo->pLinearInfo->winIndex = 0; - - pFillInfo->pResRow = NULL; - if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_NULL) { - pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); - pFillInfo->pResRow->key = INT64_MIN; - pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); - for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { - SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); - pCell->bytes = pColData->info.bytes; - pCell->type = pColData->info.type; - } - } - - pFillInfo->type = pFillSup->type; - pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); - pFillInfo->delIndex = 0; - return pFillInfo; -} - -SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, - SExecTaskInfo* pTaskInfo) { - SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; - int32_t numOfFillCols = 0; - SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); - pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); - if (!pInfo->pFillSup) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - pInfo->pPrevSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pPrevSrcBlock, pOperator->resultInfo.capacity); - - pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); - if (!pInfo->pFillInfo) { - goto _error; - } - - if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - SVariant* pVar = &(pFillCol->fillVal); - if (pCell->type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (pCell->type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pCell->type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - SET_TYPED_DATA(pCell->pData, pCell->type, v); - } else { - pCell->isNull = true; - } - } - } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL) { - for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) { - SFillColInfo* pFillCol = pInfo->pFillSup->pAllColInfo + i; - int32_t slotId = GET_DEST_SLOT_ID(pFillCol); - SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId); - pCell->isNull = true; - } - } - - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); - - pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; - pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; - - int32_t numOfOutputCols = 0; - int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - - code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->srcRowIndex = 0; - setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - return pOperator; - -_error: - destroyStreamFillOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d598fd09ca..f2c2e24a41 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -677,16 +677,6 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } } -void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (!pBlock || pBlock->info.rows == 0) { - qDebug("===stream===printDataBlock: Block is Null or Empty"); - return; - } - char* pBuf = NULL; - qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); - taosMemoryFree(pBuf); -} - typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index); int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) { @@ -3854,7 +3844,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - int64_t groupId = pSDataBlock->info.id.groupId; + uint64_t groupId = pSDataBlock->info.id.groupId; int64_t code = TSDB_CODE_SUCCESS; TSKEY* tsCols = NULL; SResultRow* pResult = NULL;