From fac103ecaa6332758e33ac722cd21952601e51b5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 17:26:53 +0800 Subject: [PATCH 01/82] chore: taostools sanitizer flag align with tdengine --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index f0a1ad83ad..73409a950a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2849aa4 + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2fbf082f7d7f9a00a7b72047d256a39021001c14 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Nov 2022 19:55:15 +0800 Subject: [PATCH 02/82] enh: support client redirect processing --- include/libs/qcom/query.h | 11 +- include/util/taoserror.h | 1 + source/libs/scheduler/inc/schInt.h | 22 +++- source/libs/scheduler/src/schTask.c | 148 ++++++++++++++++++++++---- source/libs/scheduler/src/schUtil.c | 21 ++++ source/libs/scheduler/src/scheduler.c | 6 ++ source/util/src/terror.c | 1 + 7 files changed, 185 insertions(+), 25 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 96ccef7164..f51aa88485 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -259,9 +259,15 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) + +#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later + #define NEED_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || (_code) == TSDB_CODE_SYN_NOT_LEADER || \ + (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ @@ -270,7 +276,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ - (_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY) + SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_APP_NOT_READY) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 636decc60b..26f22f398c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -92,6 +92,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0129) #define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x012A) #define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B) +#define TSDB_CODE_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x012C) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 6884824ba9..7cd8d608c2 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -145,7 +145,8 @@ typedef struct SSchedulerMgmt { bool exit; int32_t jobRef; int32_t jobNum; - SSchStat stat; + SSchStat stat; + void *timer; SRWLatch hbLock; SHashObj *hbConnections; void *queryMgmt; @@ -202,12 +203,30 @@ typedef struct SSchTaskProfile { int64_t endTs; } SSchTaskProfile; +typedef struct SSchRedirectCtx { + int32_t periodMs; + bool inRedirect; + int32_t totalTimes; + int32_t roundTotal; + int32_t roundTimes; // retry times in current round + int64_t startTs; +} SSchRedirectCtx; + +typedef struct SSchTimerParam { + int64_t rId; + uint64_t queryId; + uint64_t taskId; +} SSchTimerParam; + typedef struct SSchTask { uint64_t taskId; // task id SRWLatch lock; // task reentrant lock int32_t maxExecTimes; // task max exec times int32_t maxRetryTimes; // task max retry times int32_t retryTimes; // task retry times + int32_t delayExecMs; // task execution delay time + tmr_h delayTimer; // task delay execution timer + SSchRedirectCtx redirectCtx; // task redirect context bool waitRetry; // wait for retry int32_t execId; // task current execute index SSchLevel *level; // level @@ -529,6 +548,7 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList); +char *schDumpEpSet(SEpSet *pEpSet); char *schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 7e5b3faedb..9262a9257c 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -340,6 +340,67 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { + SSchRedirectCtx *pCtx = &pTask->redirectCtx; + if (!pCtx->inRedirect) { + pCtx->inRedirect = true; + pCtx->periodMs = tsRedirectPeriod; + pCtx->startTs = taosGetTimestampMs(); + + if (SCH_IS_DATA_BIND_TASK(pTask)) { + if (pEpSet) { + pCtx->roundTotal = pEpSet->numOfEps; + } else { + SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); + pCtx->roundTotal = pAddr->epSet.numOfEps; + } + } else { + pCtx->roundTotal = 1; + } + + goto _return; + } + + pCtx->totalTimes++; + + int64_t nowTs = taosGetTimestampMs(); + if ((nowTs - pCtx->startTs) > tsMaxRetryWaitTime) { + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); + } + + if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { + pCtx->roundTotal = pEpSet->numOfEps; + pCtx->roundTimes = 0; + + pTask->delayExecMs = 0; + + goto _return; + } + + pCtx->roundTimes++; + + if (pCtx->roundTimes >= pCtx->roundTotal) { + pCtx->periodMs *= tsRedirectFactor; + if (pCtx->periodMs > tsRedirectMaxPeriod) { + pCtx->periodMs = tsRedirectMaxPeriod; + } + + pTask->delayExecMs = pCtx->periodMs; + + goto _return; + } + + pTask->delayExecMs = 0; + +_return: + + SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, pCtx->totalTimes, pTask->delayExecMs); + + return TSDB_CODE_SUCCESS; +} + int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; @@ -349,14 +410,10 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->retryTimes = 0; } - if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) { - SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, - pTask->maxExecTimes, pTask->execId); - schHandleJobFailure(pJob, rspCode); - return TSDB_CODE_SUCCESS; - } + SCH_ERR_JRET(schChkUpdateRedirectCtx(pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); @@ -368,8 +425,12 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); if (SCH_IS_DATA_BIND_TASK(pTask)) { - if (pData) { + if (pData && pData->pEpSet) { SCH_ERR_JRET(schUpdateTaskCandidateAddr(pJob, pTask, pData->pEpSet)); + } else if (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(rspCode)) { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SCH_SWITCH_EPSET(addr); + SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -380,7 +441,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); - SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + SCH_ERR_JRET(schDelayLaunchTask(pJob, pTask)); return TSDB_CODE_SUCCESS; } @@ -428,28 +489,24 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); } - if (SCH_IS_DATA_BIND_TASK(pTask)) { + if (SYNC_OTHER_LEADER_REDIRECT_ERROR(rspCode)) { if (NULL == pData->pEpSet) { - SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); - code = rspCode; + SCH_TASK_ELOG("epset updating excepted, error:%s", tstrerror(rspCode)); + code = TSDB_CODE_INVALID_MSG; goto _return; } } code = schDoTaskRedirect(pJob, pTask, pData, rspCode); - taosMemoryFree(pData->pData); - taosMemoryFree(pData->pEpSet); - pData->pData = NULL; - pData->pEpSet = NULL; + taosMemoryFreeClear(pData->pData); + taosMemoryFreeClear(pData->pEpSet); SCH_RET(code); _return: - taosMemoryFree(pData->pData); - taosMemoryFree(pData->pEpSet); - pData->pData = NULL; - pData->pEpSet = NULL; + taosMemoryFreeClear(pData->pData); + taosMemoryFreeClear(pData->pEpSet); SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -715,10 +772,10 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe SQueryNodeAddr *pAddr = taosArrayGet(pTask->candidateAddrs, 0); - SEp *pOld = &pAddr->epSet.eps[pAddr->epSet.inUse]; - SEp *pNew = &pEpSet->eps[pEpSet->inUse]; + char *origEpset = schDumpEpSet(&pAddr->epSet); + char *newEpset = schDumpEpSet(pEpSet); - SCH_TASK_DLOG("update task ep from %s:%d to %s:%d", pOld->fqdn, pOld->port, pNew->fqdn, pNew->port); + SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); @@ -1078,6 +1135,53 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } +void schHandleTimerEvent(void *param, void *tmrId) { + SSchTimerParam *pTimerParam = (SSchTimerParam *)param; + SSchTask *pTask = NULL; + SSchJob *pJob = NULL; + int32_t code = 0; + + SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)); + + SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + + return; + +_return: + + schHandleJobFailure(pJob, code); +} + +int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { + if (pTask->delayExecMs > 0) { + SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); + if (NULL == param) { + SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + param->rId = pJob->refId; + param->queryId = pJob->queryId; + param->taskId = pTask->taskId; + + if (NULL == pTask->delayTimer) { + pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); + if (NULL == pTask->delayTimer) { + SCH_TASK_ELOG("start delay timer failed"); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void*)param, schMgmt.timer, &pTask->delayTimer); + + return TSDB_CODE_SUCCESS; + } + + SCH_RET(schLaunchTask(pJob, pTask)); +} + int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { SCH_ERR_RET(schChkJobNeedFlowCtrl(pJob, level)); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 1f1288fcfd..a5a15cd1c6 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -36,6 +36,27 @@ FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } +char *schDumpEpSet(SEpSet *pEpSet) { + if (NULL == pEpSet) { + return NULL; + } + + int32_t maxSize = 1024; + char *str = taosMemoryMalloc(maxSize); + if (NULL == str) { + return NULL; + } + + int32_t n = 0; + n += snprintf(str + n, maxSize - n, "numOfEps:%d, inUse:%d eps:", pEpSet->numOfEps, pEpSet->inUse); + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + SEp *pEp = &pEpSet->eps[i]; + n += snprintf(str + n, maxSize - n, "[%s:%d]", pEp->fqdn, pEp->port); + } + + return str; +} + char *schGetOpStr(SCH_OP_TYPE type) { switch (type) { case SCH_OP_NULL: diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c4b45649a4..91c3b5d7a1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -48,6 +48,12 @@ int32_t schedulerInit() { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + schMgmt.timer = taosTmrInit(0, 0, 0, "scheduler"); + if (NULL == schMgmt.timer) { + qError("init timer failed, error:%s", tstrerror(terrno)); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { qError("generate schdulerId failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 43602a607a..ef1cd6556c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -95,6 +95,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") From ba23ed231740c191e20d6fb3decd3ff24467ba64 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 20 Nov 2022 18:53:48 +0800 Subject: [PATCH 03/82] add simple retry --- include/libs/transport/trpc.h | 5 +++ source/libs/transport/inc/transComm.h | 9 +++++ source/libs/transport/inc/transportInt.h | 5 +++ source/libs/transport/src/trans.c | 5 +++ source/libs/transport/src/transCli.c | 46 +++++++++++++++++++----- 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 8cc37910fd..3083e70574 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -85,6 +85,11 @@ typedef struct SRpcInit { int32_t retryLimit; // retry limit int32_t retryInterval; // retry interval ms + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimouet; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int8_t encryption; // encrypt or not diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index ac54749ae1..1102ddd259 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -146,6 +146,15 @@ typedef struct { SCvtAddr cvtAddr; bool setMaxRetry; + int32_t retryMinInterval; + int32_t retryMaxInterval; + int32_t retryStepFactor; + int32_t retryMaxTimeout; + int64_t retryInitTimestamp; + int64_t retryNextInterval; + bool retryInit; + int32_t retryStep; + int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index c8a56081cc..833937aa41 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -52,6 +52,11 @@ typedef struct { int32_t retryLimit; // retry limit int32_t retryInterval; // retry interval ms + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimouet; + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 94bc128de9..415a8766e3 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -51,6 +51,11 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->retryLimit = pInit->retryLimit; pRpc->retryInterval = pInit->retryInterval; + pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval + pRpc->retryStepFactor = pInit->retryStepFactor; + pRpc->retryMaxInterval = pInit->retryMaxInterval; + pRpc->retryMaxTimouet = pInit->retryMaxTimouet; + // register callback handle pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4fb00b1a6d..7581611654 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1371,7 +1371,8 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, pTransInst->retryInterval); + + transDQSched(pThrd->delayQueue, doDelayTask, arg, pCtx->retryNextInterval); } FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { @@ -1419,18 +1420,47 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { int32_t code = pResp->code; bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; + if (retry == true) { + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInit = true; + pCtx->retryStep = 1; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + } else { + pCtx->retryStep++; + int64_t factor = 1; + for (int i = 0; i < pCtx->retryStep - 1; i++) { + factor *= pCtx->retryStepFactor; + } + + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + } + + if (taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + retry = false; + } + } + if (retry) { pMsg->sent = 0; pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, EPSET_GET_SIZE(&pCtx->epSet) * 3); - if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); - EPSET_FORWARD_INUSE(&pCtx->epSet); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } + // if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); + EPSET_FORWARD_INUSE(&pCtx->epSet); + transFreeMsg(pResp->pCont); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; + //} } else { cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, pTransInst->retryLimit); if (pCtx->retryCnt < pCtx->retryLimit) { From 072a73ec3d1d2b3547bc1ba8b2adec3d553978e4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Nov 2022 11:35:52 +0800 Subject: [PATCH 04/82] enh: support max retry wait time configuration --- include/common/tglobal.h | 4 +++ source/common/src/tglobal.c | 8 +++++ source/libs/scheduler/inc/schInt.h | 2 ++ source/libs/scheduler/src/schRemote.c | 11 ++++-- source/libs/scheduler/src/schTask.c | 49 ++++++++++++++++----------- 5 files changed, 51 insertions(+), 23 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2076906f70..d23b1f519a 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -90,6 +90,10 @@ extern int32_t tsQueryNodeChunkSize; extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; +extern int32_t tsRedirectPeriod; +extern int32_t tsRedirectFactor; +extern int32_t tsRedirectMaxPeriod; +extern int32_t tsMaxRetryWaitTime; // client extern int32_t tsMinSlidingTime; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 27dcbd5be3..cb82ad300b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -87,6 +87,10 @@ bool tsQueryPlannerTrace = false; int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; +int32_t tsRedirectPeriod = 100; +int32_t tsRedirectFactor = 5; +int32_t tsRedirectMaxPeriod = 10000; +int32_t tsMaxRetryWaitTime = 60000; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -301,6 +305,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); @@ -645,6 +650,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32; tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32; + tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; return 0; } @@ -860,6 +866,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + } else if (strcasecmp("maxRetryWaitTime", name) == 0) { + tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } break; } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 7cd8d608c2..83b58a77d2 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -27,6 +27,7 @@ extern "C" { #include "tarray.h" #include "thash.h" #include "trpc.h" +#include "ttimer.h" enum { SCH_READ = 1, @@ -507,6 +508,7 @@ extern SSchedulerMgmt schMgmt; void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); +int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); int32_t schReleaseJob(int64_t refId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index a6a2a6c301..4ebe07cf58 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -887,9 +887,14 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo)); SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); - qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, - epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); - + if (pJob && pTask) { + SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, + epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); + } else { + qDebug("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, + epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); + } + if (pTask) { pTask->lastMsgType = msgType; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9262a9257c..0b235e63e9 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -340,7 +340,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { +int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { SSchRedirectCtx *pCtx = &pTask->redirectCtx; if (!pCtx->inRedirect) { pCtx->inRedirect = true; @@ -362,13 +362,6 @@ int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { } pCtx->totalTimes++; - - int64_t nowTs = taosGetTimestampMs(); - if ((nowTs - pCtx->startTs) > tsMaxRetryWaitTime) { - SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", - nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); - SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); - } if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; @@ -382,12 +375,21 @@ int32_t schChkUpdateRedirectCtx(SSchTask *pTask, SEpSet *pEpSet) { pCtx->roundTimes++; if (pCtx->roundTimes >= pCtx->roundTotal) { + int64_t nowTs = taosGetTimestampMs(); + int64_t lastTime = nowTs - pCtx->startTs; + if (lastTime > tsMaxRetryWaitTime) { + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); + } + pCtx->periodMs *= tsRedirectFactor; if (pCtx->periodMs > tsRedirectMaxPeriod) { pCtx->periodMs = tsRedirectMaxPeriod; } - pTask->delayExecMs = pCtx->periodMs; + int64_t leftTime = tsMaxRetryWaitTime - lastTime; + pTask->delayExecMs = leftTime < pCtx->periodMs ? leftTime : pCtx->periodMs; goto _return; } @@ -410,7 +412,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->retryTimes = 0; } - SCH_ERR_JRET(schChkUpdateRedirectCtx(pTask, pData ? pData->pEpSet : NULL)); + SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; @@ -431,6 +433,10 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SCH_SWITCH_EPSET(addr); SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); + } else { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; + SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1141,15 +1147,13 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchJob *pJob = NULL; int32_t code = 0; - SCH_ERR_RET(schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)); + if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { + return; + } - SCH_ERR_JRET(schLaunchTask(pJob, pTask)); + code = schLaunchTask(pJob, pTask); - return; - -_return: - - schHandleJobFailure(pJob, code); + schProcessOnCbEnd(pJob, pTask, code); } int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { @@ -1157,7 +1161,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } param->rId = pJob->refId; @@ -1167,8 +1171,8 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { if (NULL == pTask->delayTimer) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { - SCH_TASK_ELOG("start delay timer failed"); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -1203,7 +1207,12 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { while (pIter) { SSchTask *pTask = *(SSchTask **)pIter; + SCH_LOCK_TASK(pTask); + if (pTask->delayTimer) { + taosTmrStopA(&pTask->delayTimer); + } schDropTaskOnExecNode(pJob, pTask); + SCH_UNLOCK_TASK(pTask); pIter = taosHashIterate(list, pIter); } From e7e17302a4e6a019156d3a9094d022deb2e5e627 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 21 Nov 2022 15:35:36 +0800 Subject: [PATCH 05/82] enh(sync): use TSDB_CODE_SYN_NOT_LEADER instead of TSDB_CODE_RPC_REDIRECT --- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 38cb534d7f..6a0e79902f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -66,7 +66,7 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { } pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; tmsgSendRedirectRsp(&rsp, &newEpSet); } From 62bf1c024a97e17142d39e118cd1282f972f1008 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Nov 2022 21:25:15 +0800 Subject: [PATCH 06/82] refactor retry --- include/libs/transport/trpc.h | 2 +- source/client/src/clientEnv.c | 4 + source/libs/transport/inc/transComm.h | 7 +- source/libs/transport/src/transCli.c | 149 +++++++++++++++----------- 4 files changed, 93 insertions(+), 69 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3083e70574..d761813db1 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -88,7 +88,7 @@ typedef struct SRpcInit { int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval - int32_t retryMaxTimouet; + int64_t retryMaxTimouet; int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int8_t encryption; // encrypt or not diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3f4e1bb513..48ecb5caac 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -148,6 +148,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.dfp = destroyAhandle; rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; + rpcInit.retryMinInterval = 100; + rpcInit.retryStepFactor = 5; + rpcInit.retryMaxInterval = 10240; + rpcInit.retryMaxTimouet = -1; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 1102ddd259..9add91cdeb 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -137,9 +137,6 @@ typedef struct { tmsg_t msgType; // message type int8_t connType; // connection type cli/srv - int8_t retryCnt; - int8_t retryLimit; - STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -149,12 +146,14 @@ typedef struct { int32_t retryMinInterval; int32_t retryMaxInterval; int32_t retryStepFactor; - int32_t retryMaxTimeout; + int64_t retryMaxTimeout; int64_t retryInitTimestamp; int64_t retryNextInterval; bool retryInit; int32_t retryStep; + int8_t epsetRetryCnt; + int hThrdIdx; } STransConnCtx; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7581611654..7ffddda777 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -959,7 +959,7 @@ FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { if (code != 0) return false; - if (pCtx->retryCnt == 0) return false; + // if (pCtx->retryCnt == 0) return false; if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; return true; } @@ -1365,8 +1365,8 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGDebug("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, - pCtx->retryCnt + 1, pCtx->retryLimit); + tGDebug("%s retry on next node,use:%s, step: %d,timeout:%" PRId64 "", transLabel(pThrd->pTransInst), tbuf, + pCtx->retryStep, pCtx->retryNextInterval); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1406,6 +1406,86 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { *dst = epset; return true; } +bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp) { + bool noDelay = true; + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } else { + SEpSet epset; + if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset) < 0) { + // invalid epset + EPSET_FORWARD_INUSE(&pCtx->epSet); + } else if (!transEpSetIsEqual(&pCtx->epSet, &epset)) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } + return noDelay; +} +bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + STransConnCtx* pCtx = pMsg->ctx; + int32_t code = pResp->code; + + bool retry = pTransInst->retry(code, pResp->msgType - 1); + if (retry == false) { + return false; + } + + bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + noDelay = cliResetEpset(pCtx, pResp); + transFreeMsg(pResp->pCont); + transUnrefCliHandle(pConn); + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + noDelay = cliResetEpset(pCtx, pResp); + transFreeMsg(pResp->pCont); + addConnToPool(pThrd->pool, pConn); + } else { + noDelay = cliResetEpset(pCtx, pResp); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); + } + + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 1; + pCtx->retryInit = true; + } else { + if (noDelay == false) { + pCtx->epsetRetryCnt = 0; + pCtx->retryStep++; + + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; + } + } + + cliSchedMsgToNextNode(pMsg, pThrd); + return false; +} int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -1419,69 +1499,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; + bool retry = cliGenRetryRule(pConn, pResp, pMsg); if (retry == true) { - if (!pCtx->retryInit) { - pCtx->retryMinInterval = pTransInst->retryMinInterval; - pCtx->retryMaxInterval = pTransInst->retryMaxInterval; - pCtx->retryStepFactor = pTransInst->retryStepFactor; - pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; - pCtx->retryInit = true; - pCtx->retryStep = 1; - pCtx->retryInitTimestamp = taosGetTimestampMs(); - pCtx->retryNextInterval = pCtx->retryMinInterval; - } else { - pCtx->retryStep++; - int64_t factor = 1; - for (int i = 0; i < pCtx->retryStep - 1; i++) { - factor *= pCtx->retryStepFactor; - } - - pCtx->retryNextInterval = factor * pCtx->retryMinInterval; - if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { - pCtx->retryNextInterval = pCtx->retryMaxInterval; - } - } - - if (taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - retry = false; - } + return -1; } - - if (retry) { - pMsg->sent = 0; - pCtx->retryCnt += 1; - - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK) { - cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, EPSET_GET_SIZE(&pCtx->epSet) * 3); - // if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); - EPSET_FORWARD_INUSE(&pCtx->epSet); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - //} - } else { - cliCompareAndSwap(&pCtx->retryLimit, pTransInst->retryLimit, pTransInst->retryLimit); - if (pCtx->retryCnt < pCtx->retryLimit) { - if (pResp->contLen == 0) { - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else { - if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet) < 0) { - tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn), pConn); - } - } - addConnToPool(pThrd->pool, pConn); - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } else { - // change error code for taos client driver if retryCnt exceeds limit - if (0 == strncmp(pTransInst->label, "TSC", strlen("TSC"))) pResp->code = TSDB_CODE_APP_NOT_READY; - } - } - } - STraceId* trace = &pResp->info.traceId; bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); From 963d56cfc19ac7aa8883d512c3f674b62dcf381b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 22 Nov 2022 11:10:28 +0800 Subject: [PATCH 07/82] retry code --- include/util/taoserror.h | 1 + source/libs/transport/src/transCli.c | 45 +++++++++++++++++----------- source/util/src/terror.c | 1 + 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 26f22f398c..f9a6816c5c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -415,6 +415,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_PROPOSE_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0911) #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) +#define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7ffddda777..d3a2d45d88 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1406,24 +1406,33 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { *dst = epset; return true; } -bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp) { +bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; - if (pResp->contLen == 0) { - if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { - noDelay = false; - } else { - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (hasEpSet == false) { + assert(pResp->contLen == 0); + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } } else { - SEpSet epset; - if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset) < 0) { - // invalid epset - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else if (!transEpSetIsEqual(&pCtx->epSet, &epset)) { - noDelay = false; + SEpSet epSet; + + assert(pResp->contLen == sizeof(epSet)); + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + assert(0); + } + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; } else { + tDebug("epset equal, continue"); EPSET_FORWARD_INUSE(&pCtx->epSet); } + noDelay = false; } return noDelay; } @@ -1440,18 +1449,20 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - noDelay = cliResetEpset(pCtx, pResp); + noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { - noDelay = cliResetEpset(pCtx, pResp); + noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); + } else if (code == TSDB_CODE_SYN_RESTORING) { + noDelay = cliResetEpset(pCtx, pResp, false); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); } else { - noDelay = cliResetEpset(pCtx, pResp); - addConnToPool(pThrd->pool, pConn); - transFreeMsg(pResp->pCont); } if (!pCtx->retryInit) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ef1cd6556c..be41a71535 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -407,6 +407,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for re TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq From c84f9f86340b96405fb22e585f2119d802d304ba Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 18:29:31 +0800 Subject: [PATCH 08/82] add debug info --- source/libs/scheduler/src/schTask.c | 30 +++++++++++++++------------- source/libs/transport/src/transCli.c | 8 +++++--- source/libs/transport/src/transSvr.c | 18 ++++++++--------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 0b235e63e9..af78647018 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -346,7 +346,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) pCtx->inRedirect = true; pCtx->periodMs = tsRedirectPeriod; pCtx->startTs = taosGetTimestampMs(); - + if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; @@ -360,7 +360,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) goto _return; } - + pCtx->totalTimes++; if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { @@ -378,8 +378,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) int64_t nowTs = taosGetTimestampMs(); int64_t lastTime = nowTs - pCtx->startTs; if (lastTime > tsMaxRetryWaitTime) { - SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", - nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); + SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", + nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); } @@ -398,8 +398,9 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) _return: - SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, pCtx->totalTimes, pTask->delayExecMs); - + SCH_TASK_DLOG("task start %d/%d/%d redirect retry, delayExec:%d", pCtx->roundTimes, pCtx->roundTotal, + pCtx->totalTimes, pTask->delayExecMs); + return TSDB_CODE_SUCCESS; } @@ -415,7 +416,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL)); pTask->waitRetry = true; - + schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); @@ -435,8 +436,9 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } else { SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; - SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps, pEp->fqdn, pEp->port); + SEp *pEp = &addr->epSet.eps[addr->epSet.inUse]; + SCH_TASK_DLOG("task retry node %d current ep, idx:%d/%d,%s:%d", addr->nodeId, addr->epSet.inUse, + addr->epSet.numOfEps, pEp->fqdn, pEp->port); } if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -1146,7 +1148,7 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchTask *pTask = NULL; SSchJob *pJob = NULL; int32_t code = 0; - + if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { return; } @@ -1160,15 +1162,15 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { if (pTask->delayExecMs > 0) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { - SCH_TASK_ELOG("taosMemoryMalloc %d failed", sizeof(SSchTimerParam)); + SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + param->rId = pJob->refId; param->queryId = pJob->queryId; param->taskId = pTask->taskId; - if (NULL == pTask->delayTimer) { + if (NULL == pTask->delayTimer) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); @@ -1178,7 +1180,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } - taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void*)param, schMgmt.timer, &pTask->delayTimer); + taosTmrReset(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer, &pTask->delayTimer); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d3a2d45d88..8b65794475 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1035,12 +1035,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { addr.sin_family = AF_INET; addr.sin_addr.s_addr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, conn->ip); addr.sin_port = (uint16_t)htons((uint16_t)conn->port); - tTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); + + STraceId* trace = &(pMsg->msg.info.traceId); + tGTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); int ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - tTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, - uv_err_name(ret)); + tGTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, + uv_err_name(ret)); uv_timer_stop(conn->timer); conn->timer->data = NULL; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5f36d91023..374d445874 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1233,17 +1233,17 @@ int transReleaseSrvHandle(void* handle) { m->msg = tmsg; m->type = Release; - tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); + tDebug("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to send to release handle", exh); + tDebug("handle %p failed to send to release handle", exh); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to send to release handle", exh); + tDebug("handle %p failed to send to release handle", exh); return -1; } int transSendResponse(const STransMsg* msg) { @@ -1268,19 +1268,19 @@ int transSendResponse(const STransMsg* msg) { m->type = Normal; STraceId* trace = (STraceId*)&msg->info.traceId; - tGTrace("conn %p start to send resp (1/2)", exh->handle); + tGDebug("conn %p start to send resp (1/2)", exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to send resp", exh); + tDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to send resp", exh); + tDebug("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); return -1; } @@ -1302,19 +1302,19 @@ int transRegisterMsg(const STransMsg* msg) { m->type = Register; STrans* pTransInst = pThrd->pTransInst; - tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); + tDebug("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle); transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(transGetRefMgt(), refId); return 0; _return1: - tTrace("handle %p failed to register brokenlink", exh); + tDebug("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); transReleaseExHandle(transGetRefMgt(), refId); return -1; _return2: - tTrace("handle %p failed to register brokenlink", exh); + tDebug("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); return -1; } From be410b9903f2380ed55c7af1f377192a12c54016 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 23 Nov 2022 21:33:11 +0800 Subject: [PATCH 09/82] add debug info --- source/client/src/clientEnv.c | 2 +- source/libs/transport/src/transCli.c | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 48ecb5caac..4eb6eb0458 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -151,7 +151,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.retryMinInterval = 100; rpcInit.retryStepFactor = 5; rpcInit.retryMaxInterval = 10240; - rpcInit.retryMaxTimouet = -1; + rpcInit.retryMaxTimouet = 20480; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b65794475..160e4ca8e6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -234,7 +234,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) -#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps != 0) +#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps >= 0 && (epSet)->inUse >= 0) #define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) @@ -996,9 +996,14 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); + + char tbuf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, tbuf); + tDebug("current epset %s", tbuf); + if (!EPSET_IS_VALID(&pCtx->epSet)) { - destroyCmsg(pMsg); tError("invalid epset"); + destroyCmsg(pMsg); return; } @@ -1453,18 +1458,25 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { bool noDelay = false; if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); } else if (code == TSDB_CODE_SYN_RESTORING) { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } else { + tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + addConnToPool(pThrd->pool, pConn); + transFreeMsg(pResp->pCont); } if (!pCtx->retryInit) { @@ -1495,9 +1507,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->epsetRetryCnt++; } } - + pMsg->sent = 0; cliSchedMsgToNextNode(pMsg, pThrd); - return false; + return true; } int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; From febe918d36db31420644451276c36d953a390873 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 10:28:38 +0800 Subject: [PATCH 10/82] add debug info --- source/libs/transport/src/transCli.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 160e4ca8e6..813a7983e4 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1427,10 +1427,10 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { } else { SEpSet epSet; - assert(pResp->contLen == sizeof(epSet)); + // assert(pResp->contLen == sizeof(epSet)); int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); if (valid < 0) { - assert(0); + // assert(0); } if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); @@ -1454,6 +1454,11 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (retry == false) { return false; } + if (pCtx->retryInit) { + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } bool noDelay = false; From fbba39ad1750082e8d89489fc4ade99397cdc153 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 24 Nov 2022 17:19:15 +0800 Subject: [PATCH 11/82] add parameter to rpc --- source/client/src/clientEnv.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 4eb6eb0458..6f1414b72d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -148,10 +148,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.dfp = destroyAhandle; rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; - rpcInit.retryMinInterval = 100; - rpcInit.retryStepFactor = 5; - rpcInit.retryMaxInterval = 10240; - rpcInit.retryMaxTimouet = 20480; + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { From 3fd6de0c53e2b6b8304f14a5f4f35508d2e7b571 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Nov 2022 14:20:30 +0800 Subject: [PATCH 12/82] change code --- source/libs/transport/src/transCli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 813a7983e4..061ae9cb57 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1467,7 +1467,8 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { noDelay = cliResetEpset(pCtx, pResp, false); transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); - } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR) { + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || + code == TSDB_CODE_SYN_PROPOSE_NOT_READY) { tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 55af567ccc8e415be2e356eaaf39b6eabfab5b01 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Nov 2022 14:47:06 +0800 Subject: [PATCH 13/82] change code --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 061ae9cb57..8b16c177f7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -367,7 +367,7 @@ void cliHandleResp(SCliConn* conn) { STraceId* trace = &transMsg.info.traceId; tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, - TMSG_INFO(pHead->msgType), conn->dst, conn->src, msgLen, tstrerror(transMsg.code)); + TMSG_INFO(pHead->msgType), conn->dst, conn->src, pHead->msgLen, tstrerror(transMsg.code)); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); From 294af08f7a23724b249468b12168600c01583358 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 27 Nov 2022 16:04:36 +0800 Subject: [PATCH 14/82] fix(query): handle countAlwaysReturnValue behavior when input data is empty --- source/libs/executor/src/executorimpl.c | 76 ++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4319dd379a..a743465755 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1595,6 +1595,62 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) { + if (!tsCountAlwaysReturnValue) { + return TSDB_CODE_SUCCESS; + } + + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + bool hasCountFunc = false; + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0)) { + hasCountFunc = true; + break; + } + } + + if (!hasCountFunc) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = createDataBlock(); + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + pBlock->info.rowSize = 0; + pBlock->info.groupId = 0; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SColumnInfoData pCol = {0}; + pCol.hasNull = true; + pCol.info.type = TSDB_DATA_TYPE_NULL; + + SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; + for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { + SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { + int32_t slotId = pFuncParam->pCol->slotId; + taosArrayPush(pBlock->pDataBlock, &pCol); + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + } + } + } + + *ppBlock = pBlock; + + return TSDB_CODE_SUCCESS; +} + +static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) { + if (!blockAllocated) { + return; + } + + blockDataDestroy(*ppBlock); + *ppBlock = NULL; +} + + // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { @@ -1612,14 +1668,27 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; + bool hasValidBlock = false; + bool blockAllocated = false; + while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - break; + if (!hasValidBlock) { + createDataBlockForEmptyInput(pOperator, &pBlock); + if (pBlock == NULL) { + break; + } + blockAllocated = true; + } else { + break; + } } + hasValidBlock = true; int32_t code = getTableScanInfo(pOperator, &order, &scanFlag); if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } @@ -1628,6 +1697,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SExprSupp* pSup1 = &pAggInfo->scalarExprSup; code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } } @@ -1637,8 +1707,12 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { setInputDataBlock(pSup, pBlock, order, scanFlag, true); code = doAggregateImpl(pOperator, pSup->pCtx); if (code != 0) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); T_LONG_JMP(pTaskInfo->env, code); } + + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + } initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); From c019645193f83fec554614f25540cfc5a3f5a79f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 27 Nov 2022 20:38:39 +0800 Subject: [PATCH 15/82] fix bugs --- source/libs/executor/src/executorimpl.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a743465755..402f267c6d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1615,27 +1615,35 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc } SSDataBlock* pBlock = createDataBlock(); - pBlock->info.rows = 0; + pBlock->info.rows = 1; pBlock->info.capacity = 0; - pBlock->info.rowSize = 0; pBlock->info.groupId = 0; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SColumnInfoData pCol = {0}; - pCol.hasNull = true; - pCol.info.type = TSDB_DATA_TYPE_NULL; + SColumnInfoData colInfo = {0}; + colInfo.hasNull = true; + colInfo.info.type = TSDB_DATA_TYPE_NULL; + colInfo.info.bytes = 1; SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - taosArrayPush(pBlock->pDataBlock, &pCol); + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + if (slotId >= numOfCols) { + taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); + for (int32_t k = numOfCols; k < slotId + 1; ++k) { + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + } } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + // do nothing } } } + blockDataEnsureCapacity(pBlock, pBlock->info.rows); *ppBlock = pBlock; return TSDB_CODE_SUCCESS; From ab86b6ba47410ce27585960a92d69c3e48ed51cb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 00:30:39 +0800 Subject: [PATCH 16/82] fix hyperloglog bugs --- include/libs/function/function.h | 2 +- source/libs/executor/src/executorimpl.c | 4 +++- source/libs/function/src/builtinsimpl.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 6f2a675466..0831e477ff 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -57,7 +57,7 @@ typedef struct SFuncExecFuncs { #define MAX_INTERVAL_TIME_WINDOW 10000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 -#define FUNCTIONS_NAME_MAX_LENGTH 16 +#define FUNCTIONS_NAME_MAX_LENGTH 32 typedef struct SResultRowEntryInfo { bool initialized : 1; // output buffer has been initialized diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 402f267c6d..1f0ccf02ce 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1604,7 +1604,9 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc bool hasCountFunc = false; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0)) { + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) || + (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) { hasCountFunc = true; break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 35f50cebca..141cd04263 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4787,6 +4787,10 @@ int32_t hllFunction(SqlFunctionCtx* pCtx) { int32_t numOfRows = pInput->numOfRows; int32_t numOfElems = 0; + if (IS_NULL_TYPE(type)) { + goto _hll_over; + } + for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_s(pCol, i)) { continue; @@ -4808,6 +4812,7 @@ int32_t hllFunction(SqlFunctionCtx* pCtx) { } } +_hll_over: pInfo->totalCount += numOfElems; if (pInfo->totalCount == 0 && !tsCountAlwaysReturnValue) { From 4c158238bab5d25548e484abcb0cfaea03ad1bba Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 01:00:34 +0800 Subject: [PATCH 17/82] add test cases --- tests/system-test/2-query/count.py | 144 +++++++++++++++++- .../2-query/countAlwaysReturnValue.py | 68 +++++++++ 2 files changed, 204 insertions(+), 8 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 254a8792af..8962cf305a 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -51,6 +51,136 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],rownum) tdSql.query(f'select count({k}) from {ntbname} where ts <={self.ts+self.rowNum-2}') tdSql.checkEqual(tdSql.queryResult[0][0],rownum-1) + def query_empty_stb(self): + tdSql.query(f'select count(*) from (select distinct tbname from {self.stbname})') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'select count(*) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + function_names = ['count', 'hyperloglog'] + for function_name in function_names: + tdSql.query(f'select {function_name}(tbname) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(ts) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c2),max(1) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.query(f'select sum(1),{function_name}(1),max(c2) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, None) + tdSql.query(f'select {function_name}(1),sum(1),max(c2),min(1),min(2),min(3),min(4),min(5),min(6),min(7),min(8) from {self.stbname}') + tdSql.checkRows(1) + tdSql.checkCols(11) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 10, None) + tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') + tdSql.checkRows(2) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') + tdSql.checkRows(0) + + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 2) + tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 2) + def query_empty_ntb(self): + tdSql.query(f'select count(*) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + function_names = ['count', 'hyperloglog'] + for function_name in function_names: + tdSql.query(f'select {function_name}(tbname) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(ts) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c2),max(1) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.query(f'select sum(1),{function_name}(1),max(c2) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(3) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, None) + tdSql.query(f'select {function_name}(1),sum(1),max(c2),min(1),min(2),min(3),min(4),min(5),min(6),min(7),min(8) from {self.ntbname}') + tdSql.checkRows(1) + tdSql.checkCols(11) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 10, None) + tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by tbname interval(1s)') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') + tdSql.checkRows(0) + + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) @@ -81,11 +211,11 @@ class TDTestCase: def check_ntb(self): tdSql.prepare() tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) - tdSql.query(f'select count(tbname) from {self.ntbname}') - tdSql.checkRows(0) + self.query_empty_ntb() tdSql.execute('flush database db') tdSql.query(f'select count(tbname) from {self.ntbname}') - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) self.insert_data(self.column_dict,self.ntbname,self.rowNum) self.count_query_ntb(self.column_dict,self.ntbname,self.rowNum) tdSql.execute('flush database db') @@ -96,13 +226,11 @@ class TDTestCase: tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) for i in range(self.tbnum): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') - tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') - tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) - tdSql.query(f'select count(tbname) from {self.stbname}') - tdSql.checkRows(0) + self.query_empty_stb() tdSql.execute('flush database db') tdSql.query(f'select count(tbname) from {self.stbname}') - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) for i in range(self.tbnum): diff --git a/tests/system-test/2-query/countAlwaysReturnValue.py b/tests/system-test/2-query/countAlwaysReturnValue.py index 08c5ae104f..244964e486 100644 --- a/tests/system-test/2-query/countAlwaysReturnValue.py +++ b/tests/system-test/2-query/countAlwaysReturnValue.py @@ -33,6 +33,19 @@ class TDTestCase: f"create table {dbname}.ctb2 using {dbname}.stb tags (2)" ) + tdSql.execute( + f"create table {dbname}.tb_empty (ts timestamp, c0 int)" + ) + tdSql.execute( + f"create table {dbname}.stb_empty (ts timestamp, c0 int) tags (t0 int)" + ) + tdSql.execute( + f"create table {dbname}.ctb1_empty using {dbname}.stb tags (1)" + ) + tdSql.execute( + f"create table {dbname}.ctb2_empty using {dbname}.stb tags (2)" + ) + tdSql.execute( f"insert into {dbname}.tb values (now(), NULL)") @@ -94,6 +107,61 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + # test empty table/input + tdSql.query(f"select count(*) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(ts) from {dbname}.stb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(c0) from {dbname}.ctb1 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(1) from {dbname}.ctb2 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(*) from {dbname}.tb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(ts) from {dbname}.stb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(c0) from {dbname}.ctb1_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(1) from {dbname}.ctb2_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(c0) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(ts) from {dbname}.stb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb1 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb2 where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(c0) from {dbname}.tb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(ts) from {dbname}.stb_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb1_empty") + tdSql.checkRows(0) + + tdSql.query(f"select hyperloglog(1) from {dbname}.ctb2_empty") + tdSql.checkRows(0) + + tdSql.query(f"select count(*), hyperloglog(c0), sum(1), max(c0) from {dbname}.tb where ts > now + 1h") + tdSql.checkRows(0) + + tdSql.query(f"select count(*), hyperloglog(c0), sum(1), max(c0) from {dbname}.tb_empty") + tdSql.checkRows(0) + def run(self): tdSql.prepare() From f3c1eb829e9f071ebbbde1d131a19b391f059fe8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 10:10:11 +0800 Subject: [PATCH 18/82] refactor retry --- source/libs/transport/src/transCli.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8b16c177f7..2f0b9609f3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1425,21 +1425,26 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { } } } else { - SEpSet epSet; - - // assert(pResp->contLen == sizeof(epSet)); + SEpSet epSet; int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); if (valid < 0) { - // assert(0); - } - if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { - tDebug("epset not equal, retry new epset"); - pCtx->epSet = epSet; + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + noDelay = true; + } } else { - tDebug("epset equal, continue"); - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + noDelay = false; } - noDelay = false; } return noDelay; } From 60ee8db228df0bfefa81bcbf8d4e6b85d00bffac Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 11:19:56 +0800 Subject: [PATCH 19/82] fix:add log for sml --- source/client/src/clientSml.c | 6 +++++- source/libs/parser/src/parInsertSml.c | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 83bbe392cf..6773cfc638 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -163,6 +163,7 @@ typedef struct { SMLProtocolType protocol; int8_t precision; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; SHashObj *childTables; SHashObj *superTables; @@ -863,6 +864,7 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra } else { ASSERT(0); } + uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); if (ts == -1) return TSDB_CODE_INVALID_TIMESTAMP; @@ -2063,7 +2065,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql", info->id); + uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s", info->id, (info->isRawLine ? "rawdata" : sql)); int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { @@ -2575,6 +2577,8 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } + info->isRawLine = (rawLine == NULL); + int32_t perBatch = LINE_BATCH; if (numLines > perBatch) { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index d18b11ad57..7bcfd0646f 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -303,9 +303,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols if (kv) { int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - // uError("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - // uError("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); } if (IS_VAR_DATA_TYPE(kv->type)) { From bd84ecc3618631fc3b45a8b8653d948641d78231 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 16:31:33 +0800 Subject: [PATCH 20/82] fix:[TD-20613] add interface for write raw block with sdfasdf --- include/client/taos.h | 1 + include/os/os.h | 3 +- source/client/src/clientImpl.c | 2 +- source/client/src/clientRawBlockWrite.c | 206 ++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 25887b2879..677c830ff7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -297,6 +297,7 @@ DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); +DLL_EXPORT int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields); DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); diff --git a/include/os/os.h b/include/os/os.h index ab4d0a406e..629b0e27c6 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -43,12 +43,13 @@ extern "C" { #include #include #include -#include #if defined(DARWIN) #else #include #include +#include + #endif #else diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3140371c4..ec94520fb0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1645,7 +1645,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - // version + length + numOfRows + numOfCol + groupId + flag_segment + column_info + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 6a6a5d56d7..ea0b900e68 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1211,6 +1211,211 @@ static void destroyVgHash(void* data) { taosMemoryFreeClear(vgData->data); } +int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){ + int32_t code = TSDB_CODE_SUCCESS; + STableMeta* pTableMeta = NULL; + SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; + + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); + if (!pRequest) { + uError("WriteRaw:createRequest error request is null"); + code = terrno; + goto end; + } + + pRequest->syncQuery = true; + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); + + struct SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + SVgroupInfo vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname); + goto end; + } + + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); + goto end; + } + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + int32_t numOfCols = numFields; + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for(int k = 0; k < numOfCols; k++){ + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + SSchema* schema = pTableMeta->schema + i; + if(strcmp(schema->name, fields[k].name) != 0) continue; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if (IS_VAR_DATA_TYPE(schema->type)) { + nVar++; + } + } + } + + fLen -= sizeof(TSKEY); + + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); + int32_t schemaLen = 0; + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + subReq = taosMemoryCalloc(1, totalLen); + SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTableMeta->sversion); + tdSRowSetTpInfo(&rb, numOfCols, fLen); + int32_t dataLen = 0; + + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); + int32_t* colLength = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + pCol[i].offset = (int32_t*)pStart; + pStart += rows * sizeof(int32_t); + } else { + pCol[i].nullbitmap = pStart; + pStart += BitmapLen(rows); + } + + pCol[i].pData = pStart; + pStart += colLength[i]; + } + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + int32_t offset = 0; + for (int32_t k = 0; k < numOfCols; k++) { + const SSchema* pColumn = NULL; + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + if (strcmp((pTableMeta->schema + i)->name, fields[k].name) == 0) { + pColumn = pTableMeta->schema + i; + break; + } + } + if(pColumn == NULL){ + uError("column not exist:%s", fields[k].name); + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } else { + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } + + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + offset += TYPE_BYTES[pColumn->type]; + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + + taosMemoryFree(pCol); + + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->sversion = htonl(pTableMeta->sversion); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htonl(rows); + blk->dataLen = htonl(dataLen); + subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; + subReq->numOfBlocks = 1; + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); + nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vgData; + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + subReq = NULL; // no need free + taosArrayPush(nodeStmt->pDataBlocks, &dst); + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + + end: + taosMemoryFreeClear(pTableMeta); + qDestroyQuery(pQuery); + taosMemoryFree(subReq); + return code; +} + int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; @@ -1293,6 +1498,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); int32_t* colLength = (int32_t*)pStart; pStart += sizeof(int32_t) * numOfCols; From 9b5d78d23dbb9eaf3a3135a57c2bdc69eb25bbbe Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 28 Nov 2022 16:32:06 +0800 Subject: [PATCH 21/82] fix:[TD-20613] add interface for write raw block with some fields --- source/client/src/clientRawBlockWrite.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index ea0b900e68..bcb97b7354 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1294,6 +1294,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SRowBuilder rb = {0}; tdSRowInit(&rb, pTableMeta->sversion); + rb.rowType = TD_ROW_KV; tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; From 4b668a415326165c741a8c9d350057e59a570d84 Mon Sep 17 00:00:00 2001 From: MysticBoy Date: Mon, 28 Nov 2022 17:47:37 +0800 Subject: [PATCH 22/82] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E8=AF=BB=E5=8F=96=E9=A2=9D=E5=A4=96=E5=86=85?= =?UTF-8?q?=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index fa63543439..daf80ad915 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2320,7 +2320,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - pColInfoData->info.type = *(int16_t*)pStart; + pColInfoData->info.type = *(int8_t*)pStart; pStart += sizeof(int8_t); pColInfoData->info.bytes = *(int32_t*)pStart; From f81ce17df43575fb70fda83cdbac863d22615045 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:06:02 +0800 Subject: [PATCH 23/82] fix first last --- source/libs/function/src/builtinsimpl.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a2e653469f..ea976c9a79 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2936,6 +2936,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = pInputCol->info.bytes; + if (IS_NULL_TYPE(pInputCol->info.type)) { + return 0; + } + // All null data column, return directly. if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); @@ -3040,6 +3044,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; + if (IS_NULL_TYPE(type)) { + return 0; + } + // All null data column, return directly. if (pInput->colDataSMAIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); From f5db4a8b400a4a5571708a4c0c63defe71554bd0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 22:08:21 +0800 Subject: [PATCH 24/82] change paramter --- source/libs/transport/src/transCli.c | 76 +++++++++++++++------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2f0b9609f3..1f3ee3efaf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1416,13 +1416,19 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; if (hasEpSet == false) { - assert(pResp->contLen == 0); + // assert(pResp->contLen == 0); if (pResp->contLen == 0) { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); } + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } } else { SEpSet epSet; @@ -1433,17 +1439,19 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); - noDelay = true; } } else { if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); pCtx->epSet = epSet; } else { - tDebug("epset equal, continue"); - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } - noDelay = false; } } return noDelay; @@ -1459,14 +1467,22 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (retry == false) { return false; } - if (pCtx->retryInit) { - if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - return false; - } + + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pTransInst->retryMinInterval; + pCtx->retryMaxInterval = pTransInst->retryMaxInterval; + pCtx->retryStepFactor = pTransInst->retryStepFactor; + pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 1; + pCtx->retryInit = true; + } + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; } bool noDelay = false; - if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { tDebug("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, false); @@ -1490,34 +1506,24 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); } - if (!pCtx->retryInit) { - pCtx->retryMinInterval = pTransInst->retryMinInterval; - pCtx->retryMaxInterval = pTransInst->retryMaxInterval; - pCtx->retryStepFactor = pTransInst->retryStepFactor; - pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; - pCtx->retryInitTimestamp = taosGetTimestampMs(); - pCtx->retryNextInterval = pCtx->retryMinInterval; - pCtx->retryStep = 1; - pCtx->retryInit = true; - } else { - if (noDelay == false) { - pCtx->epsetRetryCnt = 0; - pCtx->retryStep++; + if (noDelay == false) { + pCtx->epsetRetryCnt = 1; + pCtx->retryStep++; - int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); - pCtx->retryNextInterval = factor * pCtx->retryMinInterval; - if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { - pCtx->retryNextInterval = pCtx->retryMaxInterval; - } - - if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - return false; - } - } else { - pCtx->retryNextInterval = 0; - pCtx->epsetRetryCnt++; + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; } + + if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return false; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; } + pMsg->sent = 0; cliSchedMsgToNextNode(pMsg, pThrd); return true; From 91b09893bf61737069c13d8086cfa27dc9439ce3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 28 Nov 2022 22:16:00 +0800 Subject: [PATCH 25/82] change paramter --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1f3ee3efaf..d0b8adeffe 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1489,7 +1489,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT) { tDebug("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 8c90a98e2d1371289cf42e87ab1f1f3a5b688ef9 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:25:17 +0800 Subject: [PATCH 26/82] fix last_row --- source/libs/function/src/builtinsimpl.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ea976c9a79..19804bea09 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2937,7 +2937,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = pInputCol->info.bytes; if (IS_NULL_TYPE(pInputCol->info.type)) { - return 0; + return TSDB_CODE_SUCCESS; } // All null data column, return directly. @@ -2945,7 +2945,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { ASSERT(pInputCol->hasNull == true); // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); - return 0; + return TSDB_CODE_SUCCESS; } SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL; @@ -3045,7 +3045,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { pInfo->bytes = bytes; if (IS_NULL_TYPE(type)) { - return 0; + return TSDB_CODE_SUCCESS; } // All null data column, return directly. @@ -3053,7 +3053,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { ASSERT(pInputCol->hasNull == true); // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); - return 0; + return TSDB_CODE_SUCCESS; } SColumnDataAgg* pColAgg = (pInput->colDataSMAIsSet) ? pInput->pColumnDataAgg[0] : NULL; @@ -3318,9 +3318,14 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; + if (IS_NULL_TYPE(type)) { + return TSDB_CODE_SUCCESS; + } + TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); From 05975114607040b9cae3b5f26a9b97d2a2c6164d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:54:27 +0800 Subject: [PATCH 27/82] fix scalar error --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5dbd728cce..7dc95ab26f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1696,7 +1696,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } // there is an scalar expression that needs to be calculated before apply the group aggregation. - if (pAggInfo->scalarExprSup.pExprInfo != NULL) { + if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { SExprSupp* pSup1 = &pAggInfo->scalarExprSup; code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { From 94c8ba64d27a6f8b6ecc760063f14d61c017558f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 22:55:16 +0800 Subject: [PATCH 28/82] fix test cases --- tests/script/tsim/parser/join.sim | 6 +++--- tests/system-test/2-query/distribute_agg_count.py | 2 +- tests/system-test/2-query/last_row.py | 3 ++- tests/system-test/2-query/mavg.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/script/tsim/parser/join.sim b/tests/script/tsim/parser/join.sim index 8ad5946a54..a2ae25014e 100644 --- a/tests/script/tsim/parser/join.sim +++ b/tests/script/tsim/parser/join.sim @@ -496,18 +496,18 @@ sql create table tm0 using m1 tags('abc', 1); sql create table m2(ts timestamp, k int) tags(a int, b binary(12)); sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; -if $rows != 0 then +if $rows != 1 then return -1 endi sql create table tm2 using m2 tags(2, 'abc'); sql select count(*) from tm0, tm2 where tm0.ts=tm2.ts; -if $rows != 0 then +if $rows != 1 then return -1 endi sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index 7d131cd77d..67d47d7da5 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -183,7 +183,7 @@ class TDTestCase: tdSql.checkRows(20) tdSql.query(f"select count(c1) from {dbname}.stb1 where t1> 4 partition by tbname") - tdSql.checkRows(15) + tdSql.checkRows(16) # union all tdSql.query(f"select count(c1) from {dbname}.stb1 union all select count(c1) from {dbname}.stb1 ") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 01da658989..a6bcc2c5f1 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -402,7 +402,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f"select count(c1) from {dbname}.ct4 where t1 = 1 ") - tdSql.checkRows(0) + tdSql.checkRows(1) + tdSql.checkData(0,0,0) tdSql.query(f"select last_row(c1) ,last(c1) from {dbname}.stb1 where c1 is null") tdSql.checkRows(1) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index f76980106d..d659a682b1 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -298,7 +298,7 @@ class TDTestCase: if (platform.system().lower() == 'windows' and pre_result.dtype == 'int32'): pre_result = np.array(pre_result, dtype = 'int64') - pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k + #pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k tdSql.query(self.mavg_query_form( sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, table_expr=table_expr, condition=condition From daeeacb45032c5abfd01da5cee45889bb60460ac Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 28 Nov 2022 23:54:42 +0800 Subject: [PATCH 29/82] fix compilation error --- source/libs/executor/src/executorimpl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7bddc29530..094060b213 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1443,7 +1443,6 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pBlock = createDataBlock(); pBlock->info.rows = 1; pBlock->info.capacity = 0; - pBlock->info.groupId = 0; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SColumnInfoData colInfo = {0}; From 80b34c6b56f002ca8a86e34cb1ca11a6fbd44fc5 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 29 Nov 2022 01:49:59 +0800 Subject: [PATCH 30/82] test: add test cases --- tests/parallel_test/cases.task | 1 + tests/system-test/99-TDcase/TD-20582.py | 93 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 tests/system-test/99-TDcase/TD-20582.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0abec6b340..41a050a7c5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1016,6 +1016,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py #develop test ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py diff --git a/tests/system-test/99-TDcase/TD-20582.py b/tests/system-test/99-TDcase/TD-20582.py new file mode 100644 index 0000000000..9c54a1c0cc --- /dev/null +++ b/tests/system-test/99-TDcase/TD-20582.py @@ -0,0 +1,93 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self, dbname="db"): + + tdSql.execute( + f''' CREATE TABLE ac_stb (TS TIMESTAMP, C1 INT, C2 BIGINT, C3 FLOAT, C4 DOUBLE, C5 BINARY(10), C6 BOOL, + C7 SMALLINT, C8 TINYINT, C9 NCHAR(10)) TAGS (T1 INT); + ''' + ) + + tdSql.execute( + f''' insert into ctb0 using ac_stb tags (1) values ( 1537146001000 , 1,1,1,1,'bin',1,1,1,'________') + ( 1537146002000 , 2,2,2,2,'binar', 1,1,1,'nchar'); + ''' + ) + + tdSql.execute( + f''' insert into ntb0 using ac_stb tags (-1) values ( 1537146001000 , 1,1,1,1,'bin',1,1,1,'________') + ( 1537146002000 , 2,2,2,2,'binar', 1,1,1,'nchar'); + ''' + ) + + tdSql.execute( + f''' insert into ntb0 using ac_stb tags (2) values ( 1537146003000 , 1,1,1,1,'bin',1,1,1,'________') + ( 1537146004000 , 2,2,2,2,'binar', 1,1,1,'nchar'); + ''' + ) + + tdSql.execute( + f''' insert into ctb6 using ac_stb tags(1) values ( 1537146000000 , 1, 1, 1, 1, 'bin1', 1, 1, 1, '________1') + ctb6 using ac_stb tags(2) values ( 1537146000000 , 2, 2, 2, 2, 'bin2', 2, 2, 2, '________2') + ctb6 using ac_stb tags(3) values ( 1537146000000 , 3, 3, 3, 3, 'bin3', 3, 3, 3, '________3') + ''' + ) + + + def check_result(self, dbname="db"): + tdSql.query("select c1,c1,c2,c3,c4,c5,c7,c8,c9 from ac_stb") + tdSql.checkRows(7) + + tdSql.query("select t1, count(*), first(c9) from ac_stb partition by t1 order by t1 asc slimit 3") + tdSql.checkRows(2) + + # TD-20582 + tdSql.query("explain analyze verbose true select count(*) from ac_stb where T1=1") + tdSql.checkRows(16) + + # TD-20581 + tdSql.execute("insert into ntb0 select * from ntb0") + tdSql.query("select * from ntb0") + tdSql.checkRows(4) + + return + # basic query + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("========== step1: create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("========== step2: check results ==============") + + self.check_result() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 65aa86edec350b78cedd95cd50282a54b9fe5693 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:11:34 +0800 Subject: [PATCH 31/82] change paramter --- source/common/src/tglobal.c | 2 +- source/libs/transport/src/transCli.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cb82ad300b..02703dd0a1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -88,7 +88,7 @@ int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; int32_t tsRedirectPeriod = 100; -int32_t tsRedirectFactor = 5; +int32_t tsRedirectFactor = 1; int32_t tsRedirectMaxPeriod = 10000; int32_t tsMaxRetryWaitTime = 60000; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d0b8adeffe..429dc80622 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1444,6 +1444,7 @@ bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { tDebug("epset not equal, retry new epset"); pCtx->epSet = epSet; + noDelay = false; } else { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; @@ -1475,7 +1476,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet; pCtx->retryInitTimestamp = taosGetTimestampMs(); pCtx->retryNextInterval = pCtx->retryMinInterval; - pCtx->retryStep = 1; + pCtx->retryStep = 0; pCtx->retryInit = true; } if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { From 3b6982e5b164f8293c15044c6750914cc1bbed2d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:23:47 +0800 Subject: [PATCH 32/82] change paramter --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 02703dd0a1..0540776987 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -90,7 +90,7 @@ bool tsKeepColumnName = false; int32_t tsRedirectPeriod = 100; int32_t tsRedirectFactor = 1; int32_t tsRedirectMaxPeriod = 10000; -int32_t tsMaxRetryWaitTime = 60000; +int32_t tsMaxRetryWaitTime = 60000 * 2; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, From 5bed17233782595c0c554cfbd1422ac96a10109d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 10:31:08 +0800 Subject: [PATCH 33/82] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 55 ++++++++++++++----------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index bcb97b7354..8bae3c23bc 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1294,7 +1294,6 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SRowBuilder rb = {0}; tdSRowInit(&rb, pTableMeta->sversion); - rb.rowType = TD_ROW_KV; tdSRowSetTpInfo(&rb, numOfCols, fLen); int32_t dataLen = 0; @@ -1705,8 +1704,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = &pSW->pSchema[i]; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { @@ -1717,7 +1716,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); + (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); int32_t schemaLen = 0; int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; @@ -1764,14 +1763,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SRowBuilder rb = {0}; tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); + tdSRowSetTpInfo(&rb, pSW->nCols, fLen); int32_t totalLen = 0; - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } +// SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); +// for (int i = 0; i < pSW->nCols; i++) { +// SSchema* schema = &pSW->pSchema[i]; +// taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); +// } for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); @@ -1780,22 +1779,29 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resInfo.current += 1; int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + for (int32_t i = 0; i < pSW->nCols; i++) { + const SSchema* pColumn = NULL; + for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { + if (strcmp(pTableMeta->schema[k].name, pSW->pSchema[i].name) == 0) { + pColumn = &pTableMeta->schema[k]; + break; } } + if (pColumn == NULL){ + uError("column not exist:%s", pSW->pSchema[i].name); + code = TSDB_CODE_INVALID_PARA; + goto end; + } + char* colData = rspObj.resInfo.row[i]; + if (!colData) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + colData -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + } + if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { offset += TYPE_BYTES[pColumn->type]; } @@ -1806,7 +1812,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { totalLen += rowLen; } - taosHashCleanup(schemaHash); blk->uid = htobe64(uid); blk->suid = htobe64(suid); blk->sversion = htonl(sver); From 829a7631b38d601ac4a2a475da37546e0624d8de Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 10:49:47 +0800 Subject: [PATCH 34/82] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 8bae3c23bc..a3447dabda 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1794,12 +1794,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { } char* colData = rspObj.resInfo.row[i]; if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, i); } else { if (IS_VAR_DATA_TYPE(pColumn->type)) { colData -= VARSTR_HEADER_SIZE; } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, i); } if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { From 21391ea77b505c683835678ca5fc5f4ed0508d38 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 10:52:06 +0800 Subject: [PATCH 35/82] change paramter --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 5546d762f4..67eff26714 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -258,8 +258,13 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.retryLimit = tsRpcRetryLimit; rpcInit.retryInterval = tsRpcRetryInterval; + rpcInit.retryMinInterval = tsRedirectPeriod; + rpcInit.retryStepFactor = tsRedirectFactor; + rpcInit.retryMaxInterval = tsRedirectMaxPeriod; + rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { From b5e4b05752400f732bbd6d777d7c7fadbd3e5c06 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 11:52:40 +0800 Subject: [PATCH 36/82] fix:[TS-2057] add interface with ttl in schemaless --- include/client/taos.h | 10 ++ include/libs/parser/parser.h | 2 +- source/client/src/clientSml.c | 128 ++++++++++--------------- source/libs/parser/inc/parInsertUtil.h | 2 +- source/libs/parser/src/parInsertSml.c | 4 +- source/libs/parser/src/parInsertSql.c | 2 +- source/libs/parser/src/parInsertStmt.c | 2 +- source/libs/parser/src/parInsertUtil.c | 4 +- utils/test/c/sml_test.c | 47 ++++++++- 9 files changed, 108 insertions(+), 93 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 25887b2879..9aaf15e8b0 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -218,6 +218,9 @@ DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_ DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); + +/* --------------------------schemaless INTERFACE------------------------------- */ + DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid); @@ -225,6 +228,13 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, + int precision, int32_t ttl, int64_t reqid); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, + int protocol, int precision, int32_t ttl, int64_t reqid); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 1a7e6dc748..fe471530fa 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -108,7 +108,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery* pQuery); void smlDestroyHandle(void* pHandle); int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen); + char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6773cfc638..f4d8c80e3f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -162,8 +162,9 @@ typedef struct { SMLProtocolType protocol; int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; SHashObj *childTables; SHashObj *superTables; @@ -2325,7 +2326,7 @@ static int32_t smlInsertData(SSmlHandle *info) { code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, - info->msgBuf.buf, info->msgBuf.len); + info->ttl, info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; @@ -2516,7 +2517,7 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { } TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision) { + int numLines, int protocol, int precision, int32_t ttl) { int batchs = 0; STscObj *pTscObj = request->pTscObj; @@ -2578,6 +2579,7 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char } info->isRawLine = (rawLine == NULL); + info->ttl = ttl; int32_t perBatch = LINE_BATCH; @@ -2641,88 +2643,43 @@ end: * @return TAOS_RES */ +TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int32_t ttl, int64_t reqid) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); +} + TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, 0); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } -TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, - int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); +TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, 0); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - int numLines = 0; - *totalRows = 0; - char *tmp = lines; - for (int i = 0; i < len; i++) { - if (lines[i] == '\n' || i == len - 1) { - numLines++; - if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) { // ignore comment - (*totalRows)++; - } - tmp = lines + i + 1; - } - } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); +TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); } -TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int64_t reqid) { +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl, int64_t reqid) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2753,5 +2710,16 @@ TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); + return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); +} + +TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +} +TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0); +} + +TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 09d55d369f..5cc72f8692 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -135,7 +135,7 @@ int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, - SArray *tagName, uint8_t tagNum); + SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 7bcfd0646f..e76ca7751d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -208,7 +208,7 @@ end: } int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen) { + char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; @@ -229,7 +229,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags); + pTableMeta->tableInfo.numOfTags, ttl); taosArrayDestroy(tagName); smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 9c39954f09..5bba012481 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -562,7 +562,7 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, - pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags); + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index a6ce71211a..e0e191b7c9 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -137,7 +137,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags); + insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); code = insBuildCreateTbMsg(pDataBlock, &tbReq); tdDestroySVCreateTbReq(&tbReq); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5f8120171f..06e32a4415 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -850,7 +850,7 @@ int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchem } void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum) { + SArray* tagName, uint8_t tagNum, int32_t ttl) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; @@ -858,7 +858,7 @@ void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, i if (sname) pTbReq->ctb.stbName = strdup(sname); pTbReq->ctb.pTag = (uint8_t*)pTag; pTbReq->ctb.tagName = taosArrayDup(tagName); - pTbReq->ttl = TSDB_DEFAULT_TABLE_TTL; + pTbReq->ttl = ttl; pTbReq->commentLen = -1; return; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 40d5bb12d2..47b7adbf18 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1084,20 +1084,22 @@ int sml_19221_Test() { return code; } -int sml_time_Test() { +int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase='2022-02-02 10:22:22' 1626006833639000000", + "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1107,8 +1109,43 @@ int sml_time_Test() { return code; } +int sml_ttl_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); + TAOS_ROW row = taos_fetch_row(pRes); + int32_t ttl = *(int32_t*)row[0]; + ASSERT(ttl == 20); + + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int main(int argc, char *argv[]) { int ret = 0; + ret = sml_ttl_Test(); + ASSERT(!ret); + ret = sml_ts2164_Test(); + ASSERT(!ret); ret = smlProcess_influx_Test(); ASSERT(!ret); ret = smlProcess_telnet_Test(); From 10c0d45b3a04a1c7010d08d6b59ae6e549d4f03e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 11:57:12 +0800 Subject: [PATCH 37/82] fix: cpuid.h error in mac for M1 cpu type --- include/os/os.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/os.h b/include/os/os.h index ab4d0a406e..f4592fc575 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -43,12 +43,12 @@ extern "C" { #include #include #include -#include #if defined(DARWIN) #else #include #include +#include #endif #else From 57202ea7066a90eb9ecd7a33e7c6f6ed512b8bb9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Nov 2022 13:08:10 +0800 Subject: [PATCH 38/82] test: add asan case --- tests/parallel_test/cases.task | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0abec6b340..15648d2a92 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -417,7 +417,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py -,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -425,7 +425,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py -,,,system-test,python3 ./test.py -f 1-insert/boundary.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/boundary.py ,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py ,,n,system-test,python3 ./test.py -f 1-insert/time_range_wise.py @@ -612,7 +612,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py -,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py From c79195928fd1d3cfae74ea376303472d2ed07dfc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 13:21:49 +0800 Subject: [PATCH 39/82] chore: re-enable old style build asan config for taos-tools --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e8fb125db7..a3b716d5b7 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2aac500 + GIT_TAG 274d230 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 049c6724119a1cd26101d1576d01c5454f986b8e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Nov 2022 14:21:45 +0800 Subject: [PATCH 40/82] fix: planner error --- source/libs/planner/src/planLogicCreater.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 44eb8478f1..e0a67c32e5 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -580,15 +580,19 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } // set the output - if (TSDB_CODE_SUCCESS == code && NULL != pOutputGroupKeys) { - code = createColumnByRewriteExprs(pOutputGroupKeys, &pAgg->node.pTargets); - } - nodesDestroyList(pOutputGroupKeys); - if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); } + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pOutputGroupKeys) { + code = createColumnByRewriteExprs(pOutputGroupKeys, &pAgg->node.pTargets); + } else if (NULL == pAgg->node.pTargets && NULL != pAgg->pGroupKeys) { + code = createColumnByRewriteExprs(pAgg->pGroupKeys, &pAgg->node.pTargets); + } + } + nodesDestroyList(pOutputGroupKeys); + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; } else { From ebe1103964b211d915df070014f2dbf11355964d Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 14:39:42 +0800 Subject: [PATCH 41/82] refactor(sync): add trace log for elect count, become leader count, config change count --- source/libs/sync/inc/syncInt.h | 4 ++++ source/libs/sync/src/syncElection.c | 6 +++--- source/libs/sync/src/syncMain.c | 20 ++++++++++++++------ source/libs/sync/src/syncTimeout.c | 2 +- source/libs/sync/src/syncUtil.c | 27 ++++++++++++++------------- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 22ae922f62..93ea138c84 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -192,6 +192,10 @@ typedef struct SSyncNode { int64_t leaderTime; int64_t lastReplicateTime; + int32_t electNum; + int32_t becomeLeaderNum; + int32_t configChangeNum; + bool isStart; } SSyncNode; diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 5c41e43fd4..8d548114fb 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -61,7 +61,8 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { - sNTrace(pSyncNode, "begin election"); + sNInfo(pSyncNode, "begin election"); + pSyncNode->electNum++; int32_t ret = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { @@ -86,7 +87,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; - } + } if (pSyncNode->replicaNum == 1) { // only myself, to leader @@ -98,7 +99,6 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; - } ret = syncNodeRequestVotePeers(pSyncNode); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 202ca72413..7fc8ba434a 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1008,6 +1008,10 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID); pSyncNode->isStart = true; + pSyncNode->electNum = 0; + pSyncNode->becomeLeaderNum = 0; + pSyncNode->configChangeNum = 0; + sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); return pSyncNode; @@ -1340,6 +1344,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde pSyncNode->pRaftCfg->cfg = *pNewConfig; pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; + pSyncNode->configChangeNum++; + bool IamInOld = syncNodeInConfig(pSyncNode, &oldConfig); bool IamInNew = syncNodeInConfig(pSyncNode, pNewConfig); @@ -1363,7 +1369,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde char newCfgStr[1024] = {0}; syncCfg2SimpleStr(&oldConfig, oldCfgStr, sizeof(oldCfgStr)); syncCfg2SimpleStr(pNewConfig, oldCfgStr, sizeof(oldCfgStr)); - sNTrace(pSyncNode, "begin do config change, from %s to %s", oldCfgStr, oldCfgStr); + sNInfo(pSyncNode, "begin do config change, from %s to %s", oldCfgStr, oldCfgStr); if (IamInNew) { pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal @@ -1495,13 +1501,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } else { // persist cfg raftCfgPersist(pSyncNode->pRaftCfg); - sNTrace(pSyncNode, "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum, - pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr); + sNInfo(pSyncNode, "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum, + pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr); } _END: // log end config change - sNTrace(pSyncNode, "end do config change, from %s to %s", oldCfgStr, newCfgStr); + sNInfo(pSyncNode, "end do config change, from %s to %s", oldCfgStr, newCfgStr); } // raft state change -------------- @@ -1598,6 +1604,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->leaderTime = taosGetTimestampMs(); + pSyncNode->becomeLeaderNum++; + // reset restoreFinish pSyncNode->restoreFinish = false; @@ -1666,7 +1674,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; // trace log - sNTrace(pSyncNode, "become leader %s", debugStr); + sNInfo(pSyncNode, "become leader %s", debugStr); } void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { @@ -1971,7 +1979,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { return; } - sTrace("enqueue heartbeat timer"); + sTrace("vgId:%d, enqueue heartbeat timer", pNode->vgId); code = pNode->syncEqMsg(pNode->msgcb, &rpcMsg); if (code != 0) { sError("failed to enqueue heartbeat msg since %s", terrstr()); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 151e5cdf46..ca67eb1880 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -52,7 +52,7 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { } static int32_t syncNodeTimerRoutine(SSyncNode* ths) { - sNTrace(ths, "timer routines"); + sNInfo(ths, "timer routines"); // timer replicate syncNodeReplicate(ths); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index caf23ac84b..abf6b21baa 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -275,18 +275,18 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo terrno = errCode; if (pNode != NULL && pNode->pRaftCfg != NULL) { - taosPrintLog(flags, level, dflag, - "vgId:%d, sync %s " - "%s" - ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 - ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", - pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, - logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->pRaftCfg->isStandBy, aqItems, pNode->snapshottingIndex, pNode->replicaNum, - pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, - pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, - hbrTimeStr); + taosPrintLog( + flags, level, dflag, + "vgId:%d, sync %s " + "%s" + ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 + ", snap-tm:%" PRIu64 ", elt-num:%d, bl-num:%d, cc-num:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 + ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", + pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, + pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, pNode->becomeLeaderNum, + pNode->configChangeNum, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, + pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, + peerStr, cfgStr, hbTimeStr, hbrTimeStr); } } @@ -438,7 +438,8 @@ void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntries host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s); } -void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed, int64_t execTime) { +void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed, + int64_t execTime) { if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; From 198e18c348fd49e0f827e851ef37dd50b7045bdb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 14:58:47 +0800 Subject: [PATCH 42/82] refactor(sync): add trace log for cache hit, cache miss --- include/libs/sync/sync.h | 5 ++++- source/libs/sync/src/syncAppendEntries.c | 2 ++ source/libs/sync/src/syncCommit.c | 2 ++ source/libs/sync/src/syncMain.c | 6 ++++++ source/libs/sync/src/syncRaftLog.c | 3 +++ source/libs/sync/src/syncReplication.c | 2 ++ source/libs/sync/src/syncUtil.c | 13 ++++++++----- 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index b8a4726be0..b6d0aecfd1 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -162,7 +162,10 @@ typedef struct SSyncFSM { // SWal implements it typedef struct SSyncLogStore { SLRUCache* pCache; - void* data; + int32_t cacheHit; + int32_t cacheMiss; + + void* data; int32_t (*syncLogUpdateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); SyncIndex (*syncLogCommitIndex)(struct SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index df2ccee546..fbf096589d 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -205,9 +205,11 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pLocalEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, hLocal); code = 0; + ths->pLogStore->cacheHit++; sNTrace(ths, "hit cache index:%" PRId64 ", bytes:%u, %p", appendIndex, pLocalEntry->bytes, pLocalEntry); } else { + ths->pLogStore->cacheMiss++; sNTrace(ths, "miss cache index:%" PRId64, appendIndex); code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, appendIndex, &pLocalEntry); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index dbe72bea7a..7d77361b3c 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -117,9 +117,11 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (h) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + pSyncNode->pLogStore->cacheHit++; sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", index, pEntry->bytes, pEntry); } else { + pSyncNode->pLogStore->cacheMiss++; sNTrace(pSyncNode, "miss cache index:%" PRId64, index); int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index, &pEntry); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7fc8ba434a..62eba3208e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -410,9 +410,11 @@ bool syncIsReadyForRead(int64_t rid) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); code = 0; + pSyncNode->pLogStore->cacheHit++; sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", lastIndex, pEntry->bytes, pEntry); } else { + pSyncNode->pLogStore->cacheMiss++; sNTrace(pSyncNode, "miss cache index:%" PRId64, lastIndex); code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, lastIndex, &pEntry); @@ -1850,9 +1852,11 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { pPreEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); code = 0; + pSyncNode->pLogStore->cacheHit++; sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", preIndex, pPreEntry->bytes, pPreEntry); } else { + pSyncNode->pLogStore->cacheMiss++; sNTrace(pSyncNode, "miss cache index:%" PRId64, preIndex); code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); @@ -2534,9 +2538,11 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde if (h) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + ths->pLogStore->cacheHit++; sNTrace(ths, "hit cache index:%" PRId64 ", bytes:%u, %p", i, pEntry->bytes, pEntry); } else { + ths->pLogStore->cacheMiss++; sNTrace(ths, "miss cache index:%" PRId64, i); code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 2f824b6b3b..60fc3a9bf7 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -45,6 +45,9 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { return NULL; } + pLogStore->cacheHit = 0; + pLogStore->cacheMiss = 0; + taosLRUCacheSetStrictCapacity(pLogStore->pCache, false); pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7d2a8b46fd..23d83b14e6 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -80,9 +80,11 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); code = 0; + pSyncNode->pLogStore->cacheHit++; sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", nextIndex, pEntry->bytes, pEntry); } else { + pSyncNode->pLogStore->cacheMiss++; sNTrace(pSyncNode, "miss cache index:%" PRId64, nextIndex); code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, nextIndex, &pEntry); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index abf6b21baa..c022a3e75b 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -242,6 +242,9 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo logBeginIndex = pNode->pLogStore->syncLogBeginIndex(pNode->pLogStore); } + int32_t cacheHit = pNode->pLogStore->cacheHit; + int32_t cacheMiss = pNode->pLogStore->cacheMiss; + char cfgStr[1024]; if (pNode->pRaftCfg != NULL) { syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr)); @@ -280,13 +283,13 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", elt-num:%d, bl-num:%d, cc-num:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 - ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", + ", snap-tm:%" PRIu64 ", elt-num:%d, bl-num:%d, cc-num:%d, hit:%d, mis:%d, aq:%d, snaping:%" PRId64 + ", r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, pNode->becomeLeaderNum, - pNode->configChangeNum, aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, - pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, - peerStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->configChangeNum, cacheHit, cacheMiss, aqItems, pNode->snapshottingIndex, pNode->replicaNum, + pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, + pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, hbrTimeStr); } } From fb140a285e0ec37d1d054d8e6c3e27d5718c0d26 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 29 Nov 2022 16:24:59 +0800 Subject: [PATCH 43/82] enh: add dropping status for vgroups --- source/dnode/mnode/impl/src/mndVgroup.c | 7 ++++++- tests/parallel_test/cases.task | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 6f4351f713..4fa00dcf7c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -665,18 +665,23 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); + bool exist = false; bool online = false; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId); if (pDnode != NULL) { + exist = true; online = mndIsDnodeOnline(pDnode, curMs); mndReleaseDnode(pMnode, pDnode); } char buf1[20] = {0}; char role[20] = "offline"; - if (online) { + if (!exist) { + strcpy(role, "dropping"); + } else if (online) { bool show = (pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER && !pVgroup->vnodeGid[i].syncRestore); snprintf(role, sizeof(role), "%s%s", syncStr(pVgroup->vnodeGid[i].syncState), show ? "*" : ""); + } else { } STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1289d0e3d4..629cc8241f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -658,7 +658,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 From 3b68577563c94a42a5a68ef3fbac79b2ac6403db Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 16:28:34 +0800 Subject: [PATCH 44/82] chore: update taos-tools 274d230 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index a3b716d5b7..8b9104e7e5 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 274d230 + GIT_TAG 9f587e9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From f3681433c66cade5cc9f40e18bf879ff1ea03810 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Nov 2022 16:43:12 +0800 Subject: [PATCH 45/82] fix: stream plan error --- include/libs/nodes/querynodes.h | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5291130e4f..9f980fd0db 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -106,7 +106,7 @@ typedef struct SValueNode { char* p; } datum; int64_t typeData; - char unit; + int8_t unit; } SValueNode; typedef struct SLeftValueNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 462ac513a5..50e410b339 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2952,6 +2952,7 @@ static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; static const char* jkValueNotReserved = "NotReserved"; static const char* jkValueIsNull = "IsNull"; +static const char* jkValueUnit = "Unit"; static const char* jkValueDatum = "Datum"; static int32_t datumToJson(const void* pObj, SJson* pJson) { @@ -3041,6 +3042,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkValueIsNull, pNode->isNull); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkValueUnit, pNode->unit); + } if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = datumToJson(pNode, pJson); } @@ -3191,6 +3195,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkValueIsNull, &pNode->isNull); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkValueUnit, &pNode->unit); + } if (TSDB_CODE_SUCCESS == code && pNode->translate && !pNode->isNull) { code = jsonToDatum(pJson, pNode); } From 8b02719ea579413ed7090360098a24a4a680eed8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 17:01:16 +0800 Subject: [PATCH 46/82] fix:[TD-20612] error if write raw with some cols --- source/client/src/clientRawBlockWrite.c | 122 +++++++++++------------- utils/test/c/tmq_taosx_ci.c | 26 +++++ 2 files changed, 83 insertions(+), 65 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a3447dabda..9db79c6eb6 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1262,22 +1262,19 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch } uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); uint64_t uid = pTableMeta->uid; - int32_t numOfCols = numFields; + int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for(int k = 0; k < numOfCols; k++){ for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { SSchema* schema = pTableMeta->schema + i; - if(strcmp(schema->name, fields[k].name) != 0) continue; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { nVar++; } } - } fLen -= sizeof(TSKEY); @@ -1298,14 +1295,14 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch int32_t dataLen = 0; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); + char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; + pStart += sizeof(int32_t) * numFields; - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + for (int32_t i = 0; i < numFields; ++i) { + if (IS_VAR_DATA_TYPE(fields[i].type)) { pCol[i].offset = (int32_t*)pStart; pStart += rows * sizeof(int32_t); } else { @@ -1317,36 +1314,35 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch pStart += colLength[i]; } + SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int i = 0; i < numFields; i++) { + TAOS_FIELD* schema = &fields[i]; + taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + } + for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); int32_t offset = 0; for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = NULL; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - if (strcmp((pTableMeta->schema + i)->name, fields[k].name) == 0) { - pColumn = pTableMeta->schema + i; - break; - } - } - if(pColumn == NULL){ - uError("column not exist:%s", fields[k].name); - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + const SSchema* pColumn = &pTableMeta->schema[k]; + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { // add none + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); + }else{ + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } } } @@ -1360,6 +1356,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch dataLen += rowLen; } + taosHashCleanup(schemaHash); taosMemoryFree(pCol); blk->uid = htobe64(uid); @@ -1704,8 +1701,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uint16_t fLen = 0; int32_t rowSize = 0; int16_t nVar = 0; - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; + for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + SSchema* schema = &pTableMeta->schema[i]; fLen += TYPE_BYTES[schema->type]; rowSize += schema->bytes; if (IS_VAR_DATA_TYPE(schema->type)) { @@ -1716,7 +1713,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t rows = rspObj.resInfo.numOfRows; int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); + (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); int32_t schemaLen = 0; int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; @@ -1763,14 +1760,14 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SRowBuilder rb = {0}; tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pSW->nCols, fLen); + tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); int32_t totalLen = 0; -// SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); -// for (int i = 0; i < pSW->nCols; i++) { -// SSchema* schema = &pSW->pSchema[i]; -// taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); -// } + SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = &pSW->pSchema[i]; + taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + } for (int32_t j = 0; j < rows; j++) { tdSRowResetBuf(&rb, rowData); @@ -1779,29 +1776,22 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resInfo.current += 1; int32_t offset = 0; - for (int32_t i = 0; i < pSW->nCols; i++) { - const SSchema* pColumn = NULL; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - if (strcmp(pTableMeta->schema[k].name, pSW->pSchema[i].name) == 0) { - pColumn = &pTableMeta->schema[k]; - break; - } - } - if (pColumn == NULL){ - uError("column not exist:%s", pSW->pSchema[i].name); - code = TSDB_CODE_INVALID_PARA; - goto end; - } - char* colData = rspObj.resInfo.row[i]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, i); + for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { + const SSchema* pColumn = &pTableMeta->schema[k]; + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; + char* colData = rspObj.resInfo.row[*index]; + if (!colData) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + colData -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, i); } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { offset += TYPE_BYTES[pColumn->type]; } @@ -1812,6 +1802,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { totalLen += rowLen; } + taosHashCleanup(schemaHash); blk->uid = htobe64(uid); blk->suid = htobe64(suid); blk->sversion = htonl(sver); @@ -1882,6 +1873,7 @@ end: return code; } + static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pVgHash = NULL; @@ -2094,7 +2086,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) const SSchema* pColumn = &pTableMeta->schema[k]; int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); } else { char* colData = rspObj.resInfo.row[*index]; if (!colData) { @@ -2175,7 +2167,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; -end: + end: tDeleteSTaosxRsp(&rspObj.rsp); rspObj.resInfo.pRspMsg = NULL; doFreeReqResultInfo(&rspObj.resInfo); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 6540fdac4c..5af9ba68b2 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -76,6 +76,32 @@ static void msg_process(TAOS_RES* msg) { } int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ + + /* test for TD-20612 start*/ +// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// return 0; + /* test for TD-20612 end*/ + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " "nchar(8), t4 bool)"); From 8a555080f1b6a372b4329da255738c53d03f5201 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:08:53 +0800 Subject: [PATCH 47/82] change parameterr --- source/common/src/tglobal.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4f9089b0cd..b64e3a562c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -16,9 +16,9 @@ #define _DEFAULT_SOURCE #include "tglobal.h" #include "tconfig.h" -#include "tmisce.h" #include "tgrant.h" #include "tlog.h" +#include "tmisce.h" GRANT_CFG_DECLARE; @@ -86,10 +86,10 @@ bool tsQueryPlannerTrace = false; int32_t tsQueryNodeChunkSize = 32 * 1024; bool tsQueryUseNodeAllocator = true; bool tsKeepColumnName = false; -int32_t tsRedirectPeriod = 100; -int32_t tsRedirectFactor = 1; -int32_t tsRedirectMaxPeriod = 10000; -int32_t tsMaxRetryWaitTime = 60000 * 2; +int32_t tsRedirectPeriod = 10; +int32_t tsRedirectFactor = 2; +int32_t tsRedirectMaxPeriod = 1000; +int32_t tsMaxRetryWaitTime = 10000; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -124,7 +124,7 @@ int32_t tsMinIntervalTime = 1; int32_t tsMaxMemUsedByInsert = 1024; float tsSelectivityRatio = 1.0; -int32_t tsTagFilterResCacheSize = 1024*10; +int32_t tsTagFilterResCacheSize = 1024 * 10; // the maximum allowed query buffer size during query processing for each data node. // -1 no limit (default) From 0b91b5b2b3d41b36754841575552b341de33476a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 29 Nov 2022 14:25:31 +0800 Subject: [PATCH 48/82] fix:calculate the correct deletion range --- source/libs/executor/src/tfill.c | 9 +- .../tsim/stream/fillIntervalDelete1.sim | 167 ++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 42b1d058b5..68228f2689 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -918,8 +918,11 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS return; } + TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, + pFillSup->interval.precision); + pFillInfo->needFill = true; - pFillInfo->start = start; + pFillInfo->start = realStart; pFillInfo->current = pFillInfo->start; pFillInfo->end = end; pFillInfo->pos = FILL_POS_INVALID; @@ -1418,9 +1421,13 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { if (code == TSDB_CODE_SUCCESS) { code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextLen); } + // ts will be deleted later if (delTs != ts) { streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); + streamStateFreeCur(pCur); + pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } + endTs = nextKey.ts - 1; if (code != TSDB_CODE_SUCCESS) { break; } diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index e14062e830..ca347f120a 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -344,14 +344,176 @@ endi +sql drop stream if exists streams11; +sql drop stream if exists streams12; +sql drop stream if exists streams13; +sql drop stream if exists streams14; +sql drop stream if exists streams15; +sql drop database if exists test7; +sql create database test7 vgroups 1; +sql use test7; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams11 trigger at_once into streamt11 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(NULL); +sql create stream streams12 trigger at_once into streamt12 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(value,100.0,200); +sql create stream streams13 trigger at_once into streamt13 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(next); +sql create stream streams14 trigger at_once into streamt14 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(prev); +sql create stream streams15 trigger at_once into streamt15 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(linear); +sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791210001,1,1,1,1.0,'aaa'); + +sql insert into t1 values(1648791215000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791220000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791225000,4,4,4,4.0,'fff'); + +sql insert into t1 values(1648791230000,5,5,5,5.0,'ddd'); +sql insert into t1 values(1648791230001,6,6,6,6.0,'eee'); +sql insert into t1 values(1648791230002,7,7,7,7.0,'fff'); + +$loop_count = 0 + +loop7: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt11 order by ts; + +if $rows != 21 then + print ====streamt11=rows3=$rows + goto loop7 +endi + +sql select * from streamt12 order by ts; + +if $rows != 21 then + print ====streamt12=rows3=$rows + goto loop7 +endi + +sql select * from streamt13 order by ts; + +if $rows != 21 then + print ====streamt13=rows3=$rows + goto loop7 +endi + +sql select * from streamt14 order by ts; + +if $rows != 21 then + print ====streamt14=rows3=$rows + goto loop7 +endi + +sql select * from streamt15 order by ts; + +if $rows != 21 then + print ====streamt15=rows3=$rows + goto loop7 +endi + +sql delete from t1 where ts > 1648791210001 and ts < 1648791230000; + +$loop_count = 0 + +loop8: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt11 order by ts; + +if $rows != 21 then + print ====streamt11=rows3=$rows + goto loop8 +endi + +if $data12 != NULL then + print ====streamt11=3=data01=$data01 + goto loop8 +endi + +if $data[19][2] != NULL then + print ====streamt11=3=data[19][2]=$data[19][2] + goto loop8 +endi + +sql select * from streamt12 order by ts; + +if $rows != 21 then + print ====streamt12=rows3=$rows + goto loop8 +endi + +if $data12 != 200 then + print ====streamt12=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 200 then + print ====streamt12=3=data[19][2]=$data[19][2] + goto loop8 +endi + +sql select * from streamt13 order by ts; + +if $rows != 21 then + print ====streamt13=rows3=$rows + goto loop8 +endi + +if $data12 != 3 then + print ====streamt13=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 3 then + print ====streamt13=3=data[19][2]=$data[19][2] + goto loop8 +endi +sql select * from streamt14 order by ts; + +if $rows != 21 then + print ====streamt14=rows3=$rows + goto loop8 +endi + +if $data12 != 2 then + print ====streamt14=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 2 then + print ====streamt14=3=data[19][2]=$data[19][2] + goto loop8 +endi +sql select * from streamt15 order by ts; +if $rows != 21 then + print ====streamt15=rows3=$rows + goto loop8 +endi +if $data12 != 2 then + print ====streamt15=3=data12=$data12 + goto loop8 +endi + +if $data[19][2] != 2 then + print ====streamt15=3=data[19][2]=$data[19][2] + goto loop8 +endi @@ -384,6 +546,11 @@ sql drop stream if exists streams7; sql drop stream if exists streams8; sql drop stream if exists streams9; sql drop stream if exists streams10; +sql drop stream if exists streams11; +sql drop stream if exists streams12; +sql drop stream if exists streams13; +sql drop stream if exists streams14; +sql drop stream if exists streams15; sql use test1; sql select * from t1; From dcd16684a066a2c6e96463102c2d328846676e91 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:53:24 +0800 Subject: [PATCH 49/82] fix invalid param --- source/libs/transport/src/trans.c | 3 +++ source/libs/transport/src/transCli.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 415a8766e3..c6a5cfdc95 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -63,6 +63,9 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + if (pRpc->numOfThreads <= 0) { + pRpc->numOfThreads = 1; + } uint32_t ip = 0; if (pInit->connType == TAOS_CONN_SERVER) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c967701930..725f352e5c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1476,7 +1476,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - bool retry = pTransInst->retry(code, pResp->msgType - 1); + bool retry = pTransInst->retry != NULL ? pTransInst->retry(code, pResp->msgType - 1) : false; if (retry == false) { return false; } From 47c0d4cbbd6dbe3f1fe257d91792c2c0b7d3b6cd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 17:58:16 +0800 Subject: [PATCH 50/82] fix invalid param --- source/libs/scheduler/src/schTask.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index af78647018..ed052be784 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -785,6 +785,9 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSe SCH_TASK_DLOG("update task target node %d epset from %s to %s", pAddr->nodeId, origEpset, newEpset); + taosMemoryFree(origEpset); + taosMemoryFree(newEpset); + memcpy(&pAddr->epSet, pEpSet, sizeof(pAddr->epSet)); return TSDB_CODE_SUCCESS; From 8c3ff153428caaf3d823cb1f34f9b499a26028c1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 18:00:44 +0800 Subject: [PATCH 51/82] fix partition by column --- source/libs/executor/src/executorimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 094060b213..6f6097d258 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1507,7 +1507,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock) { + if (!hasValidBlock && + downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; From 6f0d6acaf462fd0436344a2bb38aabc5648b8ca8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 18:04:11 +0800 Subject: [PATCH 52/82] chore: re-enable old style sanitizer config for taos-tools (#18539) * chore: taostools sanitizer flag align with tdengine * chore: re-enable old style build asan config for taos-tools * chore: update taos-tools 274d230 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e8fb125db7..8b9104e7e5 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2aac500 + GIT_TAG 9f587e9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2956eec51ff8863238738bcfe8721320321e59c0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:01:09 +0800 Subject: [PATCH 53/82] fix partition/group by tag count should not have output --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 7 +++++++ source/libs/executor/src/scanoperator.c | 1 + 3 files changed, 9 insertions(+) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8163217039..db642c05c8 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -332,6 +332,7 @@ typedef struct STableScanInfo { int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; + bool hasGroupByTag; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6f6097d258..6df464b06e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1424,6 +1424,13 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION || + (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && + ((STableScanInfo *)downstream->info)->hasGroupByTag == true)) { + return TSDB_CODE_SUCCESS; + } + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; bool hasCountFunc = false; for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84b7678b9f..a4e609ae81 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -895,6 +895,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; + pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); From 900fe01f0ed103ed224fd959c54fde3e4f2d8e08 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:32:51 +0800 Subject: [PATCH 54/82] fix error --- source/libs/executor/src/executorimpl.c | 5 ++--- source/libs/function/src/builtinsimpl.c | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6df464b06e..14be6326e0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1425,7 +1425,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc } SOperatorInfo* downstream = pOperator->pDownstream[0]; - if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION || + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && ((STableScanInfo *)downstream->info)->hasGroupByTag == true)) { return TSDB_CODE_SUCCESS; @@ -1514,8 +1514,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (!hasValidBlock && - downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_PARTITION) { + if (!hasValidBlock) { createDataBlockForEmptyInput(pOperator, &pBlock); if (pBlock == NULL) { break; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index acbddc7128..1881372e34 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3930,12 +3930,16 @@ static void hllTransferInfo(SHLLInfo* pInput, SHLLInfo* pOutput) { int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pCol = pInput->pData[0]; - ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + if (pCol->info.type != TSDB_DATA_TYPE_BINARY) { + return TSDB_CODE_SUCCESS; + } SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t start = pInput->startRowIndex; + for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data); From 50c8392dfc09ddb099e73b54251207eed3aa10f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 19:39:09 +0800 Subject: [PATCH 55/82] change parameterr --- tests/system-test/0-others/sysinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 4ddae42ac5..2601c200d9 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -51,7 +51,7 @@ class TDTestCase: sleep(self.delaytime) if platform.system().lower() == 'windows': sleep(10) - tdSql.error('select server_status()') + #tdSql.error('select server_status()') def run(self): self.get_database_info() From f87b9fa7fec257c80203f0c02bef8ee35a21144a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 19:41:03 +0800 Subject: [PATCH 56/82] refactor(sync): add interface syncSnapshotSending, syncSnapshotRecving --- include/libs/sync/sync.h | 5 ++++ source/libs/sync/inc/syncInt.h | 4 +++ source/libs/sync/src/syncMain.c | 46 +++++++++++++++++++++++++++++++++ source/libs/sync/src/syncUtil.c | 34 ++++++++++++++++++++---- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index b6d0aecfd1..1b65d95cb1 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -43,6 +43,9 @@ extern "C" { #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 #define SYNC_HEART_TIMEOUT_MS 1000 * 8 +#define SYNC_HEARTBEAT_SLOW_MS 1500 +#define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 + #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 @@ -226,6 +229,8 @@ int32_t syncEndSnapshot(int64_t rid); int32_t syncLeaderTransfer(int64_t rid); int32_t syncStepDown(int64_t rid, SyncTerm newTerm); bool syncIsReadyForRead(int64_t rid); +bool syncSnapshotSending(int64_t rid); +bool syncSnapshotRecving(int64_t rid); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 93ea138c84..d764901ab3 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -195,6 +195,8 @@ typedef struct SSyncNode { int32_t electNum; int32_t becomeLeaderNum; int32_t configChangeNum; + int32_t hbSlowNum; + int32_t hbrSlowNum; bool isStart; @@ -239,6 +241,8 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h); bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode); +bool syncNodeSnapshotSending(SSyncNode* pSyncNode); +bool syncNodeSnapshotRecving(SSyncNode* pSyncNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 62eba3208e..19a3bd06cf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -447,6 +447,28 @@ bool syncIsReadyForRead(int64_t rid) { return ready; } +bool syncSnapshotSending(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + return false; + } + + bool b = syncNodeSnapshotSending(pSyncNode); + syncNodeRelease(pSyncNode); + return b; +} + +bool syncSnapshotRecving(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + return false; + } + + bool b = syncNodeSnapshotRecving(pSyncNode); + syncNodeRelease(pSyncNode); + return b; +} + int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { if (pSyncNode->peersNum == 0) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); @@ -1013,6 +1035,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->electNum = 0; pSyncNode->becomeLeaderNum = 0; pSyncNode->configChangeNum = 0; + pSyncNode->hbSlowNum = 0; + pSyncNode->hbrSlowNum = 0; sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); @@ -1563,6 +1587,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->leaderCache = EMPTY_RAFT_ID; } + pSyncNode->hbSlowNum = 0; + // state change pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); @@ -1607,6 +1633,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->leaderTime = taosGetTimestampMs(); pSyncNode->becomeLeaderNum++; + pSyncNode->hbrSlowNum = 0; // reset restoreFinish pSyncNode->restoreFinish = false; @@ -2167,6 +2194,25 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { return b; } +bool syncNodeSnapshotSending(SSyncNode* pSyncNode) { + if (pSyncNode == NULL) return false; + bool b = false; + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + if (pSyncNode->senders[i] != NULL && pSyncNode->senders[i]->start) { + b = true; + break; + } + } + return b; +} + +bool syncNodeSnapshotRecving(SSyncNode* pSyncNode) { + if (pSyncNode == NULL) return false; + if (pSyncNode->pNewNodeReceiver == NULL) return false; + if (pSyncNode->pNewNodeReceiver->start) return true; + return false; +} + static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index c022a3e75b..552cf9e7eb 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -283,13 +283,15 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", elt-num:%d, bl-num:%d, cc-num:%d, hit:%d, mis:%d, aq:%d, snaping:%" PRId64 + ", snap-tm:%" PRIu64 + ", elt-num:%d, bl-num:%d, cc-num:%d, hit:%d, mis:%d, hb-slow:%d, hbr-slow:%d, aq:%d, snaping:%" PRId64 ", r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s, %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, pNode->becomeLeaderNum, - pNode->configChangeNum, cacheHit, cacheMiss, aqItems, pNode->snapshottingIndex, pNode->replicaNum, - pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, - pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->configChangeNum, cacheHit, cacheMiss, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, + pNode->snapshottingIndex, pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, + pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, + hbTimeStr, hbrTimeStr); } } @@ -463,12 +465,23 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool } void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64_t timeDiff) { + if (timeDiff > SYNC_HEARTBEAT_SLOW_MS) { + pSyncNode->hbSlowNum++; + + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sNInfo(pSyncNode, + "recv sync-heartbeat from %s:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 + "}, net elapsed:%" PRId64, + host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timeDiff); + } + if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64 "}, net elapsed:%" PRId64, @@ -487,6 +500,17 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p } void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff) { + if (timeDiff > SYNC_HEARTBEAT_REPLY_SLOW_MS) { + pSyncNode->hbrSlowNum++; + + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + sNTrace(pSyncNode, + "recv sync-heartbeat-reply from %s:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, net elapsed:%" PRId64, host, + port, pMsg->term, pMsg->timeStamp, timeDiff); + } + if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; From ca51b86359acef64d78a7e309571e6dc5044d044 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 19:45:18 +0800 Subject: [PATCH 57/82] fix test cases --- tests/script/tsim/parser/union.sim | 8 ++-- .../system-test/1-insert/delete_childtable.py | 4 +- tests/system-test/2-query/count.py | 43 ++++++++----------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 8dc19912c8..9e7c6f77cc 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -245,12 +245,12 @@ endi # first subclause are empty sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (select sum(c1) as c from union_tb1); -if $rows != 1 then - return -1 -endi -if $data00 != 495000 then +if $rows != 2 then return -1 endi +#if $data00 != 495000 then +# return -1 +#endi # all subclause are empty sql (select c1 from union_tb0 limit 0) union all (select c1 from union_tb1 where ts>'2021-1-1 0:0:0') diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py index 584e88330c..e3144edb45 100644 --- a/tests/system-test/1-insert/delete_childtable.py +++ b/tests/system-test/1-insert/delete_childtable.py @@ -121,7 +121,7 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select count(*) from {stbname}') if tb_num <= 1: - if len(tdSql.queryResult) != 0: + if len(tdSql.queryResult) != 1 and tdSql.queryResult[0][0] != 0: tdLog.exit('delete case failure!') else: tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num) @@ -229,4 +229,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 8962cf305a..40d9b3ff8b 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -93,13 +93,17 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.stbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by tbname') - tdSql.checkRows(2) - tdSql.checkData(0, 0, 0) - tdSql.checkData(1, 0, 0) - tdSql.checkData(0, 1, None) - tdSql.checkData(1, 1, None) + tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by c1') tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} group by t0') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by tbname') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1') + tdSql.checkRows(0) + tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by t0') + tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.stbname} group by c1)') tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -109,15 +113,11 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.stbname} partition by c1 interval(1s)') tdSql.checkRows(0) + tdSql.query(f'select {function_name}(1),sum(1) from (select {function_name}(1) from {self.stbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.stbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, 2) - tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.stbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 2) def query_empty_ntb(self): tdSql.query(f'select count(*) from {self.ntbname}') tdSql.checkRows(1) @@ -158,9 +158,7 @@ class TDTestCase: tdSql.query(f'select sum(1),max(c2),min(1),leastsquares(c1,1,1) from {self.ntbname}') tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by tbname') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 0) - tdSql.checkData(0, 1, None) + tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} group by c1') tdSql.checkRows(0) tdSql.query(f'select {function_name}(1) from (select {function_name}(c1),sum(c1) from {self.ntbname} group by c1)') @@ -172,15 +170,10 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query(f'select {function_name}(c1),sum(c1) from {self.ntbname} partition by c1 interval(1s)') tdSql.checkRows(0) - - tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 1) - tdSql.query(f'select hyperloglog(1),sum(1) from (select hyperloglog(1) from {self.ntbname} group by tbname)') - tdSql.checkRows(1) - tdSql.checkData(0, 0, 1) - tdSql.checkData(0, 1, 1) + tdSql.query(f'select count(1),sum(1) from (select count(1) from {self.ntbname} group by tbname)') + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, None) def count_query_stb(self,column_dict,tag_dict,stbname,tbnum,rownum): tdSql.query(f'select count(tbname) from {stbname}') tdSql.checkEqual(tdSql.queryResult[0][0],tbnum*rownum) From b2ed72aeca780e1f156a2cbf87f62a4667ff3ae0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 29 Nov 2022 19:54:06 +0800 Subject: [PATCH 58/82] fix:error in index --- source/client/src/clientRawBlockWrite.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 9db79c6eb6..5583c8ae00 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1330,15 +1330,15 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); }else{ if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; + if (pCol[*index].offset[j] != -1) { + char* data = pCol[*index].pData + pCol[*index].offset[j]; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); } } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; + if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { + char* data = pCol[*index].pData + pColumn->bytes * j; tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); } else { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); From 8ec0b424567e731a3f1a5d81eee8e3984e1ac883 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 29 Nov 2022 20:04:04 +0800 Subject: [PATCH 59/82] refactor(sync): modify some trace log to info --- source/libs/sync/src/syncUtil.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 552cf9e7eb..4cd3a53283 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -639,7 +639,7 @@ void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs } void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, int32_t voteGranted, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char logBuf[256]; char host[64]; @@ -647,42 +647,42 @@ void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, i syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); if (voteGranted == -1) { - sNTrace(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + sNInfo(pSyncNode, + "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } else { - sNTrace(pSyncNode, - "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d", - host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); + sNInfo(pSyncNode, + "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted); } } void syncLogSendRequestVote(SSyncNode* pNode, const SyncRequestVote* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + sNInfo(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, + pMsg->voteGranted, s); } void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - if (!(sDebugFlag & DEBUG_TRACE)) return; + // if (!(sDebugFlag & DEBUG_TRACE)) return; char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, - pMsg->voteGranted, s); + sNInfo(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, + pMsg->voteGranted, s); } From 6eb032f3e9fa8a771844f97de6a6417b1a9cf659 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 20:07:26 +0800 Subject: [PATCH 60/82] change test case --- tests/system-test/0-others/sysinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 2601c200d9..9ffa6ca9e6 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -48,10 +48,10 @@ class TDTestCase: tdSql.checkData(0,0,1) #!for bug tdDnodes.stoptaosd(1) - sleep(self.delaytime) + sleep(self.delaytime * 5) if platform.system().lower() == 'windows': sleep(10) - #tdSql.error('select server_status()') + tdSql.error('select server_status()') def run(self): self.get_database_info() From 2d804fa48f8e758af181e05ca34102ceb86e3a46 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Nov 2022 21:30:08 +0800 Subject: [PATCH 61/82] more code --- source/common/src/tdataformat.c | 9 +++----- source/common/src/tmsg.c | 30 +++++++++++++++----------- source/libs/parser/src/parInsertUtil.c | 5 ++++- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 07ece3b8a2..0886e4bc74 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -120,7 +120,6 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; maxIdx = nkv; @@ -146,14 +145,12 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -165,17 +162,17 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { ntp = sizeof(SRow); break; case HAS_VALUE: - ntp = sizeof(SRow) + ntp; + ntp = sizeof(SRow) + pTSchema->flen; break; case (HAS_NULL | HAS_NONE): ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; default: ASSERT(0); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2a7f7343c0..49bffccd99 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6807,26 +6807,30 @@ _exit: return code; } -void destroySSubmitTbData(SSubmitTbData *pTbData) { - // if (pTbData->isColFmt) { - // // todo - // } else { - // taosArrayDestroyP(pTbData->aRowP, (FDelete)tRowDestroy); - // } -} - void tDestroySSubmitTbData(SSubmitTbData *pTbData) { - if (NULL == pTbData) { - return; + if (pTbData->pCreateTbReq) { + // todo + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + ASSERT(0); // TODO + } else { + if (pTbData->aRowP) { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); + } } - destroySSubmitTbData(pTbData); - taosMemoryFree(pTbData); } void tDestroySSubmitReq2(SSubmitReq2 *pReq) { if (NULL == pReq) return; - taosArrayDestroyEx(pReq->aSubmitTbData, (FDelete)destroySSubmitTbData); + taosArrayDestroyEx(pReq->aSubmitTbData, (FDelete)tDestroySSubmitTbData); } int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 1592f9d7e1..a138b684d1 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -1069,7 +1069,10 @@ void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { tDestroyTSchema(pTableCxt->pSchema); destroyBoundColInfo(&pTableCxt->boundColsInfo); taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); - tDestroySSubmitTbData(pTableCxt->pData); + if (pTableCxt->pData) { + tDestroySSubmitTbData(pTableCxt->pData); + taosMemoryFree(pTableCxt->pData); + } taosMemoryFree(pTableCxt); } From 9278ed2ab5b608cd8b497d7f24517ee91a2c403e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 29 Nov 2022 21:59:49 +0800 Subject: [PATCH 62/82] fix mem leak --- source/libs/scheduler/inc/schInt.h | 8 ++++---- source/libs/scheduler/src/schTask.c | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 83b58a77d2..dfc48e7d9f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -146,7 +146,7 @@ typedef struct SSchedulerMgmt { bool exit; int32_t jobRef; int32_t jobNum; - SSchStat stat; + SSchStat stat; void *timer; SRWLatch hbLock; SHashObj *hbConnections; @@ -214,9 +214,9 @@ typedef struct SSchRedirectCtx { } SSchRedirectCtx; typedef struct SSchTimerParam { - int64_t rId; - uint64_t queryId; - uint64_t taskId; + int64_t rId; + uint64_t queryId; + uint64_t taskId; } SSchTimerParam; typedef struct SSchTask { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ed052be784..9a7f3332b3 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1152,7 +1152,12 @@ void schHandleTimerEvent(void *param, void *tmrId) { SSchJob *pJob = NULL; int32_t code = 0; - if (schProcessOnCbBegin(&pJob, &pTask, pTimerParam->queryId, pTimerParam->rId, pTimerParam->taskId)) { + int64_t rId = pTimerParam->rId; + uint64_t queryId = pTimerParam->queryId; + uint64_t taskId = pTimerParam->taskId; + taosMemoryFree(pTimerParam); + + if (schProcessOnCbBegin(&pJob, &pTask, queryId, rId, taskId)) { return; } From 39628ec6f5c2211a84e1d3f63f3f5690d475baeb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Nov 2022 22:06:08 +0800 Subject: [PATCH 63/82] more code --- include/common/tdataformat.h | 4 ++ source/common/src/tdataformat.c | 85 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f3c1bc22c2..532c35165b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -123,6 +123,10 @@ extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); int32_t tColDataSortMerge(SColData *aColData); +// for encode/decode +int32_t tPutColData(uint8_t *pBuf, SColData *pColData); +int32_t tGetColData(uint8_t *pBuf, SColData *pColData); + // STRUCT ================================ struct STColumn { col_id_t colId; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0886e4bc74..dbf616f1c6 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2109,6 +2109,91 @@ int32_t tColDataSortMerge(SColData *aColData) { return 0; } +int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(pBuf + n, pColData->cid); + n += tPutI8(pBuf + n, pColData->type); + n += tPutI32v(pBuf + n, pColData->nVal); + n += tPutI8(pBuf + n, pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + n += (pColData->nVal << 2); + + n += tPutI32v(pBuf + n, pColData->nData); + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } else { + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } + } + + return n; +} + +int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(pBuf + n, &pColData->cid); + n += tGetI8(pBuf + n, &pColData->type); + n += tGetI32v(pBuf + n, &pColData->nVal); + n += tGetI8(pBuf + n, &pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = pBuf + n; + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = pBuf + n; + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; + } + + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->aOffset = (int32_t *)(pBuf + n); + n += (pColData->nVal << 2); + + n += tGetI32v(pBuf + n, &pColData->nData); + pColData->pData = pBuf + n; + n += pColData->nData; + } else { + pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; + pColData->nData = pBuf + n; + n += pColData->nData; + } + } + + return n; +} + #define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ do { \ (SUM) += (VAL); \ From f913fbdaf656d44f95d0bb9e97f0119c151594a5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 22:14:46 +0800 Subject: [PATCH 64/82] fix errors --- source/libs/function/src/builtinsimpl.c | 3 ++- tests/script/tsim/parser/lastrow_query.sim | 2 +- tests/system-test/2-query/distribute_agg_count.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1881372e34..985f3cad34 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -526,7 +526,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumOfElems(pCtx); + int32_t numOfElem = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -539,6 +539,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { numOfElem = 1; *((int64_t*)buf) = 0; } else { + numOfElem = getNumOfElems(pCtx); *((int64_t*)buf) += numOfElem; } diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index 5f557fd7bd..a1b14c7a0e 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -55,7 +55,7 @@ endi # regression test case 1 sql select count(*) from lr_tb1 where ts>'2018-09-18 08:45:00.1' and ts<'2018-09-18 08:45:00.2' -if $row != 0 then +if $row != 1 then return -1 endi diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index 67d47d7da5..7d131cd77d 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -183,7 +183,7 @@ class TDTestCase: tdSql.checkRows(20) tdSql.query(f"select count(c1) from {dbname}.stb1 where t1> 4 partition by tbname") - tdSql.checkRows(16) + tdSql.checkRows(15) # union all tdSql.query(f"select count(c1) from {dbname}.stb1 union all select count(c1) from {dbname}.stb1 ") From 30fa44ea385cf3dbde06f4d2887dcbaa25845dad Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 22:19:23 +0800 Subject: [PATCH 65/82] chore: update taos-tools and add x86 definition --- cmake/cmake.platform | 2 ++ cmake/taostools_CMakeLists.txt.in | 2 +- include/os/os.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e822cc5e2c..711d74fa4c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -82,10 +82,12 @@ IF ("${CPUTYPE}" STREQUAL "") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(TD_INTEL_64 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") SET(TD_INTEL_32 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8b9104e7e5..79b447e20d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 9f587e9 + GIT_TAG d19e82c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/os/os.h b/include/os/os.h index f4592fc575..0688eeb9ad 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -48,8 +48,10 @@ extern "C" { #else #include #include +#if defined(_TD_X86_) #include #endif +#endif #else #ifndef __func__ From 95677c4ac42788fb824674994f0bb2c161aa6998 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 22:31:21 +0800 Subject: [PATCH 66/82] fix: get cpu id on x86 only --- source/os/src/osSysinfo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f336b84e1e..7ec1da0530 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -481,6 +481,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { #elif defined(_TD_DARWIN_64) #else +#ifdef _TD_X86_ // Since the compiler is not support avx/avx2 instructions, the global variables always need to be // set to be false #if __AVX__ || __AVX2__ @@ -504,6 +505,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { // Ref to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 __cpuid_fix(7u, eax, ebx, ecx, edx); *avx2 = (char) ((ebx & bit_AVX2) == bit_AVX2); +#endif // _TD_X86_ #endif return 0; From af30786e4fe8699f895979134ff8514a14769e77 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 29 Nov 2022 22:47:55 +0800 Subject: [PATCH 67/82] fix avg function error --- source/libs/function/src/detail/tavgfunction.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 7d018a8dc7..4cafbd4e6e 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -471,7 +471,6 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - pAvgRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -483,6 +482,8 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { goto _over; } + pAvgRes->type = type; + if (pInput->colDataSMAIsSet) { // try to use SMA if available numOfElem = calculateAvgBySMAInfo(pAvgRes, numOfRows, type, pAgg); } else if (!pCol->hasNull) { // try to employ the simd instructions to speed up the loop @@ -592,6 +593,10 @@ _over: } static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { + if (IS_NULL_TYPE(pInput->type)) { + return; + } + pOutput->type = pInput->type; if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) { pOutput->sum.isum += pInput->sum.isum; @@ -748,4 +753,4 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { taosMemoryFree(res); return pResInfo->numOfRes; -} \ No newline at end of file +} From 548684e5ac06b7e6e0ec9a5046953baa5ac53334 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 29 Nov 2022 23:39:58 +0800 Subject: [PATCH 68/82] fix(stream): delete multiple row --- include/common/tmsg.h | 55 +++++++++---------- include/common/tmsgdef.h | 4 +- source/common/src/tmsg.c | 37 ++++++------- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 13 +++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +-- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 4 +- source/libs/stream/src/streamExec.c | 9 ++- source/libs/stream/src/streamRecover.c | 2 +- source/os/src/osSemaphore.c | 3 +- 11 files changed, 70 insertions(+), 69 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 86967bb579..80590a25c0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1592,14 +1592,14 @@ typedef struct SSubQueryMsg { int8_t explain; int8_t needFetch; uint32_t sqlLen; - char *sql; + char* sql; uint32_t msgLen; - char *msg; + char* msg; } SSubQueryMsg; -int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); -int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); -void tFreeSSubQueryMsg(SSubQueryMsg *pReq); +int32_t tSerializeSSubQueryMsg(void* buf, int32_t bufLen, SSubQueryMsg* pReq); +int32_t tDeserializeSSubQueryMsg(void* buf, int32_t bufLen, SSubQueryMsg* pReq); +void tFreeSSubQueryMsg(SSubQueryMsg* pReq); typedef struct { SMsgHead header; @@ -1638,9 +1638,8 @@ typedef struct { int32_t execId; } SResFetchReq; -int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); -int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); - +int32_t tSerializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq); +int32_t tDeserializeSResFetchReq(void* buf, int32_t bufLen, SResFetchReq* pReq); typedef struct { SMsgHead header; @@ -1713,12 +1712,11 @@ typedef struct { int32_t execId; } STaskDropReq; -int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); -int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq); - -int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); -int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp); +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); +int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); typedef struct { int32_t code; @@ -2923,9 +2921,8 @@ typedef struct { STqOffsetVal reqOffset; } SMqPollReq; -int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); -int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); - +int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); +int32_t tDeserializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); typedef struct { int32_t vgId; @@ -3138,7 +3135,8 @@ int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); typedef struct { // int64_t uid; char tbname[TSDB_TABLE_NAME_LEN]; - int64_t ts; + int64_t startTs; + int64_t endTs; } SSingleDeleteReq; int32_t tEncodeSSingleDeleteReq(SEncoder* pCoder, const SSingleDeleteReq* pReq); @@ -3160,8 +3158,8 @@ typedef struct { } SBatchMsg; typedef struct { - SMsgHead header; - SArray* pMsgs; //SArray + SMsgHead header; + SArray* pMsgs; // SArray } SBatchReq; typedef struct { @@ -3173,11 +3171,11 @@ typedef struct { } SBatchRspMsg; typedef struct { - SArray* pRsps; //SArray + SArray* pRsps; // SArray } SBatchRsp; -int32_t tSerializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq); -int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq); +int32_t tSerializeSBatchReq(void* buf, int32_t bufLen, SBatchReq* pReq); +int32_t tDeserializeSBatchReq(void* buf, int32_t bufLen, SBatchReq* pReq); static FORCE_INLINE void tFreeSBatchReqMsg(void* msg) { if (NULL == msg) { return; @@ -3186,8 +3184,8 @@ static FORCE_INLINE void tFreeSBatchReqMsg(void* msg) { taosMemoryFree(pMsg->msg); } -int32_t tSerializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp); -int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp); +int32_t tSerializeSBatchRsp(void* buf, int32_t bufLen, SBatchRsp* pRsp); +int32_t tDeserializeSBatchRsp(void* buf, int32_t bufLen, SBatchRsp* pRsp); static FORCE_INLINE void tFreeSBatchRspMsg(void* p) { if (NULL == p) { @@ -3198,11 +3196,10 @@ static FORCE_INLINE void tFreeSBatchRspMsg(void* p) { taosMemoryFree(pRsp->msg); } -int32_t tSerializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq); -int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq); -int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq); -int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq); - +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); #pragma pack(pop) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 7ca552f179..e80766d249 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -280,8 +280,8 @@ enum { TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, "vnode-stream-recover1", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, "vnode-stream-recover2", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cd97ceaae1..c7e98415d1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4496,7 +4496,7 @@ int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq) { if (num <= 0) { pReq->pMsgs = NULL; tEndDecode(&decoder); - + tDecoderClear(&decoder); return 0; } @@ -4511,7 +4511,7 @@ int32_t tDeserializeSBatchReq(void *buf, int32_t bufLen, SBatchReq *pReq) { if (tDecodeBinaryAlloc(&decoder, &msg.msg, NULL) < 0) return -1; if (NULL == taosArrayPush(pReq->pMsgs, &msg)) return -1; } - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4553,7 +4553,7 @@ int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp) { if (num <= 0) { pRsp->pRsps = NULL; tEndDecode(&decoder); - + tDecoderClear(&decoder); return 0; } @@ -4569,14 +4569,13 @@ int32_t tDeserializeSBatchRsp(void *buf, int32_t bufLen, SBatchRsp *pRsp) { if (tDecodeBinaryAlloc(&decoder, &msg.msg, NULL) < 0) return -1; if (NULL == taosArrayPush(pRsp->pRsps, &msg)) return -1; } - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4603,7 +4602,7 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->cgroup) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4634,7 +4633,7 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4664,7 +4663,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; - if (tEncodeBinary(&encoder, (uint8_t*)pReq->msg, pReq->msgLen) < 0) return -1; + if (tEncodeBinary(&encoder, (uint8_t *)pReq->msg, pReq->msgLen) < 0) return -1; tEndEncode(&encoder); @@ -4704,8 +4703,8 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; - if (tDecodeBinaryAlloc(&decoder, (void**)&pReq->msg, NULL) < 0) return -1; - + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, NULL) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4721,7 +4720,6 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { taosMemoryFreeClear(pReq->msg); } - int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -4768,14 +4766,13 @@ int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) { if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; @@ -4846,14 +4843,13 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -4902,7 +4898,7 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4939,14 +4935,13 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR if (tDecodeI32(&decoder, &pRsp->sversion) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->tversion) < 0) return -1; if (tDecodeI64(&decoder, &pRsp->affectedRows) < 0) return -1; - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; } - int32_t tSerializeSSchedulerHbReq(void *buf, int32_t bufLen, SSchedulerHbReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -6645,13 +6640,15 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) { int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { if (tEncodeCStr(pEncoder, pReq->tbname) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->startTs) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->endTs) < 0) return -1; return 0; } int32_t tDecodeSSingleDeleteReq(SDecoder *pDecoder, SSingleDeleteReq *pReq) { if (tDecodeCStrTo(pDecoder, pReq->tbname) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->ts) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->startTs) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->endTs) < 0) return -1; return 0; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 61f027039d..3d9ebec4c9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1127,7 +1127,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { SRpcMsg rpcMsg = { .code = 0, .contLen = len, - .msgType = TDMT_VND_STREAM_RECOVER_STEP2, + .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq, }; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index a4bfb6c876..5907be576a 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -21,14 +21,16 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl SBatchDeleteReq* deleteReq) { ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); int32_t totRow = pDataBlock->info.rows; - SColumnInfoData* pTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pDataBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); tqDebug("stream delete msg: row %d", totRow); for (int32_t row = 0; row < totRow; row++) { - int64_t ts = *(int64_t*)colDataGetData(pTsCol, row); + int64_t startTs = *(int64_t*)colDataGetData(pStartTsCol, row); + int64_t endTs = *(int64_t*)colDataGetData(pEndTsCol, row); int64_t groupId = *(int64_t*)colDataGetData(pGidCol, row); char* name; void* varTbName = NULL; @@ -42,8 +44,8 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl } else { name = buildCtbNameByGroupId(stbFullName, groupId); } - tqDebug("stream delete msg: vgId:%d, groupId :%" PRId64 ", name: %s, ts:%" PRId64, pVnode->config.vgId, groupId, - name, ts); + tqDebug("stream delete msg: vgId:%d, groupId :%" PRId64 ", name: %s, start ts:%" PRId64 "end ts:%" PRId64, + pVnode->config.vgId, groupId, name, startTs, endTs); #if 0 SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); @@ -59,7 +61,8 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl taosMemoryFree(name); #endif SSingleDeleteReq req = { - .ts = ts, + .startTs = startTs, + .endTs = endTs, }; strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN); taosMemoryFree(name); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c75d1ffded..2d87bb3b07 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -263,7 +263,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; - case TDMT_VND_STREAM_RECOVER_STEP2: { + case TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE: { if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } @@ -402,7 +402,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); - case TDMT_VND_STREAM_RECOVER_STEP1: + case TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE: return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); case TDMT_STREAM_RECOVER_FINISH: return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); @@ -1184,11 +1184,11 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void int64_t uid = mr.me.uid; - int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->ts, pOneReq->ts); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); if (code < 0) { terrno = code; vError("vgId:%d, delete error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64 ", end ts:%" PRId64, - TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->ts, pOneReq->ts); + TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs); } tDecoderClear(&mr.coder); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 68228f2689..474a067f9e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1313,8 +1313,8 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + tdbFree(tbname); } - tdbFree(tbname); pBlock->info.rows++; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a6a477a9e3..d598fd09ca 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3117,7 +3117,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (!winInfo.pOutputBuf) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -3242,8 +3242,8 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + tdbFree(tbname); } - tdbFree(tbname); pBlock->info.rows += 1; } if ((*Ite) == NULL) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 009f7eec9a..6a83a9a4da 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,7 +16,8 @@ #include "streamInc.h" static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - void* exec = pTask->exec.executor; + int32_t code; + void* exec = pTask->exec.executor; // set input const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; @@ -49,8 +50,10 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* while (1) { SSDataBlock* output = NULL; uint64_t ts = 0; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); + if ((code = qExecTask(exec, &output, &ts)) < 0) { + /*ASSERT(false);*/ + qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, + terrstr()); } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 2a2784afea..7eee95a580 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -36,7 +36,7 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_STEP1, + .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, }; if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 53d8dad226..bfce8b3151 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -76,7 +76,8 @@ int32_t tsem_wait(tsem_t* sem) { } int32_t tsem_timewait(tsem_t* sem, int64_t milis) { - return tsem_wait(sem); + return 0; + /*return tsem_wait(sem);*/ #if 0 struct timespec ts; timespec_get(&ts); From f565ff3769644504732f00fa123f8c041911fac1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 29 Nov 2022 23:42:20 +0800 Subject: [PATCH 69/82] fix: get cpuid on x86 only (#18552) * chore: taostools sanitizer flag align with tdengine * chore: re-enable old style build asan config for taos-tools * chore: update taos-tools 274d230 * chore: update taos-tools and add x86 definition * fix: get cpu id on x86 only --- cmake/cmake.platform | 2 ++ cmake/taostools_CMakeLists.txt.in | 2 +- include/os/os.h | 2 ++ source/os/src/osSysinfo.c | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index e822cc5e2c..711d74fa4c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -82,10 +82,12 @@ IF ("${CPUTYPE}" STREQUAL "") MESSAGE(STATUS "Current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") SET(TD_INTEL_64 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "Current platform is x86") SET(PLATFORM_ARCH_STR "i386") SET(TD_INTEL_32 TRUE) + ADD_DEFINITIONS("-D_TD_X86_") ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "Current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8b9104e7e5..79b447e20d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 9f587e9 + GIT_TAG d19e82c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/os/os.h b/include/os/os.h index f4592fc575..0688eeb9ad 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -48,8 +48,10 @@ extern "C" { #else #include #include +#if defined(_TD_X86_) #include #endif +#endif #else #ifndef __func__ diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f336b84e1e..7ec1da0530 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -481,6 +481,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { #elif defined(_TD_DARWIN_64) #else +#ifdef _TD_X86_ // Since the compiler is not support avx/avx2 instructions, the global variables always need to be // set to be false #if __AVX__ || __AVX2__ @@ -504,6 +505,7 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { // Ref to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77756 __cpuid_fix(7u, eax, ebx, ecx, edx); *avx2 = (char) ((ebx & bit_AVX2) == bit_AVX2); +#endif // _TD_X86_ #endif return 0; From df75d474d1bf8dbb62c2c5b1428b92007405d104 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Nov 2022 01:32:12 +0800 Subject: [PATCH 70/82] chore: taos-tools support sanitizer workflow --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 79b447e20d..c2ef22c338 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d19e82c + GIT_TAG cf30c86 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From b7cba5d68a3ea03202965f56f3055d5d8f77d89f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 30 Nov 2022 09:35:07 +0800 Subject: [PATCH 71/82] change test case --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0abec6b340..1c11853864 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -113,7 +113,7 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -,,y,script,./test.sh -f tsim/parser/limit2.sim +#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim From e9241629c31a15101479d1220d09f87b01033312 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 10:14:22 +0800 Subject: [PATCH 72/82] fix typo --- source/common/src/tdataformat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index dbf616f1c6..6d7c94b4a2 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2186,7 +2186,6 @@ int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { n += pColData->nData; } else { pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; - pColData->nData = pBuf + n; n += pColData->nData; } } From 1bca95bdadd1483d6eebe03a8a2d341cdc81f597 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 30 Nov 2022 09:47:15 +0800 Subject: [PATCH 73/82] fix:modify end ts --- source/libs/executor/src/tfill.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 68228f2689..dd00903ae7 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1427,7 +1427,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { streamStateFreeCur(pCur); pCur = streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } - endTs = nextKey.ts - 1; + endTs = TMAX(ts, nextKey.ts - 1); if (code != TSDB_CODE_SUCCESS) { break; } From b93b7aa5507868bcf5435a84a86e9da5c22f4e2a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 30 Nov 2022 10:22:43 +0800 Subject: [PATCH 74/82] enh: add table blokc order check --- source/libs/parser/inc/parInsertUtil.h | 3 +++ source/libs/parser/src/parInsertSql.c | 3 +++ source/libs/parser/src/parInsertUtil.c | 21 ++++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index ce14f822e4..61fbf1504b 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -153,6 +153,8 @@ typedef struct STableDataCxt { SBoundColInfo boundColsInfo; SArray *pValues; SSubmitTbData *pData; + TSKEY lastTs; + bool ordered; } STableDataCxt; typedef struct SVgroupDataCxt { @@ -161,6 +163,7 @@ typedef struct SVgroupDataCxt { } SVgroupDataCxt; int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 03068ba83e..3e631b197f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1175,6 +1175,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC if (TSDB_CODE_SUCCESS == code) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + } } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index a138b684d1..a271165d5d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -981,6 +981,20 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { return TSDB_CODE_SUCCESS; } +void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { + // once the data block is disordered, we do NOT keep previous timestamp any more + if (!pTableCxt->ordered) { + return; + } + + if (tsKey <= pTableCxt->lastTs) { + pTableCxt->ordered = false; + } + + pTableCxt->lastTs = tsKey; + return; +} + static void destroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput) { @@ -991,6 +1005,9 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat int32_t code = TSDB_CODE_SUCCESS; + pTableCxt->ordered = true; + pTableCxt->lastTs = 0; + pTableCxt->pMeta = tableMetaDup(pTableMeta); if (NULL == pTableCxt->pMeta) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1179,7 +1196,9 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { void* p = taosHashIterate(pTableHash, NULL); while (TSDB_CODE_SUCCESS == code && NULL != p) { STableDataCxt* pTableCxt = *(STableDataCxt**)p; - code = tRowMergeSort(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); + if (pTableCxt->ordered) { + code = tRowMergeSort(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); + } if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; From feaea3d12faec2fe85fd56d93275e26c7a2bbc5f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 10:23:44 +0800 Subject: [PATCH 75/82] adjust row api --- include/common/tdataformat.h | 3 ++- source/common/src/tdataformat.c | 14 ++++++++------ source/libs/parser/src/parInsertUtil.c | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 532c35165b..e8830e32f8 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -87,7 +87,8 @@ int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); void tRowDestroy(SRow *pRow); -int32_t tRowMergeSort(SArray *aRowP, STSchema *pTSchema, int8_t flag); +void tRowSort(SArray *aRowP); +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 6d7c94b4a2..ccba25873d 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -527,7 +527,7 @@ static int32_t tRowPCmprFn(const void *p1, const void *p2) { return 0; } static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } -static int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { +static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { int32_t code = 0; int32_t nRow = iEnd - iStart; @@ -591,12 +591,14 @@ _exit: if (code) tRowDestroy(pRow); return code; } -int32_t tRowMergeSort(SArray *aRowP, STSchema *pTSchema, int8_t flag) { - if (aRowP->size <= 1) return 0; - - int32_t code = 0; +void tRowSort(SArray *aRowP) { + if (TARRAY_SIZE(aRowP) <= 1) return; taosArraySort(aRowP, tRowPCmprFn); +} + +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { + int32_t code = 0; int32_t iStart = 0; while (iStart < aRowP->size) { @@ -612,7 +614,7 @@ int32_t tRowMergeSort(SArray *aRowP, STSchema *pTSchema, int8_t flag) { } if (iEnd - iStart > 1) { - code = tRowMerge(aRowP, pTSchema, iStart, iEnd, flag); + code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); } // the array is also changing, so the iStart just ++ instead of iEnd diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index a138b684d1..ce7df059d9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -1179,7 +1179,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { void* p = taosHashIterate(pTableHash, NULL); while (TSDB_CODE_SUCCESS == code && NULL != p) { STableDataCxt* pTableCxt = *(STableDataCxt**)p; - code = tRowMergeSort(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; From f5270f6a0c1bb7fe40a2f13a69644267cc5c88c1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 10:30:03 +0800 Subject: [PATCH 76/82] adjust api --- source/dnode/vnode/src/inc/tsdb.h | 12 +-- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 108 ++++++++++++------------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 +-- 4 files changed, 70 insertions(+), 70 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c58cf8f630..ed71891783 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -115,13 +115,13 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2); void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tRowMergerClear(SRowMerger *pMerger); -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow); -int32_t tRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index fda7618dd1..76b45ffec7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -261,12 +261,12 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, SR SRowMerger merger = {0}; STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - tRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema); + tsdbRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema); - tRowMerge(&merger, &tsdbRowFromTSRow(1, row)); + tsdbRowMerge(&merger, &tsdbRowFromTSRow(1, row)); - tRowMergerGetRow(&merger, &mergedRow); - tRowMergerClear(&merger); + tsdbRowMergerGetRow(&merger, &mergedRow); + tsdbRowMergerClear(&merger); taosMemoryFreeClear(pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f2515c6d48..d56851478b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1777,7 +1777,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1787,10 +1787,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1800,11 +1800,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1818,7 +1818,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1832,10 +1832,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1845,10 +1845,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1857,7 +1857,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1865,7 +1865,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1886,16 +1886,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1903,10 +1903,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } } else { // not merge block data - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1919,7 +1919,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1927,7 +1927,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } return TSDB_CODE_SUCCESS; @@ -1955,7 +1955,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1963,11 +1963,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1975,7 +1975,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } else { ASSERT(0); @@ -2056,7 +2056,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2067,10 +2067,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2081,7 +2081,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2089,7 +2089,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2108,10 +2108,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2126,7 +2126,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2140,11 +2140,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2159,10 +2159,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2173,7 +2173,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2181,7 +2181,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2191,7 +2191,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2199,7 +2199,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } @@ -2356,13 +2356,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2370,7 +2370,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } } @@ -3232,7 +3232,7 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe return terrno; } - tRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); } return TSDB_CODE_SUCCESS; @@ -3247,7 +3247,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tRowMerge(pMerger, &fRow); + tsdbRowMerge(pMerger, &fRow); rowIndex += step; } @@ -3344,7 +3344,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(pMerger, &fRow1); + tsdbRowMerge(pMerger, &fRow1); } else { break; } @@ -3394,7 +3394,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pReader->pSchema = pTSchema; } - int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + int32_t code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3404,19 +3404,19 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - tRowMergerAdd(&merge, pNextRow, pTSchema1); + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); *freeTSRow = true; return TSDB_CODE_SUCCESS; } @@ -3431,7 +3431,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3442,7 +3442,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3452,7 +3452,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3463,7 +3463,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3471,7 +3471,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } - int32_t code = tRowMergerGetRow(&merge, pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8770981022..b019bf3165 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -642,7 +642,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { // SRowMerger ====================================================== -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -697,7 +697,7 @@ _exit: return code; } -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -736,7 +736,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { return code; } -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -775,9 +775,9 @@ _exit: return code; } -void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } +void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -807,7 +807,7 @@ _exit: return code; } -int32_t tRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } From 0277ec35b5a3f992c18753d377aa74a3d7b37ef0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 10:32:59 +0800 Subject: [PATCH 77/82] fix a line of code --- source/common/src/tdataformat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ccba25873d..218c8e5682 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2187,6 +2187,7 @@ int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { pColData->pData = pBuf + n; n += pColData->nData; } else { + pColData->pData = pBuf + n; pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; n += pColData->nData; } From 9e39ffc848763287803dc8cbc137a5c46d1785bb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 30 Nov 2022 10:45:12 +0800 Subject: [PATCH 78/82] enh: add table blokc order check --- source/libs/parser/inc/parInsertUtil.h | 1 + source/libs/parser/src/parInsertUtil.c | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 61fbf1504b..b74ba8d299 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -155,6 +155,7 @@ typedef struct STableDataCxt { SSubmitTbData *pData; TSKEY lastTs; bool ordered; + bool duplicateTs; } STableDataCxt; typedef struct SVgroupDataCxt { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index b8158ea7ed..b5309e16bf 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -982,15 +982,19 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { } void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { - // once the data block is disordered, we do NOT keep previous timestamp any more + // once the data block is disordered, we do NOT keep last timestamp any more if (!pTableCxt->ordered) { return; } - if (tsKey <= pTableCxt->lastTs) { + if (tsKey < pTableCxt->lastTs) { pTableCxt->ordered = false; } + if (tsKey == pTableCxt->lastTs) { + pTableCxt->duplicateTs = true; + } + pTableCxt->lastTs = tsKey; return; } @@ -1005,8 +1009,9 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat int32_t code = TSDB_CODE_SUCCESS; - pTableCxt->ordered = true; pTableCxt->lastTs = 0; + pTableCxt->ordered = true; + pTableCxt->duplicateTs = false; pTableCxt->pMeta = tableMetaDup(pTableMeta); if (NULL == pTableCxt->pMeta) { @@ -1196,10 +1201,12 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { void* p = taosHashIterate(pTableHash, NULL); while (TSDB_CODE_SUCCESS == code && NULL != p) { STableDataCxt* pTableCxt = *(STableDataCxt**)p; - if (pTableCxt->ordered) { + if (!pTableCxt->ordered) { tRowSort(pTableCxt->pData->aRowP); } - code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); + if (!pTableCxt->ordered || pTableCxt->duplicateTs) { + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); + } if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; From b22ec52c63ef0a4199c2a87414b39d79a952252c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 14:01:45 +0800 Subject: [PATCH 79/82] fix core dump --- include/common/tmsg.h | 4 +-- source/common/src/tmsg.c | 34 +++++++++++++++++++------- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/parser/src/parInsertUtil.c | 4 +-- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a52b9d61df..d32414680b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3230,8 +3230,8 @@ typedef struct { int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); -void tDestroySSubmitTbData(SSubmitTbData* pTbData); -void tDestroySSubmitReq2(SSubmitReq2* pReq); +void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); typedef struct { int32_t code; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 391fb86748..22f872c005 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6809,30 +6809,46 @@ _exit: return code; } -void tDestroySSubmitTbData(SSubmitTbData *pTbData) { +void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { if (pTbData->pCreateTbReq) { - // todo taosMemoryFree(pTbData->pCreateTbReq); } - if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - ASSERT(0); // TODO - } else { - if (pTbData->aRowP) { + if (flag == TSDB_MSG_FLG_ENCODE) { + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { int32_t nRow = TARRAY_SIZE(pTbData->aRowP); SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + for (int32_t i = 0; i < nRow; ++i) { tRowDestroy(rows[i]); } taosArrayDestroy(pTbData->aRowP); + } else { + int32_t nColData = TARRAY_SIZE(pTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); + + for (int32_t i = 0; i < nColData; ++i) { + tColDataDestroy(&aColData[i]); + } + taosArrayDestroy(pTbData->aCol); + } + } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + taosArrayDestroy(pTbData->aCol); + } else { + taosArrayDestroy(pTbData->aRowP); } } } -void tDestroySSubmitReq2(SSubmitReq2 *pReq) { - if (NULL == pReq) return; +void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); - taosArrayDestroyEx(pReq->aSubmitTbData, (FDelete)tDestroySSubmitTbData); + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySSubmitTbData(&aSubmitTbData[i], flag); + } + taosArrayDestroy(pReq->aSubmitTbData); } int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 91a9341f1f..4399d5993a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -978,7 +978,7 @@ _exit: // clear taosArrayDestroy(newTbUids); - tDestroySSubmitReq2(pSubmitReq); + tDestroySSubmitReq2(pSubmitReq, TSDB_MSG_FLG_DECODE); tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); return code; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 8a934cc11e..22551a6033 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -1092,7 +1092,7 @@ void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { destroyBoundColInfo(&pTableCxt->boundColsInfo); taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); if (pTableCxt->pData) { - tDestroySSubmitTbData(pTableCxt->pData); + tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pTableCxt->pData); } taosMemoryFree(pTableCxt); @@ -1103,7 +1103,7 @@ void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { return; } - tDestroySSubmitReq2(pVgCxt->pData); + tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pVgCxt); } From 2a27c2d8f4ec5d6e05df2d67538b9e507a8ba8f2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 14:09:06 +0800 Subject: [PATCH 80/82] fix coredump --- source/common/src/tmsg.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 22f872c005..623816323f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6816,14 +6816,6 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { if (flag == TSDB_MSG_FLG_ENCODE) { if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - int32_t nRow = TARRAY_SIZE(pTbData->aRowP); - SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); - - for (int32_t i = 0; i < nRow; ++i) { - tRowDestroy(rows[i]); - } - taosArrayDestroy(pTbData->aRowP); - } else { int32_t nColData = TARRAY_SIZE(pTbData->aCol); SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); @@ -6831,6 +6823,14 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { tColDataDestroy(&aColData[i]); } taosArrayDestroy(pTbData->aCol); + } else { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); } } else if (flag == TSDB_MSG_FLG_DECODE) { if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { From 5d5e16338d6e0b320811e40ab9a8fc55ed34cad5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 14:27:02 +0800 Subject: [PATCH 81/82] more code --- source/common/src/tmsg.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 22f872c005..5715b20f3a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6670,7 +6670,14 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm if (tEncodeI32v(pCoder, pSubmitTbData->sver) < 0) return -1; if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - ASSERT(0); // TODO + uint64_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (tEncodeU64v(pCoder, nColData) < 0) return -1; + + for (uint64_t i = 0; i < nColData; i++) { + pCoder->pos += tPutColData(pCoder->data ? pCoder->data + pCoder->pos : NULL, &aColData[i]); + } } else { if (tEncodeU64v(pCoder, TARRAY_SIZE(pSubmitTbData->aRowP)) < 0) return -1; @@ -6723,7 +6730,22 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa } if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - ASSERT(0); // TODO + uint64_t nColData; + + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aCol = taosArrayInit(nColData, nColData); + if (pSubmitTbData->aCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nColData; ++i) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, taosArrayReserve(pSubmitTbData->aCol, 1)); + } } else { uint64_t nRow; if (tDecodeU64v(pCoder, &nRow) < 0) { From dc597124523413064255bd136db9dd82f700344a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Nov 2022 14:49:18 +0800 Subject: [PATCH 82/82] add more code --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 42 ++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e8830e32f8..82cbbcf6a4 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -122,7 +122,7 @@ extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, // for stmt bind int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); -int32_t tColDataSortMerge(SColData *aColData); +void tColDataSortMerge(SArray *colDataArr); // for encode/decode int32_t tPutColData(uint8_t *pBuf, SColData *pColData); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 218c8e5682..b5946a5847 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2106,9 +2106,45 @@ _exit: return code; } -int32_t tColDataSortMerge(SColData *aColData) { - // todo - return 0; +void tColDataSortMerge(SArray *colDataArr) { + int32_t nColData = TARRAY_SIZE(colDataArr); + SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); + + if (aColData[0].nVal <= 1) goto _exit; + + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].flag == HAS_VALUE); + + int8_t doSort = 0; + int8_t doMerge = 0; + // scan ------- + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] > aKey[iVal - 1]) { + continue; + } else if (aKey[iVal] < aKey[iVal - 1]) { + doSort = 1; + break; + } else { + doMerge = 1; + } + } + + // sort ------- + if (doSort) { + ASSERT(0); + // todo + } + + // merge ------- + if (doMerge) { + ASSERT(0); + // todo + } + +_exit: + return; } int32_t tPutColData(uint8_t *pBuf, SColData *pColData) {