From b96e434ab3aea54dffc68448d7fa1c773de6f160 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 27 Apr 2022 18:10:57 +0800 Subject: [PATCH 01/90] format --- include/libs/sync/sync.h | 2 +- source/libs/sync/src/syncEnv.c | 6 +++--- source/libs/sync/src/syncMain.c | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ffe435a8b2..1176094630 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -89,7 +89,7 @@ typedef struct SSyncFSM { struct SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 // abstract definition of log store in raft diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 4b2bc4130a..945d59646b 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -37,7 +37,7 @@ bool syncEnvIsStart() { int32_t syncEnvStart() { int32_t ret = 0; taosSeedRand(taosGetTimestampSec()); - //gSyncEnv = doSyncEnvStart(gSyncEnv); + // gSyncEnv = doSyncEnvStart(gSyncEnv); gSyncEnv = doSyncEnvStart(); assert(gSyncEnv != NULL); sTrace("syncEnvStart ok!"); @@ -97,14 +97,14 @@ static SSyncEnv *doSyncEnvStart() { // start tmr thread pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); - atomic_store_8(&(pSyncEnv->isStart), 1); + atomic_store_8(&(pSyncEnv->isStart), 1); return pSyncEnv; } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { assert(pSyncEnv == gSyncEnv); if (pSyncEnv != NULL) { - atomic_store_8(&(pSyncEnv->isStart), 0); + atomic_store_8(&(pSyncEnv->isStart), 0); taosTmrCleanUp(pSyncEnv->pTimerManager); taosMemoryFree(pSyncEnv); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 262c9d5530..2dda089349 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -459,18 +459,17 @@ void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { syncNodeBecomeLeader(pSyncNode); - + syncNodeLog2("==state change become leader immediately==", pSyncNode); - + // Raft 3.6.2 Committing entries from previous terms - + // use this now syncNodeAppendNoop(pSyncNode); syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica return; } - syncNodeBecomeFollower(pSyncNode); // for test From a273559e93ed613f995740498c4ce8773cb0c85a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 27 Apr 2022 19:13:57 +0800 Subject: [PATCH 02/90] add send redirect --- include/common/tmsgcb.h | 3 +++ source/common/src/tmsgcb.c | 4 ++++ source/dnode/mgmt/implement/src/dmTransport.c | 20 ++++++++++++++----- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 18 +++++++++-------- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 02d8b76b9b..4002db06ea 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -42,6 +42,7 @@ typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueTy typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq); typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp); +typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type); typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc); @@ -52,6 +53,7 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; + SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -62,6 +64,7 @@ int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq); void tmsgSendRsp(const SRpcMsg* pRsp); +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg); void tmsgReleaseHandle(void* handle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 5b4bb539e3..c9cbb73884 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -34,6 +34,10 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } +void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { + return (*tsDefaultMsgCb.sendRedirectRspFp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); +} + void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg); } diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index fb8e5e7fb2..7bfb497d8c 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -128,10 +128,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -309,6 +309,15 @@ static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_NODE_REDIRECT); + if (pWrapper->procType != DND_PROC_CHILD) { + rpcSendRedirectRsp(pRsp->handle, pNewEpSet); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} + static inline void dmRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != DND_PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); @@ -464,7 +473,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -538,6 +547,7 @@ SMsgCb dmGetMsgcb(SMgmtWrapper *pWrapper) { SMsgCb msgCb = { .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, + .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartupByWrapper, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7285503a73..040df158e4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -136,7 +136,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // sync integration response for (int i = 0; i < taosArrayGetSize(pArray); i++) { SNodeMsg *pMsg; - SRpcMsg * pRpc; + SRpcMsg *pRpc; pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); pRpc = &pMsg->rpcMsg; @@ -151,6 +151,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { rsp.code = TSDB_CODE_SYN_NOT_LEADER; tmsgSendRsp(&rsp); + // SEpSet epSet; TODO + } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; tmsgSendRsp(&rsp); @@ -175,7 +177,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -218,7 +220,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -231,7 +233,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg * pMsg = NULL; + SNodeMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -248,7 +250,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { - SRpcMsg * pRpc = &pMsg->rpcMsg; + SRpcMsg *pRpc = &pMsg->rpcMsg; SMsgHead *pHead = pRpc->pCont; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); @@ -317,7 +319,7 @@ int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt * pMgmt = pWrapper->pMgmt; + SVnodesMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); @@ -325,7 +327,7 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt * pMgmt = pWrapper->pMgmt; + SVnodesMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -335,7 +337,7 @@ int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; - SMsgHead * pHead = pRpc->pCont; + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; From 9a603170b422ea75303f25779bfab17ea2c987f9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 27 Apr 2022 19:28:01 +0800 Subject: [PATCH 03/90] add redirect before syncPropose --- include/libs/sync/sync.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++++++--- source/libs/sync/src/syncMain.c | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1176094630..a83c6caa1c 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -149,6 +149,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); ESyncState syncGetMyRole(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); +void syncGetEpSet(int64_t rid, SEpSet* pEpSet); typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 040df158e4..dfb5c56ea5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -149,9 +149,13 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - rsp.code = TSDB_CODE_SYN_NOT_LEADER; - tmsgSendRsp(&rsp); - // SEpSet epSet; TODO + // rsp.code = TSDB_CODE_SYN_NOT_LEADER; + // tmsgSendRsp(&rsp); + + rsp.code = TSDB_CODE_RPC_REDIRECT; + SEpSet newEpSet; + syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); + tmsgSendRedirectRsp(&rsp, &newEpSet); } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 2dda089349..fbb906e4d7 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -153,6 +153,24 @@ SyncTerm syncGetMyTerm(int64_t rid) { return term; } +void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + memset(pEpSet, 0, sizeof(*pEpSet)); + return; + } + assert(rid == pSyncNode->rid); + pEpSet->numOfEps = 0; + for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + snprintf(pEpSet->eps->fqdn, sizeof(pEpSet->eps->fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); + pEpSet->eps->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + (pEpSet->numOfEps)++; + } + pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); +} + int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { From 2f66f50c0428bd45759fb13f84b4f3a94e11dca4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Apr 2022 14:49:25 +0800 Subject: [PATCH 04/90] refactor(rpc): refactor timeout --- source/libs/index/inc/indexUtil.h | 4 ++-- source/libs/transport/src/transCli.c | 1 - source/libs/transport/src/transComm.c | 11 ++++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/index/inc/indexUtil.h b/source/libs/index/inc/indexUtil.h index 814d61afd7..f1676ed411 100644 --- a/source/libs/index/inc/indexUtil.h +++ b/source/libs/index/inc/indexUtil.h @@ -68,7 +68,7 @@ extern "C" { */ void iIntersection(SArray *interResults, SArray *finalResult); -/* multi sorted result intersection +/* multi sorted result union * input: [1, 2, 4, 5] * [2, 3, 4, 5] * [1, 4, 5] @@ -76,7 +76,7 @@ void iIntersection(SArray *interResults, SArray *finalResult); */ void iUnion(SArray *interResults, SArray *finalResult); -/* sorted array +/* see example * total: [1, 2, 4, 5, 7, 8] * except: [4, 5] * return: [1, 2, 7, 8] saved in total diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e02bfbafd8..c3b40aa8e0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -892,7 +892,6 @@ static void doDelayTask(void* param) { SCliMsg* pMsg = arg->param1; SCliThrdObj* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); taosMemoryFree(arg); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 01a20a466a..1bd7d0857e 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -425,10 +425,19 @@ void transDQDestroy(SDelayQueue* queue) { } int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { + uint64_t now = taosGetTimestampMs(); SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask)); task->func = func; task->arg = arg; - task->execTime = taosGetTimestampMs() + timeoutMs; + task->execTime = now + timeoutMs; + + HeapNode* minNode = heapMin(queue->heap); + if (minNode) { + SDelayTask* minTask = container_of(minNode, SDelayTask, node); + if (minTask->execTime < task->execTime) { + timeoutMs = minTask->execTime <= now ? 0 : now - minTask->execTime; + } + } tTrace("timer %p put task into queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); From 485bd970a23fa00aacc6bebb1a7c19a05f5cf913 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 28 Apr 2022 15:49:00 +0800 Subject: [PATCH 05/90] refactor(rpc): refactor timeout --- source/libs/transport/src/transSrv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b78edf6cd0..92341bf872 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -808,18 +808,18 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); int fds[2]; - if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; } uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - thrd->pTransInst = shandle; thrd->fd = fds[0]; thrd->pipe = &(srv->pipe[i][1]); // init read From 4d40af9929046042a050b1df997464508b98a08e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 28 Apr 2022 16:41:13 +0800 Subject: [PATCH 06/90] add redirect --- include/libs/sync/sync.h | 1 + source/dnode/mgmt/implement/src/dmTransport.c | 11 ++++++++++- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/libs/sync/src/syncMain.c | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a83c6caa1c..ca10465ed4 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -150,6 +150,7 @@ ESyncState syncGetMyRole(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); +int32_t syncGetVgId(int64_t rid); typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 7bfb497d8c..7abf5d65cd 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -310,7 +310,7 @@ static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { - ASSERT(pRsp->code == TSDB_CODE_NODE_REDIRECT); + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); if (pWrapper->procType != DND_PROC_CHILD) { rpcSendRedirectRsp(pRsp->handle, pNewEpSet); } else { @@ -402,6 +402,14 @@ SProcCfg dmGenProcCfg(SMgmtWrapper *pWrapper) { return cfg; } +bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + static int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -416,6 +424,7 @@ static int32_t dmInitClient(SDnode *pDnode) { rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; rpcInit.parent = pDnode; + rpcInit.rfp = rpcRfp; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index dfb5c56ea5..bfeec26d23 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -151,7 +151,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { // rsp.code = TSDB_CODE_SYN_NOT_LEADER; // tmsgSendRsp(&rsp); - + dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl))); rsp.code = TSDB_CODE_RPC_REDIRECT; SEpSet newEpSet; syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbb906e4d7..857b572cab 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -141,6 +141,18 @@ const char* syncGetMyRoleStr(int64_t rid) { return s; } +int32_t syncGetVgId(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_STATE_ERROR; + } + assert(rid == pSyncNode->rid); + int32_t vgId = pSyncNode->vgId; + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return vgId; +} + SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -278,6 +290,8 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + sTrace("syncPropose msgType:%d ", pMsg->msgType); + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { From 193a84717d5bb449b66dfaf2b7db2031db041530 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 29 Apr 2022 18:20:50 +0800 Subject: [PATCH 07/90] udaf pass the first time --- source/libs/function/src/tudf.c | 17 +++++++++-------- source/libs/function/src/udfd.c | 2 +- source/libs/function/test/udf2.c | 4 ++-- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index df46999379..9aac2bfe0c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1176,7 +1176,7 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte // input: interBuf // output: resultData int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { - int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int8_t callType = TSDB_UDF_CALL_AGG_FIN; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; } @@ -1243,12 +1243,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } SUdfUvSession *session = (SUdfUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; memset(udfRes, 0, envSize); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->session = (SUdfUvSession *)handle; SUdfInterBuf buf = {0}; if (callUdfAggInit(handle, &buf) != 0) { @@ -1260,7 +1260,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; int32_t numOfCols = pInput->numOfInputCols; @@ -1320,13 +1319,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf, - .bufLen = session->outputLen, - .numOfResult = udfRes->finalResNum}; + SUdfInterBuf resultBuf = {0}; SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; callUdfAggFinalize(session, &state, &resultBuf); + + udfRes->finalResBuf = resultBuf.buf; + udfRes->finalResNum = resultBuf.numOfResult; + teardownUdf(session); if (resultBuf.numOfResult == 1) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ae24a832c8..06fa49e1c2 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -124,7 +124,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char *finishSuffix = "_finish"; strncpy(finishFuncName, processFuncName, strlen(processFuncName)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc)); + uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); //TODO: merge } return 0; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 83187c5855..69ed515d2b 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -27,7 +27,7 @@ int32_t udf2_start(SUdfInterBuf *buf) { int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { int64_t sumSquares = *(int64_t*)interBuf->buf; for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { SUdfColumn* col = block->udfCols[i]; //TODO: check the bitmap for null value int32_t* rows = (int32_t*)col->colData.fixLenCol.data; @@ -35,7 +35,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte } } - *(int64_t*)newInterBuf = sumSquares; + *(int64_t*)(newInterBuf->buf) = sumSquares; newInterBuf->bufLen = sizeof(int64_t); //TODO: if all null value, numOfResult = 0; newInterBuf->numOfResult = 1; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f1fe68c55a..92da476319 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -154,6 +154,7 @@ static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicCond } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0e6ec4f945..828044623d 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1885,6 +1885,7 @@ static const char* jkFunctionName = "Name"; static const char* jkFunctionId = "Id"; static const char* jkFunctionType = "Type"; static const char* jkFunctionParameter = "Parameters"; +static const char* jkFunctionUdfBufSize = "UdfBufSize"; static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { const SFunctionNode* pNode = (const SFunctionNode*)pObj; @@ -1902,6 +1903,9 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkFunctionParameter, pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionUdfBufSize, pNode->udfBufSize); + } return code; } @@ -1922,6 +1926,9 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkFunctionParameter, &pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionUdfBufSize, &pNode->udfBufSize); + } return code; } From c3484ce8c372a14031476e8b8b7318f532a18487 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 29 Apr 2022 20:04:34 +0800 Subject: [PATCH 08/90] stmt query --- source/client/src/clientStmt.c | 18 +- source/libs/scalar/src/sclvector.c | 22 +- tests/script/api/batchprepare.c | 1100 ++-------------------------- 3 files changed, 86 insertions(+), 1054 deletions(-) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5ddffa0cbd..f7ef1f7e81 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -649,6 +649,19 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { + pStmt->bInfo.needParse = false; + } + + if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) { + taos_free_result(pStmt->exec.pRequest); + pStmt->exec.pRequest = NULL; + } + + if (NULL == pStmt->exec.pRequest) { + STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); + } + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } @@ -658,8 +671,11 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList)); pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag; pStmt->exec.pRequest->body.pDag = NULL; + STMT_ERR_RET(stmtBackupQueryFields(pStmt)); + } else { + STMT_ERR_RET(stmtRestoreQueryFields(pStmt)); } - + *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues); } else { STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a75b2521bd..84aa5559d0 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -297,6 +297,22 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn taosMemoryFree(t); } +static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) { + int32_t inputLen = varDataLen(buf); + + char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + taosMemoryFree(t); + return; + } + varDataSetLen(t, len); + + colDataAppend(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); +} + + //TODO opt performance, tmp is not needed. int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { int32_t bufSize = pIn->columnData->info.bytes; @@ -313,6 +329,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in func = varToUnsigned; } else if (IS_FLOAT_TYPE(outType)) { func = varToFloat; + } else if (outType == TSDB_DATA_TYPE_BINARY) { // nchar -> binary + ASSERT(inType == TSDB_DATA_TYPE_NCHAR); + func = ncharToVar; + vton = true; } else if (outType == TSDB_DATA_TYPE_NCHAR) { // binary -> nchar ASSERT(inType == TSDB_DATA_TYPE_VARCHAR); func = varToNchar; @@ -608,7 +628,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 0, 7, 0, 0, /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, -/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 7, 7, 7, 0, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f14885b72e..7bea19ae90 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -11,8 +11,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR, TSDB_DATA_TYPE_BOOL}; -int32_t optrIdxList[] = {2, 11, 6}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE}; +int32_t optrIdxList[] = {0, 1, 2}; typedef struct { char* oper; @@ -169,8 +169,8 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 -CaseCtrl gCaseCtrl = { +#if 0 +CaseCtrl gCaseCtrl = { // default .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -186,23 +186,15 @@ CaseCtrl gCaseCtrl = { .checkParamNum = false, .printRes = true, .runTimes = 0, -// .caseIdx = -1, -// .caseNum = -1, + .caseIdx = -1, + .caseNum = -1, .caseRunIdx = -1, -// .caseRunNum = -1, - - -// .optrIdxListNum = tListLen(optrIdxList), -// .optrIdxList = optrIdxList, -// .bindColTypeNum = tListLen(bindColTypeList), -// .bindColTypeList = bindColTypeList, - .caseIdx = 22, - .caseNum = 1, - .caseRunNum = 1, - + .caseRunNum = -1, }; -#else -CaseCtrl gCaseCtrl = { +#endif + +#if 0 +CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindNullNum = 0, .prepareStb = false, .printCreateTblSql = false, @@ -211,17 +203,50 @@ CaseCtrl gCaseCtrl = { .rowNum = 0, .bindColNum = 0, .bindRowNum = 0, - .bindColTypeNum = tListLen(bindColTypeList), - .bindColTypeList = bindColTypeList, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, .checkParamNum = false, .printRes = true, .runTimes = 0, - .caseIdx = 2, - .caseNum = 1, .caseRunIdx = -1, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .caseIdx = 22, + .caseNum = 1, .caseRunNum = 1, }; +#endif +#if 1 +CaseCtrl gCaseCtrl = { // query case with specified col&oper + .bindNullNum = 0, + .prepareStb = false, + .printCreateTblSql = false, + .printQuerySql = true, + .printStmtSql = true, + .rowNum = 0, + .bindColNum = 0, + .bindRowNum = 0, + .bindColTypeNum = 0, + .bindColTypeList = NULL, + .optrIdxListNum = 0, + .optrIdxList = NULL, + .checkParamNum = true, + .printRes = true, + .runTimes = 0, + .caseRunIdx = -1, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseIdx = 22, + .caseNum = 1, + .caseRunNum = 1, +}; #endif int32_t taosGetTimeOfDay(struct timeval *tv) { @@ -3218,1035 +3243,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { return 0; } -//samets -int stmt_funcb4(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - -//1table 18000 reocrds -int stmt_funcb5(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[18000]; - int8_t v1[18000]; - int16_t v2[18000]; - int32_t v4[18000]; - int64_t v8[18000]; - float f4[18000]; - double f8[18000]; - char bin[18000][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(18000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 18000); - char* no_null = taosMemoryMalloc(sizeof(char) * 18000); - - for (int i = 0; i < 18000; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[18000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 18000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 18000; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 18000; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 18000; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 18000; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 18000; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 18000; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 18000; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 18000; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 18000; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into m0 values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 1; zz++) { - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1table 200000 reocrds -int stmt_funcb_ssz1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int b[30000]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 30000 * 3000); - - int *lb = taosMemoryMalloc(30000 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 3000*10); - char* no_null = taosMemoryMalloc(sizeof(int) * 200000); - - for (int i = 0; i < 30000; ++i) { - lb[i] = 40; - no_null[i] = 0; - v.b[i] = (int8_t)(i % 2); - } - - for (int i = 0; i < 30000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[30000*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 30000; - - params[i+1].buffer_type = TSDB_DATA_TYPE_INT; - params[i+1].buffer_length = sizeof(int); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = no_null; - params[i+1].num = 30000; - } - - int64_t tts = 0; - for (int64_t i = 0; i < 90000000LL; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 10; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - ++id; - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(no_null); - - return 0; -} - - -//one table 60 records one time -int stmt_funcb_s1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - taos_stmt_bind_param_batch(stmt, params + id * 10); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - ++id; - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - - - - - -//300 tables 60 records single column bind -int stmt_funcb_sc1(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//1 tables 60 records single column bind -int stmt_funcb_sc2(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 900000 * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 900000*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 9000000; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[60*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 60; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 60; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 60; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 60; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 60; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 60; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 60; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 60; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 60; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 60; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 54000000; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int l = 0; l < 3000; l++) { - for (int zz = 0; zz < 300; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - } - - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} - - -//10 tables [1...10] records single column bind -int stmt_funcb_sc3(TAOS_STMT *stmt) { - struct { - int64_t *ts; - int8_t b[60]; - int8_t v1[60]; - int16_t v2[60]; - int32_t v4[60]; - int64_t v8[60]; - float f4[60]; - double f8[60]; - char bin[60][40]; - } v = {0}; - - v.ts = taosMemoryMalloc(sizeof(int64_t) * 60); - - int *lb = taosMemoryMalloc(60 * sizeof(int)); - - TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 60*10); - char* is_null = taosMemoryMalloc(sizeof(char) * 60); - char* no_null = taosMemoryMalloc(sizeof(char) * 60); - - for (int i = 0; i < 60; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - int g = 0; - for (int i = 0; i < 600; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = g%10+1; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = g%10+1; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = g%10+1; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = g%10+1; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = g%10+1; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = g%10+1; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = g%10+1; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = g%10+1; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = g%10+1; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = g%10+1; - ++g; - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 60; ++i) { - v.ts[i] = tts + i; - } - - unsigned long long starttime = taosGetTimestampUs(); - - char *sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; - int code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); - } - - int id = 0; - for (int zz = 0; zz < 10; zz++) { - char buf[32]; - sprintf(buf, "m%d", zz); - code = taos_stmt_set_tbname(stmt, buf); - if (code != 0){ - printf("failed to execute taos_stmt_set_tbname. code:0x%x\n", code); - } - - for (int col=0; col < 10; ++col) { - taos_stmt_bind_single_param_batch(stmt, params + id++, col); - } - - taos_stmt_add_batch(stmt); - } - - if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); - } - - unsigned long long endtime = taosGetTimestampUs(); - printf("insert total %d records, used %u seconds, avg:%u useconds\n", 3000*300*60, (endtime-starttime)/1000000UL, (endtime-starttime)/(3000*300*60)); - - taosMemoryFree(v.ts); - taosMemoryFree(lb); - taosMemoryFree(params); - taosMemoryFree(is_null); - taosMemoryFree(no_null); - - return 0; -} #endif From cbd39f831fdea7b61c70004838d5b3b9f168a9f3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 29 Apr 2022 20:20:07 +0800 Subject: [PATCH 09/90] fix(os): switch to JOM compilation. --- cmake/cmake.define | 1 - include/os/osDef.h | 2 +- source/libs/qworker/inc/qworkerInt.h | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index fb6ba8cc2e..851d8f9a73 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -44,7 +44,6 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) SET(COMMON_FLAGS "/W3 /D_WIN32") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) diff --git a/include/os/osDef.h b/include/os/osDef.h index 8da0c2c00c..6f6199de7a 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -62,7 +62,7 @@ extern "C" { #define strncasecmp _strnicmp #define wcsncasecmp _wcsnicmp #define strtok_r strtok_s - #define snprintf _snprintf + // #define snprintf _snprintf #define in_addr_t unsigned long // #define socklen_t int diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index a2b1353093..ca471262ff 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "osDef.h" #include "qworker.h" #include "tlockfree.h" #include "ttimer.h" @@ -301,9 +302,9 @@ typedef struct SQWorkerMgmt { extern SQWorkerMgmt gQwMgmt; -FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } +static FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } -FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } +static FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } #ifdef __cplusplus From 5d4a77dc80cd42e28101ff3adc4568b836a0f5f0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 29 Apr 2022 21:23:36 +0800 Subject: [PATCH 10/90] fix: invalid write in multi-process mode --- source/dnode/mgmt/implement/src/dmExec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index 06001028b5..bb0206d229 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -192,15 +192,15 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { dmStopNode(pWrapper); - taosWLockLatch(&pWrapper->latch); - (*pWrapper->fp.closeFp)(pWrapper); - taosWUnLockLatch(&pWrapper->latch); - if (pWrapper->procObj) { taosProcCleanup(pWrapper->procObj); pWrapper->procObj = NULL; } + taosWLockLatch(&pWrapper->latch); + (*pWrapper->fp.closeFp)(pWrapper); + taosWUnLockLatch(&pWrapper->latch); + dInfo("node:%s, has been closed", pWrapper->name); } From 50bba1397cd04d1f95f17be96a29b0ffad403c6a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 29 Apr 2022 21:32:54 +0800 Subject: [PATCH 11/90] feat: bsma logic optimization --- include/common/tdataformat.h | 8 ++++---- include/common/tmsg.h | 4 +++- source/common/src/tdataformat.c | 12 ++++++------ source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a0f4351bbf..36286d5944 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -64,19 +64,19 @@ typedef struct { col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) int32_t type : 8; // column type int32_t bytes : 24; // column bytes (0~16M) - int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... + int32_t flags : 8; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON int32_t offset : 24; // point offset in STpRow after the header part. } STColumn; #pragma pack(pop) #define colType(col) ((col)->type) -#define colSma(col) ((col)->sma) +#define colFlags(col) ((col)->flags) #define colColId(col) ((col)->colId) #define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) #define colSetType(col, t) (colType(col) = (t)) -#define colSetSma(col, s) (colSma(col) = (s)) +#define colSetFlags(col, f) (colFlags(col) = (f)) #define colSetColId(col, id) (colColId(col) = (id)) #define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) @@ -146,7 +146,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3d6bb06015..cc418b05a3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -281,8 +281,10 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == SCHEMA_SMA_ON) + #define SSCHMEA_TYPE(s) ((s)->type) -#define SSCHMEA_SMA(s) ((s)->sma) +#define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 2ed08ac81f..df598046e1 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -87,7 +87,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); tlen += taosEncodeFixedI8(buf, colType(pCol)); - tlen += taosEncodeFixedI8(buf, colSma(pCol)); + tlen += taosEncodeFixedI8(buf, colFlags(pCol)); tlen += taosEncodeFixedI16(buf, colColId(pCol)); tlen += taosEncodeFixedI16(buf, colBytes(pCol)); } @@ -110,14 +110,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; - int8_t sma = 0; + int8_t flags = 0; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); - buf = taosDecodeFixedI8(buf, &sma); + buf = taosDecodeFixedI8(buf, &flags); buf = taosDecodeFixedI16(buf, &colId); buf = taosDecodeFixedI32(buf, &bytes); - if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) { + if (tdAddColToSchema(&schemaBuilder, type, flags, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; } @@ -153,7 +153,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { pBuilder->version = version; } -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) { +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { @@ -166,7 +166,7 @@ int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); colSetType(pCol, type); colSetColId(pCol, colId); - colSetSma(pCol, sma); + colSetFlags(pCol, flags); if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 6a8af73694..fc23534247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -935,7 +935,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlockCol->type = pDataCol->type; pAggrBlkCol->colId = pDataCol->colId; - if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) { + if (isSuper && IS_BSMA_ON(pColumn) && tDataTypes[pDataCol->type].statisFunc) { #if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bf35241dc4..7d3f7cadc9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3330,7 +3330,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat int32_t* slotIds = pHandle->suppInfo.slotIds; for (int32_t i = 1; i < numOfCols; ++i) { ASSERT(colIds[i] == pHandle->pSchema->columns[slotIds[i]].colId); - if (pHandle->pSchema->columns[slotIds[i]].sma) { + if (IS_BSMA_ON(&(pHandle->pSchema->columns[slotIds[i]]))) { if (pHandle->suppInfo.pstatis[i].numOfNull == -1) { // set the column data are all NULL pHandle->suppInfo.pstatis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } else { From 53720052eddb240889f7fd152596129e98c9aaf8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 29 Apr 2022 21:48:28 +0800 Subject: [PATCH 12/90] fix: invalid write in multi-process mode --- source/dnode/mgmt/implement/src/dmExec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmExec.c b/source/dnode/mgmt/implement/src/dmExec.c index bb0206d229..06001028b5 100644 --- a/source/dnode/mgmt/implement/src/dmExec.c +++ b/source/dnode/mgmt/implement/src/dmExec.c @@ -192,15 +192,15 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { dmStopNode(pWrapper); + taosWLockLatch(&pWrapper->latch); + (*pWrapper->fp.closeFp)(pWrapper); + taosWUnLockLatch(&pWrapper->latch); + if (pWrapper->procObj) { taosProcCleanup(pWrapper->procObj); pWrapper->procObj = NULL; } - taosWLockLatch(&pWrapper->latch); - (*pWrapper->fp.closeFp)(pWrapper); - taosWUnLockLatch(&pWrapper->latch); - dInfo("node:%s, has been closed", pWrapper->name); } From 2584b034df5ae8338c8cc4df88802900e3ac84dd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 30 Apr 2022 10:29:29 +0800 Subject: [PATCH 13/90] add syncGetEpSet --- source/dnode/mgmt/implement/src/dmTransport.c | 22 +++++++++++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 1 + source/libs/sync/src/syncMain.c | 9 ++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 1dabc83f57..446894556e 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -320,6 +320,27 @@ static inline void dmSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } +static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { + ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); + ASSERT(pRsp->pCont == NULL); + if (pWrapper->procType != DND_PROC_CHILD) { + SRpcMsg resp = {0}; + SMEpSet msg = {.epSet = *pNewEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + resp.pCont = rpcMallocCont(len); + resp.contLen = len; + tSerializeSMEpSet(resp.pCont, len, &msg); + + resp.code = TSDB_CODE_RPC_REDIRECT; + resp.handle = pRsp->handle; + resp.refId = pRsp->refId; + rpcSendResponse(&resp); + } else { + taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); + } +} + +#if 0 static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp, const SEpSet *pNewEpSet) { ASSERT(pRsp->code == TSDB_CODE_RPC_REDIRECT); if (pWrapper->procType != DND_PROC_CHILD) { @@ -328,6 +349,7 @@ static inline void dmSendRedirectRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp taosProcPutToParentQ(pWrapper->procObj, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_FUNC_RSP); } } +#endif static inline void dmRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != DND_PROC_CHILD) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 407158283b..f3d7253e71 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -155,6 +155,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rsp.code = TSDB_CODE_RPC_REDIRECT; SEpSet newEpSet; syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); + newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; tmsgSendRedirectRsp(&rsp, &newEpSet); } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 857b572cab..9a8dcc57d9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -174,11 +174,16 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { assert(rid == pSyncNode->rid); pEpSet->numOfEps = 0; for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - snprintf(pEpSet->eps->fqdn, sizeof(pEpSet->eps->fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); - pEpSet->eps->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); + pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; (pEpSet->numOfEps)++; + + sInfo("syncGetEpSet index:%d %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; + + sInfo("syncGetEpSet pEpSet->inUse:%d ", pEpSet->inUse); taosReleaseRef(tsNodeRefId, pSyncNode->rid); } From 5126d06c360eafbfdc41159858502a76c2e141ba Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 30 Apr 2022 10:56:06 +0800 Subject: [PATCH 14/90] modify alter_option.sim replica 1 -> 3 --- tests/script/tsim/db/alter_option.sim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 417e53daff..cc69386fb1 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,8 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +#sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -86,7 +87,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 1 then # replica +if $data4_db != 3 then # replica return -1 endi if $data5_db != nostrict then # strict From 913354057ecf847977c25bf01216290370fb9f49 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 30 Apr 2022 11:40:12 +0800 Subject: [PATCH 15/90] feat: fill physical plan --- include/libs/nodes/nodes.h | 2 + include/libs/nodes/plannodes.h | 23 +- include/libs/nodes/querynodes.h | 17 +- source/libs/command/src/explain.c | 346 ++++++++-------- source/libs/executor/src/executorimpl.c | 28 +- source/libs/nodes/src/nodesCloneFuncs.c | 12 +- source/libs/nodes/src/nodesCodeFuncs.c | 105 ++++- source/libs/nodes/src/nodesTraverseFuncs.c | 31 +- source/libs/nodes/src/nodesUtilFuncs.c | 25 +- source/libs/parser/src/parAstCreater.c | 6 + source/libs/parser/src/parTokenizer.c | 368 +++++++++--------- source/libs/parser/test/parSelectTest.cpp | 125 +++--- source/libs/planner/src/planLogicCreater.c | 48 ++- source/libs/planner/src/planPhysiCreater.c | 47 ++- source/libs/planner/test/planIntervalTest.cpp | 8 +- 15 files changed, 706 insertions(+), 485 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 6513a44e88..115bb4349d 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -185,6 +185,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_WINDOW, + QUERY_NODE_LOGIC_PLAN_FILL, QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_PLAN_PARTITION, QUERY_NODE_LOGIC_SUBPLAN, @@ -202,6 +203,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_PARTITION, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index a34d4e3c80..10688ce239 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -102,7 +102,6 @@ typedef struct SWindowLogicNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; int64_t sessionGap; SNode* pTspk; SNode* pStateExpr; @@ -110,6 +109,13 @@ typedef struct SWindowLogicNode { int64_t watermark; } SWindowLogicNode; +typedef struct SFillLogicNode { + SLogicNode node; + EFillMode mode; + SNode* pWStartTs; + SNode* pValues; // SNodeListNode +} SFillLogicNode; + typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; @@ -223,10 +229,12 @@ typedef struct SProjectPhysiNode { typedef struct SJoinPhysiNode { SPhysiNode node; EJoinType joinType; - SNode* pOnConditions; // in or out tuple ? + SNode* pOnConditions; SNodeList* pTargets; } SJoinPhysiNode; +typedef SJoinPhysiNode SSortMergeJoinPhysiNode; + typedef struct SAggPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function @@ -263,9 +271,16 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; - SFillNode* pFill; } SIntervalPhysiNode; +typedef struct SFillPhysiNode { + SPhysiNode node; + EFillMode mode; + SNode* pWStartTs; // SColumnNode + SNode* pValues; // SNodeListNode + SNodeList* pTargets; +} SFillPhysiNode; + typedef struct SMultiTableIntervalPhysiNode { SIntervalPhysiNode interval; SNodeList* pPartitionKeys; @@ -340,7 +355,7 @@ typedef struct SQueryPlan { int32_t numOfSubplans; SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SExplainInfo explainInfo; - SArray* pPlaceholderValues; + SArray* pPlaceholderValues; } SQueryPlan; void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5f1ee157e1..29f31f25a0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -89,7 +89,7 @@ typedef struct SValueNode { char* p; } datum; int64_t typeData; - char unit; + char unit; } SValueNode; typedef struct SOperatorNode { @@ -211,7 +211,8 @@ typedef enum EFillMode { typedef struct SFillNode { ENodeType type; // QUERY_NODE_FILL EFillMode mode; - SNode* pValues; // SNodeListNode + SNode* pValues; // SNodeListNode + SNode* pWStartTs; // _wstartts pseudo column } SFillNode; typedef struct SSelectStmt { @@ -300,7 +301,7 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* SNodeList** pCols); typedef bool (*FFuncClassifier)(int32_t funcId); -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs); int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeType type, SNodeList** pNodes); @@ -314,11 +315,11 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -void* nodesGetValueFromNode(SValueNode* pNode); -int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value); -char* nodesGetStrValueFromNode(SValueNode* pNode); -char* getFillModeString(EFillMode mode); -void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); +void* nodesGetValueFromNode(SValueNode* pNode); +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value); +char* nodesGetStrValueFromNode(SValueNode* pNode); +char* getFillModeString(EFillMode mode); +void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); #ifdef __cplusplus } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2aae4303eb..2e94ec8d0c 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -13,14 +13,13 @@ * along with this program. If not, see . */ -#include "query.h" -#include "plannodes.h" #include "commandInt.h" +#include "plannodes.h" +#include "query.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); - void qExplainFreeResNode(SExplainResNode *resNode) { if (NULL == resNode) { return; @@ -28,12 +27,10 @@ void qExplainFreeResNode(SExplainResNode *resNode) { taosMemoryFreeClear(resNode->pExecInfo); - SNode* node = NULL; - FOREACH(node, resNode->pChildren) { - qExplainFreeResNode((SExplainResNode *)node); - } + SNode *node = NULL; + FOREACH(node, resNode->pChildren) { qExplainFreeResNode((SExplainResNode *)node); } nodesClearList(resNode->pChildren); - + taosMemoryFreeClear(resNode); } @@ -59,24 +56,24 @@ void qExplainFreeCtx(SExplainCtx *pCtx) { taosMemoryFreeClear(rsp->subplanInfo); } } - + pIter = taosHashIterate(pCtx->groupHash, pIter); } } - + taosHashCleanup(pCtx->groupHash); taosArrayDestroy(pCtx->rows); taosMemoryFree(pCtx); } int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, double ratio, EExplainMode mode) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); @@ -95,7 +92,7 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d ctx->tbuf = tbuf; ctx->rows = rows; ctx->groupHash = groupHash; - + *pCtx = ctx; return TSDB_CODE_SUCCESS; @@ -110,9 +107,9 @@ _return: } int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { - int32_t tlen = 0; + int32_t tlen = 0; SNodeList *pPhysiChildren = NULL; - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -120,47 +117,47 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; pPhysiChildren = pTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; pPhysiChildren = pSTblScanNode->scan.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; pPhysiChildren = pPrjNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; pPhysiChildren = pJoinNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; pPhysiChildren = pAggNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; pPhysiChildren = pExchNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; pPhysiChildren = pSortNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; pPhysiChildren = pIntNode->window.node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; pPhysiChildren = pSessNode->window.node.pChildren; break; @@ -178,7 +175,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } } - SNode* node = NULL; + SNode *node = NULL; SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); @@ -195,14 +192,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SExplainRsp *rsp = NULL; + SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); return TSDB_CODE_QRY_APP_ERROR; } - + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); } @@ -217,7 +214,7 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai qError("physical node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); @@ -226,15 +223,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai int32_t code = 0; resNode->pNode = pNode; - + if (group->nodeExecInfo) { QRY_ERR_JRET(qExplainGenerateResNodeExecInfo(&resNode->pExecInfo, group)); } - + QRY_ERR_JRET(qExplainGenerateResChildren(pNode, group, &resNode->pChildren)); ++group->physiPlanNum; - + *pResNode = resNode; return TSDB_CODE_SUCCESS; @@ -242,15 +239,15 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai _return: qExplainFreeResNode(resNode); - + QRY_RET(code); } int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = *len; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = *len; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->startupCost > maxExecInfo.startupCost) { @@ -263,20 +260,20 @@ int32_t qExplainBufAppendExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { maxExecInfo.numOfRows = execInfo->numOfRows; } } - + EXPLAIN_ROW_APPEND(EXPLAIN_EXECINFO_FORMAT, maxExecInfo.startupCost, maxExecInfo.totalCost, maxExecInfo.numOfRows); *len = tlen; - + return TSDB_CODE_SUCCESS; } int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t *len) { - int32_t tlen = 0; - bool gotVerbose = false; - int32_t nodeNum = taosArrayGetSize(pExecInfo); + int32_t tlen = 0; + bool gotVerbose = false; + int32_t nodeNum = taosArrayGetSize(pExecInfo); SExplainExecInfo maxExecInfo = {0}; - + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pExecInfo, i); if (execInfo->verboseInfo) { @@ -289,11 +286,10 @@ int32_t qExplainBufAppendVerboseExecInfo(SArray *pExecInfo, char *tbuf, int32_t } *len = tlen; - + return TSDB_CODE_SUCCESS; } - int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t level) { SQueryExplainRowInfo row = {0}; row.buf = taosMemoryMalloc(len); @@ -304,7 +300,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; + row.len = len; ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { @@ -316,21 +312,21 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t return TSDB_CODE_SUCCESS; } -static uint8_t getIntervalPrecision(SIntervalPhysiNode* pIntNode) { - return ((SColumnNode*)pIntNode->window.pTspk)->node.resType.precision; +static uint8_t getIntervalPrecision(SIntervalPhysiNode *pIntNode) { + return ((SColumnNode *)pIntNode->window.pTspk)->node.resType.precision; } int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { - int32_t tlen = 0; - bool isVerboseLine = false; - char *tbuf = ctx->tbuf; - bool verbose = ctx->verbose; - SPhysiNode* pNode = pResNode->pNode; + int32_t tlen = 0; + bool isVerboseLine = false; + char *tbuf = ctx->tbuf; + bool verbose = ctx->verbose; + SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); return TSDB_CODE_QRY_APP_ERROR; } - + switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; @@ -339,7 +335,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTagScanNode->pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); @@ -350,80 +346,85 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTagScanNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); if (tlen) { EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } + } } break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_TABLE_SCAN_FORMAT, pTblScanNode->scanSeq[0], pTblScanNode->scanSeq[1]); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, + pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN:{ + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SYSTBL_SCAN_FORMAT, pSTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSTblScanNode->scan.pScanCols->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { + if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSTblScanNode->scan.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -431,91 +432,96 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (pSTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - } break; } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT:{ + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_PROJECTION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pPrjNode->pProjections->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pPrjNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pPrjNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - + if (pPrjNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pPrjNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_JOIN:{ + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pJoinNode->pTargets->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pJoinNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pJoinNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pJoinNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT); + QRY_ERR_RET( + nodesNodeToSQL(pJoinNode->pOnConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } break; } - case QUERY_NODE_PHYSICAL_PLAN_AGG:{ + case QUERY_NODE_PHYSICAL_PLAN_AGG: { SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_AGG_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pAggNode->pAggFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->totalRowSize); @@ -523,57 +529,61 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_GROUPS_FORMAT, pAggNode->pGroupKeys->length); } - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pAggNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pAggNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pAggNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pAggNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE:{ + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); + SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcGroupId, sizeof(pExchNode->srcGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcGroupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + EXPLAIN_ROW_NEW(level, EXPLAIN_EXCHANGE_FORMAT, group->nodeNum); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->totalRowSize); - EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pExchNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pExchNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pExchNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pExchNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -582,14 +592,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainAppendGroupResRows(ctx, pExchNode->srcGroupId, level + 1)); break; } - case QUERY_NODE_PHYSICAL_PLAN_SORT:{ + case QUERY_NODE_PHYSICAL_PLAN_SORT: { SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SORT_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, pSortNode->pSortKeys->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->totalRowSize); @@ -599,29 +609,31 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSortNode->node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSortNode->node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSortNode->node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSortNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{ + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pIntNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -630,41 +642,39 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pIntNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); uint8_t precision = getIntervalPrecision(pIntNode); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), - pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), - INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), pIntNode->slidingUnit); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), + pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), + INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), + pIntNode->slidingUnit); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - - if (pIntNode->pFill) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_FORMAT, getFillModeString(pIntNode->pFill->mode)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - } if (pIntNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pIntNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } } break; } - case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW:{ + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: { SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_SESSION_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - } + } EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pSessNode->window.pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->totalRowSize); @@ -672,10 +682,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - if (verbose) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pSessNode->window.node.pOutputDataBlockDesc->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSessNode->window.node.pOutputDataBlockDesc->outputRowSize); EXPLAIN_ROW_END(); @@ -686,8 +696,9 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pSessNode->window.node.pConditions) { - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); - QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pSessNode->window.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } @@ -702,7 +713,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i return TSDB_CODE_SUCCESS; } - int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); @@ -712,29 +722,28 @@ int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32 int32_t code = 0; QRY_ERR_RET(qExplainResNodeToRowsImpl(pResNode, ctx, level)); - SNode* pNode = NULL; - FOREACH(pNode, pResNode->pChildren) { - QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); - } + SNode *pNode = NULL; + FOREACH(pNode, pResNode->pChildren) { QRY_ERR_RET(qExplainResNodeToRows((SExplainResNode *)pNode, ctx, level + 1)); } return TSDB_CODE_SUCCESS; } int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { SExplainResNode *node = NULL; - int32_t code = 0; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + QRY_ERR_RET(qExplainGenerateResNode(group->plan->pNode, group, &node)); if ((EXPLAIN_MODE_ANALYZE == ctx->mode) && (group->physiPlanNum != group->physiPlanExecNum)) { - qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, groupId); + qError("physiPlanNum %d mismatch with physiExecNum %d in group %d", group->physiPlanNum, group->physiPlanExecNum, + groupId); QRY_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -743,21 +752,21 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level) { _return: qExplainFreeResNode(node); - + QRY_RET(code); } - int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = (SExplainCtx *)ctx; - int32_t rowNum = taosArrayGetSize(pCtx->rows); + int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - + int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -768,13 +777,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); // payload length - *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + *(int32_t *)rsp->data = + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; // group id - *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0; // column length - int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); // varchar column offset segment int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); @@ -782,7 +792,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { // varchar data real payload char *data = (char *)(offset + rowNum); - char* start = data; + char *start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); offset[i] = data - start; @@ -800,11 +810,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { - int32_t code = 0; + int32_t code = 0; SNodeListNode *plans = NULL; int32_t taskNum = 0; SExplainGroup *pGroup = NULL; - SExplainCtx *ctx = NULL; + SExplainCtx *ctx = NULL; if (pDag->numOfSubplans <= 0) { qError("invalid subplan num:%d", pDag->numOfSubplans); @@ -817,13 +827,15 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { QRY_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SHashObj *groupHash = taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + SHashObj *groupHash = + taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - QRY_ERR_JRET(qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); + QRY_ERR_JRET( + qExplainInitCtx(&ctx, groupHash, pDag->explainInfo.verbose, pDag->explainInfo.ratio, pDag->explainInfo.mode)); for (int32_t i = 0; i < levelNum; ++i) { plans = (SNodeListNode *)nodesListGetNode(pDag->pSubplans, i); @@ -850,7 +862,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { SExplainGroup group = {0}; group.nodeNum = 1; group.plan = plan; - + if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -872,7 +884,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { *pCtx = ctx; return TSDB_CODE_SUCCESS; - + _return: qExplainFreeCtx(ctx); @@ -886,7 +898,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { } int32_t tlen = 0; - char *tbuf = pCtx->tbuf; + char *tbuf = pCtx->tbuf; EXPLAIN_SUM_ROW_NEW(EXPLAIN_RATIO_TIME_FORMAT, pCtx->ratio); EXPLAIN_SUM_ROW_END(); @@ -911,11 +923,11 @@ int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } -int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { +int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp) { SExplainResNode *node = NULL; - int32_t code = 0; - bool groupDone = false; - SExplainCtx *ctx = (SExplainCtx *)pCtx; + int32_t code = 0; + bool groupDone = false; + SExplainCtx *ctx = (SExplainCtx *)pCtx; SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { @@ -931,30 +943,32 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { - qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), group->nodeNum); + qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), + group->nodeNum); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { - qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, groupId); + qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, + groupId); taosMemoryFreeClear(pRspMsg->subplanInfo); taosWUnLockLatch(&group->lock); - + QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); - + taosWUnLockLatch(&group->lock); if (groupDone && (taosHashGetSize(pCtx->groupHash) == atomic_add_fetch_32(&pCtx->groupDoneNum, 1))) { @@ -969,14 +983,13 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, i return TSDB_CODE_SUCCESS; } - int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { - int32_t code = 0; + int32_t code = 0; SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); - + _return: qExplainFreeCtx(pCtx); QRY_RET(code); @@ -984,7 +997,7 @@ _return: int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) { QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx)); - + (*pCtx)->reqStartTs = startTs; (*pCtx)->jobStartTs = taosGetTimestampUs(); @@ -994,7 +1007,7 @@ int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { int32_t code = 0; pCtx->jobDoneTs = taosGetTimestampUs(); - + atomic_store_8((int8_t *)&pCtx->execDone, true); if (taosHashGetSize(pCtx->groupHash) == atomic_load_32(&pCtx->groupDoneNum)) { @@ -1006,6 +1019,3 @@ int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { return TSDB_CODE_SUCCESS; } - - - diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d8e69c4b70..9f17a4cc7b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1179,7 +1179,7 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S } int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList) { + int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; @@ -1258,7 +1258,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc SColumnInfoData idata = {.info = pResColData->info, .hasNull = true}; SScalarParam dest = {.columnData = &idata}; - int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); + int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pBlockList); return code; @@ -3959,9 +3959,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, - pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); + code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { goto _error; } @@ -4781,8 +4780,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->pScalarExprInfo != NULL) { - int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, pAggInfo->numOfScalarExpr, - NULL); + int32_t code = projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, + pAggInfo->numOfScalarExpr, NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; longjmp(pTaskInfo->env, pTaskInfo->code); @@ -6540,8 +6539,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, pScanPhyNode->node.pConditions); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo, + pScanPhyNode->node.pConditions); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -6622,10 +6621,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, pTableGroupInfo, pTaskInfo); - if (pIntervalPhyNode->pFill != NULL) { - pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, - false, pTaskInfo); - } + // if (pIntervalPhyNode->pFill != NULL) { + // pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, + // NULL, + // false, pTaskInfo); + // } } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; @@ -6927,7 +6927,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { goto _error; - } + } #if 0 return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); #endif diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 92da476319..5e9e4e1d57 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -191,6 +191,7 @@ static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); + CLONE_NODE_FIELD(pWStartTs); return (SNode*)pDst; } @@ -269,11 +270,18 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pFuncs); - CLONE_NODE_FIELD(pFill); CLONE_NODE_FIELD(pTspk); return (SNode*)pDst; } +static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_FIELD(pWStartTs); + CLONE_NODE_FIELD(pValues); + return (SNode*)pDst; +} + static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); @@ -370,6 +378,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillCopy((const SFillLogicNode*)pNode, (SFillLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PARTITION: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6dcd2a0baf..e1ea23226b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -192,6 +192,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicExchange"; case QUERY_NODE_LOGIC_PLAN_WINDOW: return "LogicWindow"; + case QUERY_NODE_LOGIC_PLAN_FILL: + return "LogicFill"; case QUERY_NODE_LOGIC_PLAN_SORT: return "LogicSort"; case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -222,6 +224,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -564,6 +568,44 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkFillLogicPlanMode = "Mode"; +static const char* jkFillLogicPlanWStartTs = "WStartTs"; +static const char* jkFillLogicPlanValues = "Values"; + +static int32_t logicFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillLogicNode* pNode = (const SFillLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillLogicPlanValues, nodeToJson, pNode->pValues); + } + + return code; +} + +static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { + SFillLogicNode* pNode = (SFillLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillLogicPlanValues, &pNode->pValues); + } + + return code; +} + static const char* jkSortLogicPlanSortKeys = "SortKeys"; static int32_t logicSortNodeToJson(const void* pObj, SJson* pJson) { @@ -1382,7 +1424,6 @@ static const char* jkIntervalPhysiPlanOffset = "Offset"; static const char* jkIntervalPhysiPlanSliding = "Sliding"; static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; -static const char* jkIntervalPhysiPlanFill = "Fill"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -1403,9 +1444,6 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanSlidingUnit, pNode->slidingUnit); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); - } return code; } @@ -1429,8 +1467,50 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkIntervalPhysiPlanSlidingUnit, &pNode->slidingUnit); } + + return code; +} + +static const char* jkFillPhysiPlanMode = "Mode"; +static const char* jkFillPhysiPlanWStartTs = "WStartTs"; +static const char* jkFillPhysiPlanValues = "Values"; +static const char* jkFillPhysiPlanTargets = "Targets"; + +static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { + const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillPhysiPlanWStartTs, nodeToJson, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillPhysiPlanValues, nodeToJson, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkFillPhysiPlanTargets, pNode->pTargets); + } + + return code; +} + +static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { + SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanWStartTs, &pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillPhysiPlanValues, &pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkFillPhysiPlanTargets, &pNode->pTargets); } return code; @@ -2328,6 +2408,7 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) { static const char* jkFillMode = "Mode"; static const char* jkFillValues = "Values"; +static const char* jkFillWStartTs = "WStartTs"; static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { const SFillNode* pNode = (const SFillNode*)pObj; @@ -2336,6 +2417,9 @@ static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkFillValues, nodeToJson, pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkFillWStartTs, nodeToJson, pNode->pWStartTs); + } return code; } @@ -2347,6 +2431,9 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkFillWStartTs, &pNode->pWStartTs); + } return code; } @@ -2707,6 +2794,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicProjectNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: break; + case QUERY_NODE_LOGIC_PLAN_FILL: + return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: return logicSortNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -2735,6 +2824,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return physiFillNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -2795,6 +2886,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_FILL: + return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: return jsonToLogicSortNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -2821,6 +2914,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return jsonToPhysiFillNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 88f5d914e4..f2c043b9ea 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -132,9 +132,14 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa case QUERY_NODE_NODE_LIST: res = walkExprs(((SNodeListNode*)pNode)->pNodeList, order, walker, pContext); break; - case QUERY_NODE_FILL: - res = walkExpr(((SFillNode*)pNode)->pValues, order, walker, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = walkExpr(pFill->pValues, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pFill->pWStartTs, order, walker, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = walkExpr(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; @@ -272,9 +277,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_NODE_LIST: res = rewriteExprs(((SNodeListNode*)pNode)->pNodeList, order, rewriter, pContext); break; - case QUERY_NODE_FILL: - res = rewriteExpr(&(((SFillNode*)pNode)->pValues), order, rewriter, pContext); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + res = rewriteExpr(&pFill->pValues, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&(pFill->pWStartTs), order, rewriter, pContext); + } break; + } case QUERY_NODE_RAW_EXPR: res = rewriteExpr(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; @@ -333,6 +343,9 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_PARTITION_BY: nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesWalkExpr(((SIntervalWindowNode*)pSelect->pWindow)->pFill, walker, pContext); + } nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: nodesWalkExpr(pSelect->pHaving, walker, pContext); @@ -362,6 +375,9 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit case SQL_CLAUSE_PARTITION_BY: nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: + if (NULL != pSelect->pWindow && QUERY_NODE_INTERVAL_WINDOW == nodeType(pSelect->pWindow)) { + nodesRewriteExpr(&(((SIntervalWindowNode*)pSelect->pWindow)->pFill), rewriter, pContext); + } nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); @@ -496,14 +512,9 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)pNode; + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); - if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { - res = walkPhysiPlan((SNode*)pInterval->pFill, order, walker, pContext); - } break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d7dba10718..0bceb84f4a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -213,6 +213,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); + case QUERY_NODE_LOGIC_PLAN_FILL: + return makeNode(type, sizeof(SFillLogicNode)); case QUERY_NODE_LOGIC_PLAN_SORT: return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_PLAN_PARTITION: @@ -243,6 +245,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_FILL: + return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: @@ -373,9 +377,12 @@ void nodesDestroyNode(SNodeptr pNode) { case QUERY_NODE_NODE_LIST: nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); break; - case QUERY_NODE_FILL: - nodesDestroyNode(((SFillNode*)pNode)->pValues); + case QUERY_NODE_FILL: { + SFillNode* pFill = (SFillNode*)pNode; + nodesDestroyNode(pFill->pValues); + nodesDestroyNode(pFill->pWStartTs); break; + } case QUERY_NODE_RAW_EXPR: nodesDestroyNode(((SRawExprNode*)pNode)->pNode); break; @@ -554,7 +561,6 @@ void nodesDestroyNode(SNodeptr pNode) { SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); - nodesDestroyNode(pLogicNode->pFill); nodesDestroyNode(pLogicNode->pTspk); break; } @@ -630,12 +636,9 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pPhyNode->pSortKeys); break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { - SIntervalPhysiNode* pPhyNode = (SIntervalPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); - nodesDestroyNode(pPhyNode->pFill); + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; - } case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; @@ -894,7 +897,7 @@ void* nodesGetValueFromNode(SValueNode* pNode) { return NULL; } -int32_t nodesSetValueNodeValue(SValueNode* pNode, void *value) { +int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: pNode->datum.b = *(bool*)value; @@ -1192,7 +1195,7 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs) { +int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { return TSDB_CODE_FAILED; } @@ -1203,7 +1206,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod return TSDB_CODE_OUT_OF_MEMORY; } *pFuncs = NULL; - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_GROUP_BY, collectFuncs, &cxt); + nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pFuncs); return cxt.errCode; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 1701976f8d..c021f65090 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -501,6 +501,12 @@ SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { CHECK_OUT_OF_MEM(fill); fill->mode = mode; fill->pValues = pValues; + fill->pWStartTs = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == fill->pWStartTs) { + nodesDestroyNode(fill); + CHECK_OUT_OF_MEM(fill->pWStartTs); + } + strcpy(((SFunctionNode*)fill->pWStartTs)->functionName, "_wstartts"); return (SNode*)fill; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index ed123d50bd..44d68a7886 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -26,193 +26,192 @@ typedef struct SKeyword { uint8_t len; // length } SKeyword; +// clang-format off // keywords in sql string static SKeyword keywordTable[] = { - {"ACCOUNT", TK_ACCOUNT}, - {"ACCOUNTS", TK_ACCOUNTS}, - {"ADD", TK_ADD}, - {"AGGREGATE", TK_AGGREGATE}, - {"ALL", TK_ALL}, - {"ALTER", TK_ALTER}, - {"ANALYZE", TK_ANALYZE}, - {"AND", TK_AND}, - {"APPS", TK_APPS}, - {"AS", TK_AS}, - {"ASC", TK_ASC}, - {"AT_ONCE", TK_AT_ONCE}, - {"BETWEEN", TK_BETWEEN}, - {"BINARY", TK_BINARY}, - {"BIGINT", TK_BIGINT}, - // {"BLOCKS", TK_BLOCKS}, - {"BNODE", TK_BNODE}, - {"BNODES", TK_BNODES}, - {"BOOL", TK_BOOL}, - {"BUFFER", TK_BUFFER}, - {"BUFSIZE", TK_BUFSIZE}, - {"BY", TK_BY}, - {"CACHE", TK_CACHE}, - {"CACHELAST", TK_CACHELAST}, - {"CAST", TK_CAST}, - {"CLUSTER", TK_CLUSTER}, - {"COLUMN", TK_COLUMN}, - {"COMMENT", TK_COMMENT}, - {"COMP", TK_COMP}, - {"COMPACT", TK_COMPACT}, - {"CONNS", TK_CONNS}, - {"CONNECTION", TK_CONNECTION}, - {"CONNECTIONS", TK_CONNECTIONS}, - {"COUNT", TK_COUNT}, - {"CREATE", TK_CREATE}, - {"DATABASE", TK_DATABASE}, - {"DATABASES", TK_DATABASES}, - {"DAYS", TK_DAYS}, - {"DBS", TK_DBS}, - {"DELAY", TK_DELAY}, - {"DESC", TK_DESC}, - {"DESCRIBE", TK_DESCRIBE}, - {"DISTINCT", TK_DISTINCT}, - {"DNODE", TK_DNODE}, - {"DNODES", TK_DNODES}, - {"DOUBLE", TK_DOUBLE}, - {"DROP", TK_DROP}, - {"EXISTS", TK_EXISTS}, - {"EXPLAIN", TK_EXPLAIN}, - {"FILE_FACTOR", TK_FILE_FACTOR}, - {"FILL", TK_FILL}, - {"FIRST", TK_FIRST}, - {"FLOAT", TK_FLOAT}, - {"FROM", TK_FROM}, - {"FSYNC", TK_FSYNC}, - {"FUNCTION", TK_FUNCTION}, - {"FUNCTIONS", TK_FUNCTIONS}, - {"GRANTS", TK_GRANTS}, - {"GROUP", TK_GROUP}, - {"HAVING", TK_HAVING}, - {"IF", TK_IF}, - {"IMPORT", TK_IMPORT}, - {"IN", TK_IN}, - {"INDEX", TK_INDEX}, - {"INDEXES", TK_INDEXES}, - {"INNER", TK_INNER}, - {"INT", TK_INT}, - {"INSERT", TK_INSERT}, - {"INTEGER", TK_INTEGER}, - {"INTERVAL", TK_INTERVAL}, - {"INTO", TK_INTO}, - {"IS", TK_IS}, - {"JOIN", TK_JOIN}, - {"JSON", TK_JSON}, - {"KEEP", TK_KEEP}, - {"KILL", TK_KILL}, - {"LAST", TK_LAST}, - {"LAST_ROW", TK_LAST_ROW}, - {"LICENCE", TK_LICENCE}, - {"LIKE", TK_LIKE}, - {"LIMIT", TK_LIMIT}, - {"LINEAR", TK_LINEAR}, - {"LOCAL", TK_LOCAL}, - {"MATCH", TK_MATCH}, - {"MAXROWS", TK_MAXROWS}, - {"MINROWS", TK_MINROWS}, - {"MINUS", TK_MINUS}, - {"MNODE", TK_MNODE}, - {"MNODES", TK_MNODES}, - {"MODIFY", TK_MODIFY}, - {"MODULES", TK_MODULES}, - {"NCHAR", TK_NCHAR}, - {"NMATCH", TK_NMATCH}, - {"NONE", TK_NONE}, - {"NOT", TK_NOT}, - {"NOW", TK_NOW}, - {"NULL", TK_NULL}, - {"NULLS", TK_NULLS}, - {"OFFSET", TK_OFFSET}, - {"ON", TK_ON}, - {"OR", TK_OR}, - {"ORDER", TK_ORDER}, - {"OUTPUTTYPE", TK_OUTPUTTYPE}, - {"PARTITION", TK_PARTITION}, - {"PASS", TK_PASS}, - {"PAGES", TK_PAGES}, - {"PAGESIZE", TK_PAGESIZE}, - {"PORT", TK_PORT}, - {"PPS", TK_PPS}, - {"PRECISION", TK_PRECISION}, - {"PRIVILEGE", TK_PRIVILEGE}, - {"PREV", TK_PREV}, - {"QNODE", TK_QNODE}, - {"QNODES", TK_QNODES}, - {"QTIME", TK_QTIME}, - {"QUERIES", TK_QUERIES}, - {"QUERY", TK_QUERY}, - // {"QUORUM", TK_QUORUM}, - {"RATIO", TK_RATIO}, - {"REPLICA", TK_REPLICA}, - {"RESET", TK_RESET}, - {"RETENTIONS", TK_RETENTIONS}, - {"ROLLUP", TK_ROLLUP}, - {"SCHEMA", TK_SCHEMA}, - {"SCORES", TK_SCORES}, - {"SELECT", TK_SELECT}, - {"SESSION", TK_SESSION}, - {"SET", TK_SET}, - {"SHOW", TK_SHOW}, + {"ACCOUNT", TK_ACCOUNT}, + {"ACCOUNTS", TK_ACCOUNTS}, + {"ADD", TK_ADD}, + {"AGGREGATE", TK_AGGREGATE}, + {"ALL", TK_ALL}, + {"ALTER", TK_ALTER}, + {"ANALYZE", TK_ANALYZE}, + {"AND", TK_AND}, + {"APPS", TK_APPS}, + {"AS", TK_AS}, + {"ASC", TK_ASC}, + {"AT_ONCE", TK_AT_ONCE}, + {"BETWEEN", TK_BETWEEN}, + {"BINARY", TK_BINARY}, + {"BIGINT", TK_BIGINT}, + {"BNODE", TK_BNODE}, + {"BNODES", TK_BNODES}, + {"BOOL", TK_BOOL}, + {"BUFFER", TK_BUFFER}, + {"BUFSIZE", TK_BUFSIZE}, + {"BY", TK_BY}, + {"CACHE", TK_CACHE}, + {"CACHELAST", TK_CACHELAST}, + {"CAST", TK_CAST}, + {"CLUSTER", TK_CLUSTER}, + {"COLUMN", TK_COLUMN}, + {"COMMENT", TK_COMMENT}, + {"COMP", TK_COMP}, + {"COMPACT", TK_COMPACT}, + {"CONNS", TK_CONNS}, + {"CONNECTION", TK_CONNECTION}, + {"CONNECTIONS", TK_CONNECTIONS}, + {"COUNT", TK_COUNT}, + {"CREATE", TK_CREATE}, + {"DATABASE", TK_DATABASE}, + {"DATABASES", TK_DATABASES}, + {"DAYS", TK_DAYS}, + {"DBS", TK_DBS}, + {"DELAY", TK_DELAY}, + {"DESC", TK_DESC}, + {"DESCRIBE", TK_DESCRIBE}, + {"DISTINCT", TK_DISTINCT}, + {"DNODE", TK_DNODE}, + {"DNODES", TK_DNODES}, + {"DOUBLE", TK_DOUBLE}, + {"DROP", TK_DROP}, + {"EXISTS", TK_EXISTS}, + {"EXPLAIN", TK_EXPLAIN}, + {"FILE_FACTOR", TK_FILE_FACTOR}, + {"FILL", TK_FILL}, + {"FIRST", TK_FIRST}, + {"FLOAT", TK_FLOAT}, + {"FROM", TK_FROM}, + {"FSYNC", TK_FSYNC}, + {"FUNCTION", TK_FUNCTION}, + {"FUNCTIONS", TK_FUNCTIONS}, + {"GRANTS", TK_GRANTS}, + {"GROUP", TK_GROUP}, + {"HAVING", TK_HAVING}, + {"IF", TK_IF}, + {"IMPORT", TK_IMPORT}, + {"IN", TK_IN}, + {"INDEX", TK_INDEX}, + {"INDEXES", TK_INDEXES}, + {"INNER", TK_INNER}, + {"INT", TK_INT}, + {"INSERT", TK_INSERT}, + {"INTEGER", TK_INTEGER}, + {"INTERVAL", TK_INTERVAL}, + {"INTO", TK_INTO}, + {"IS", TK_IS}, + {"JOIN", TK_JOIN}, + {"JSON", TK_JSON}, + {"KEEP", TK_KEEP}, + {"KILL", TK_KILL}, + {"LAST", TK_LAST}, + {"LAST_ROW", TK_LAST_ROW}, + {"LICENCE", TK_LICENCE}, + {"LIKE", TK_LIKE}, + {"LIMIT", TK_LIMIT}, + {"LINEAR", TK_LINEAR}, + {"LOCAL", TK_LOCAL}, + {"MATCH", TK_MATCH}, + {"MAXROWS", TK_MAXROWS}, + {"MINROWS", TK_MINROWS}, + {"MINUS", TK_MINUS}, + {"MNODE", TK_MNODE}, + {"MNODES", TK_MNODES}, + {"MODIFY", TK_MODIFY}, + {"MODULES", TK_MODULES}, + {"NCHAR", TK_NCHAR}, + {"NEXT", TK_NEXT}, + {"NMATCH", TK_NMATCH}, + {"NONE", TK_NONE}, + {"NOT", TK_NOT}, + {"NOW", TK_NOW}, + {"NULL", TK_NULL}, + {"NULLS", TK_NULLS}, + {"OFFSET", TK_OFFSET}, + {"ON", TK_ON}, + {"OR", TK_OR}, + {"ORDER", TK_ORDER}, + {"OUTPUTTYPE", TK_OUTPUTTYPE}, + {"PARTITION", TK_PARTITION}, + {"PASS", TK_PASS}, + {"PAGES", TK_PAGES}, + {"PAGESIZE", TK_PAGESIZE}, + {"PORT", TK_PORT}, + {"PPS", TK_PPS}, + {"PRECISION", TK_PRECISION}, + {"PRIVILEGE", TK_PRIVILEGE}, + {"PREV", TK_PREV}, + {"QNODE", TK_QNODE}, + {"QNODES", TK_QNODES}, + {"QTIME", TK_QTIME}, + {"QUERIES", TK_QUERIES}, + {"QUERY", TK_QUERY}, + {"RATIO", TK_RATIO}, + {"REPLICA", TK_REPLICA}, + {"RESET", TK_RESET}, + {"RETENTIONS", TK_RETENTIONS}, + {"ROLLUP", TK_ROLLUP}, + {"SCHEMA", TK_SCHEMA}, + {"SCORES", TK_SCORES}, + {"SELECT", TK_SELECT}, + {"SESSION", TK_SESSION}, + {"SET", TK_SET}, + {"SHOW", TK_SHOW}, {"SINGLE_STABLE", TK_SINGLE_STABLE}, - {"SLIDING", TK_SLIDING}, - {"SLIMIT", TK_SLIMIT}, - {"SMA", TK_SMA}, - {"SMALLINT", TK_SMALLINT}, - {"SNODE", TK_SNODE}, - {"SNODES", TK_SNODES}, - {"SOFFSET", TK_SOFFSET}, - {"STABLE", TK_STABLE}, - {"STABLES", TK_STABLES}, - {"STATE", TK_STATE}, - {"STATE_WINDOW", TK_STATE_WINDOW}, - {"STORAGE", TK_STORAGE}, - {"STREAM", TK_STREAM}, - {"STREAMS", TK_STREAMS}, - // {"STREAM_MODE", TK_STREAM_MODE}, - {"STRICT", TK_STRICT}, - {"SYNCDB", TK_SYNCDB}, - {"TABLE", TK_TABLE}, - {"TABLES", TK_TABLES}, - {"TAG", TK_TAG}, - {"TAGS", TK_TAGS}, - {"TBNAME", TK_TBNAME}, - {"TIMESTAMP", TK_TIMESTAMP}, - {"TIMEZONE", TK_TIMEZONE}, - {"TINYINT", TK_TINYINT}, - {"TODAY", TK_TODAY}, - {"TOPIC", TK_TOPIC}, - {"TOPICS", TK_TOPICS}, - {"TRIGGER", TK_TRIGGER}, - {"TSERIES", TK_TSERIES}, - {"TTL", TK_TTL}, - {"UNION", TK_UNION}, - {"UNSIGNED", TK_UNSIGNED}, - {"USE", TK_USE}, - {"USER", TK_USER}, - {"USERS", TK_USERS}, - {"USING", TK_USING}, - {"VALUE", TK_VALUE}, - {"VALUES", TK_VALUES}, - {"VARCHAR", TK_VARCHAR}, - {"VARIABLES", TK_VARIABLES}, - {"VERBOSE", TK_VERBOSE}, - {"VGROUPS", TK_VGROUPS}, - {"VNODES", TK_VNODES}, - {"WAL", TK_WAL}, - {"WATERMARK", TK_WATERMARK}, - {"WHERE", TK_WHERE}, - {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, - {"WITH", TK_WITH}, - {"_QENDTS", TK_QENDTS}, - {"_QSTARTTS", TK_QSTARTTS}, - {"_ROWTS", TK_ROWTS}, - {"_WDURATION", TK_WDURATION}, - {"_WENDTS", TK_WENDTS}, - {"_WSTARTTS", TK_WSTARTTS}, + {"SLIDING", TK_SLIDING}, + {"SLIMIT", TK_SLIMIT}, + {"SMA", TK_SMA}, + {"SMALLINT", TK_SMALLINT}, + {"SNODE", TK_SNODE}, + {"SNODES", TK_SNODES}, + {"SOFFSET", TK_SOFFSET}, + {"STABLE", TK_STABLE}, + {"STABLES", TK_STABLES}, + {"STATE", TK_STATE}, + {"STATE_WINDOW", TK_STATE_WINDOW}, + {"STORAGE", TK_STORAGE}, + {"STREAM", TK_STREAM}, + {"STREAMS", TK_STREAMS}, + {"STRICT", TK_STRICT}, + {"SYNCDB", TK_SYNCDB}, + {"TABLE", TK_TABLE}, + {"TABLES", TK_TABLES}, + {"TAG", TK_TAG}, + {"TAGS", TK_TAGS}, + {"TBNAME", TK_TBNAME}, + {"TIMESTAMP", TK_TIMESTAMP}, + {"TIMEZONE", TK_TIMEZONE}, + {"TINYINT", TK_TINYINT}, + {"TODAY", TK_TODAY}, + {"TOPIC", TK_TOPIC}, + {"TOPICS", TK_TOPICS}, + {"TRIGGER", TK_TRIGGER}, + {"TSERIES", TK_TSERIES}, + {"TTL", TK_TTL}, + {"UNION", TK_UNION}, + {"UNSIGNED", TK_UNSIGNED}, + {"USE", TK_USE}, + {"USER", TK_USER}, + {"USERS", TK_USERS}, + {"USING", TK_USING}, + {"VALUE", TK_VALUE}, + {"VALUES", TK_VALUES}, + {"VARCHAR", TK_VARCHAR}, + {"VARIABLES", TK_VARIABLES}, + {"VERBOSE", TK_VERBOSE}, + {"VGROUPS", TK_VGROUPS}, + {"VNODES", TK_VNODES}, + {"WAL", TK_WAL}, + {"WATERMARK", TK_WATERMARK}, + {"WHERE", TK_WHERE}, + {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, + {"WITH", TK_WITH}, + {"_QENDTS", TK_QENDTS}, + {"_QSTARTTS", TK_QSTARTTS}, + {"_ROWTS", TK_ROWTS}, + {"_WDURATION", TK_WDURATION}, + {"_WENDTS", TK_WENDTS}, + {"_WSTARTTS", TK_WSTARTTS}, // {"ID", TK_ID}, // {"STRING", TK_STRING}, // {"EQ", TK_EQ}, @@ -279,6 +278,7 @@ static SKeyword keywordTable[] = { // {"PARTITIONS", TK_PARTITIONS}, // {"MODE", TK_MODE}, }; +// clang-format on static const char isIdChar[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 4c3a4e8ab9..e765728cdf 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -24,170 +24,179 @@ class ParserSelectTest : public ParserTestBase {}; TEST_F(ParserSelectTest, basic) { useDb("root", "test"); - run("select * from t1"); + run("SELECT * FROM t1"); - run("select * from test.t1"); + run("SELECT * FROM test.t1"); - run("select ts, c1 from t1"); + run("SELECT ts, c1 FROM t1"); - run("select ts, t.c1 from (select * from t1) t"); + run("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); - run("select * from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1"); + run("SELECT * FROM t1 tt1, t1 tt2 where tt1.c1 = tt2.c1"); } TEST_F(ParserSelectTest, constant) { useDb("root", "test"); - run("select 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s from t1"); + run("SELECT 123, 20.4, 'abc', \"wxy\", timestamp '2022-02-09 17:30:20', true, false, 10s FROM t1"); - run("select 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " - "timestamp '2022-02-09 17:30:20', true, false, 15s from t1"); + run("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " + "timestamp '2022-02-09 17:30:20', true, false, 15s FROM t1"); - run("select 123 + 45 from t1 where 2 - 1"); + run("SELECT 123 + 45 FROM t1 where 2 - 1"); } TEST_F(ParserSelectTest, expression) { useDb("root", "test"); - run("select ts + 10s, c1 + 10, concat(c2, 'abc') from t1"); + run("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); - run("select ts > 0, c1 < 20 and c2 = 'qaz' from t1"); + run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1"); - run("select ts > 0, c1 between 10 and 20 and c2 = 'qaz' from t1"); + run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1"); } TEST_F(ParserSelectTest, condition) { useDb("root", "test"); - run("select c1 from t1 where ts in (true, false)"); + run("SELECT c1 FROM t1 where ts in (true, false)"); - run("select * from t1 where c1 > 10 and c1 is not null"); + run("SELECT * FROM t1 where c1 > 10 and c1 is not null"); } TEST_F(ParserSelectTest, pseudoColumn) { useDb("root", "test"); - run("select _wstartts, _wendts, count(*) from t1 interval(10s)"); + run("SELECT _wstartts, _wendts, COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(ParserSelectTest, multiResFunc) { useDb("root", "test"); - run("select last(*), first(*), last_row(*) from t1"); + run("SELECT last(*), first(*), last_row(*) FROM t1"); - run("select last(c1, c2), first(t1.*), last_row(c3) from t1"); + run("SELECT last(c1, c2), first(t1.*), last_row(c3) FROM t1"); - run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); } TEST_F(ParserSelectTest, timelineFunc) { useDb("root", "test"); - run("select last(*), first(*) from t1"); + run("SELECT last(*), first(*) FROM t1"); - run("select last(*), first(*) from t1 group by c1"); + run("SELECT last(*), first(*) FROM t1 group by c1"); - run("select last(*), first(*) from t1 interval(10s)"); + run("SELECT last(*), first(*) FROM t1 INTERVAL(10s)"); - run("select diff(c1) from t1"); + run("SELECT diff(c1) FROM t1"); } TEST_F(ParserSelectTest, clause) { useDb("root", "test"); // group by clause - run("select count(*) cnt from t1 where c1 > 0"); + run("SELECT COUNT(*) cnt FROM t1 where c1 > 0"); - run("select count(*), c2 cnt from t1 where c1 > 0 group by c2"); + run("SELECT COUNT(*), c2 cnt FROM t1 where c1 > 0 group by c2"); - run("select count(*) cnt from t1 where c1 > 0 group by c2 having count(c1) > 10"); + run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 having COUNT(c1) > 10"); - run("select count(*), c1, c2 + 10, c1 + c2 cnt from t1 where c1 > 0 group by c2, c1"); + run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 where c1 > 0 group by c2, c1"); - run("select count(*), c1 + 10, c2 cnt from t1 where c1 > 0 group by c1 + 10, c2"); + run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 where c1 > 0 group by c1 + 10, c2"); // order by clause - run("select count(*) cnt from t1 where c1 > 0 group by c2 order by cnt"); + run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 order by cnt"); - run("select count(*) cnt from t1 where c1 > 0 group by c2 order by 1"); + run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 order by 1"); // distinct clause - // run("select distinct c1, c2 from t1 where c1 > 0 order by c1"); + // run("SELECT distinct c1, c2 FROM t1 where c1 > 0 order by c1"); - // run("select distinct c1 + 10, c2 from t1 where c1 > 0 order by c1 + 10, c2"); + // run("SELECT distinct c1 + 10, c2 FROM t1 where c1 > 0 order by c1 + 10, c2"); - // run("select distinct c1 + 10 cc1, c2 cc2 from t1 where c1 > 0 order by cc1, c2"); + // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 where c1 > 0 order by cc1, c2"); - // run("select distinct count(c2) from t1 where c1 > 0 group by c1 order by count(c2)"); + // run("SELECT distinct COUNT(c2) FROM t1 where c1 > 0 group by c1 order by COUNT(c2)"); } -TEST_F(ParserSelectTest, window) { +// INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] +// fill_mod_and_val = { NONE | PREV | NULL | LINEAR | NEXT | value_mod } +// value_mod = VALUE , val ... +TEST_F(ParserSelectTest, interval) { useDb("root", "test"); - - run("select count(*) from t1 interval(10s)"); + // INTERVAL(interval_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); + // INTERVAL(interval_val, interval_offset) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s)"); + // INTERVAL(interval_val, interval_offset) SLIDING (sliding_val) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s) SLIDING(7s)"); + // INTERVAL(interval_val) FILL(NONE) + run("SELECT COUNT(*) FROM t1 INTERVAL(10s) FILL(NONE)"); } TEST_F(ParserSelectTest, semanticError) { useDb("root", "test"); // TSDB_CODE_PAR_INVALID_COLUMN - run("select c1, cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT c1, cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); - run("select t1.c1, t1.cc1 from t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT t1.c1, t1.cc1 FROM t1", TSDB_CODE_PAR_INVALID_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_TABLE_NOT_EXIST - run("select * from t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT * FROM t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); - run("select * from test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT * FROM test.t10", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); - run("select t2.c1 from t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); + run("SELECT t2.c1 FROM t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_AMBIGUOUS_COLUMN - run("select c2 from t1 tt1, t1 tt2 where tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 tt1, t1 tt2 where tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_VALUE_TYPE - run("select timestamp '2010' from t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); + run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION - run("select c2 from t1 tt1 join t1 tt2 on count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, + run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 where count(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 where COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 group by count(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 group by COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT - run("select c2 from t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); - run("select c2 from t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 order by 2", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION - run("select count(*) cnt from t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*) cnt FROM t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*) cnt from t1 group by c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 group by c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*), c1 cnt from t1 group by c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*), c1 cnt FROM t1 group by c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select count(*) cnt from t1 group by c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 group by c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SINGLE_GROUP - run("select count(*), c1 from t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*), c1 FROM t1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); - run("select count(*) from t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT COUNT(*) FROM t1 order by c1", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); - run("select c1 from t1 order by count(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT c1 FROM t1 order by COUNT(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION - run("select distinct c1, c2 from t1 where c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1, c2 FROM t1 where c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select distinct c1 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1 FROM t1 where c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("select distinct c2 from t1 where c1 > 0 order by count(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c2 FROM t1 where c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 00f2808294..fc0c5d5431 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -274,7 +274,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } if (TSDB_CODE_SUCCESS == code) { - code = nodesCollectFuncs(pSelect, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsScanPseudoColumnFunc, &pScan->pScanPseudoCols); } pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); @@ -440,7 +440,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pAgg->pAggFuncs); + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); } // rewrite the expression in subsequent clauses @@ -474,7 +474,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { - int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs); + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (pCxt->pPlanCxt->streamQuery) { pWindow->triggerType = pCxt->pPlanCxt->triggerType; @@ -559,14 +559,6 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return TSDB_CODE_OUT_OF_MEMORY; } - if (NULL != pInterval->pFill) { - pWindow->pFill = nodesCloneNode(pInterval->pFill); - if (NULL == pWindow->pFill) { - nodesDestroyNode(pWindow); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } @@ -589,6 +581,37 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } +static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || + NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { + return TSDB_CODE_SUCCESS; + } + + SFillNode* pFillNode = (SFillNode*)(((SIntervalWindowNode*)pSelect->pWindow)->pFill); + + SFillLogicNode* pFill = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); + + pFill->mode = pFillNode->mode; + pFill->pValues = nodesCloneNode(pFillNode->pValues); + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if ((NULL != pFillNode->pValues && NULL == pFill->pValues) || NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pOrderByList) { return TSDB_CODE_SUCCESS; @@ -753,6 +776,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createFillLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 901afce718..10dac9422a 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -17,8 +17,8 @@ #include "catalog.h" #include "functionMgt.h" -#include "tglobal.h" #include "systable.h" +#include "tglobal.h" typedef struct SSlotIdInfo { int16_t slotId; @@ -880,12 +880,6 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil pInterval->intervalUnit = pWindowLogicNode->intervalUnit; pInterval->slidingUnit = pWindowLogicNode->slidingUnit; - pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); - if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { - nodesDestroyNode(pInterval); - return TSDB_CODE_OUT_OF_MEMORY; - } - return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); } @@ -1035,6 +1029,43 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi return code; } +static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, + SPhysiNode** pPhyNode) { + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode, + QUERY_NODE_PHYSICAL_PLAN_FILL); + if (NULL == pFill) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->node.pTargets, &pFill->pTargets); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pFill->pTargets, pFill->node.pOutputDataBlockDesc); + } + + if (TSDB_CODE_SUCCESS == code) { + pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); + if (NULL == pFill->pWStartTs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pFillNode->pValues) { + pFill->pValues = nodesCloneNode(pFillNode->pValues); + if (NULL == pFill->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pFill; + } else { + nodesDestroyNode(pFill); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { @@ -1054,6 +1085,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_PARTITION: return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_FILL: + return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index 6cc67a3446..88810a0eb8 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -23,19 +23,19 @@ class PlanIntervalTest : public PlannerTestBase {}; TEST_F(PlanIntervalTest, basic) { useDb("root", "test"); - run("select count(*) from t1 interval(10s)"); + run("SELECT COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(PlanIntervalTest, pseudoCol) { useDb("root", "test"); - run("select _wstartts, _wduration, _wendts, count(*) from t1 interval(10s)"); + run("SELECT _WSTARTTS, _WDURATION, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(PlanIntervalTest, fill) { useDb("root", "test"); - run("select count(*) from t1 interval(10s) fill(linear)"); + run("SELECT COUNT(*) FROM t1 INTERVAL(10s) FILL(LINEAR)"); - run("select count(*), sum(c1) from t1 interval(10s) fill(value, 10, 20)"); + run("SELECT COUNT(*), sum(c1) FROM t1 INTERVAL(10s) FILL(VALUE, 10, 20)"); } From aa1eace9020728055742d6e4e7ded1b4eab2c1fc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 30 Apr 2022 12:14:27 +0800 Subject: [PATCH 16/90] fix(rpc):fix rpc fix --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 80b5d88a90..c9c3ee4e9c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -937,7 +937,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tDeserializeSMEpSet(pResp->pCont, pResp->contLen, &emsg); pCtx->epSet = emsg.epSet; } - addConnToPool(pThrd, pConn); + addConnToPool(pThrd->pool, pConn); tTrace("use remote epset, current in use: %d, retry count%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); From 77d3a3461ea6d804914ca0e4487bcc3c8fa3fa13 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 30 Apr 2022 12:17:27 +0800 Subject: [PATCH 17/90] fix(rpc):fix rpc fix --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c9c3ee4e9c..30e53f521c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -938,7 +938,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->epSet = emsg.epSet; } addConnToPool(pThrd->pool, pConn); - tTrace("use remote epset, current in use: %d, retry count%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, + tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); From c6cf9638b5b2558490f448d92755d4f54e0badc4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 30 Apr 2022 12:40:10 +0800 Subject: [PATCH 18/90] modify case alter_option.sim --- tests/script/tsim/db/alter_option.sim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index cc69386fb1..417e53daff 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,8 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 -#sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -87,7 +86,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 3 then # replica +if $data4_db != 1 then # replica return -1 endi if $data5_db != nostrict then # strict From 0ebd90b549eaa178b8709993a03f69b29f286039 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 30 Apr 2022 13:48:07 +0800 Subject: [PATCH 19/90] fix(query): set the correct primary timestamp column for first/last function. --- source/libs/executor/src/executorimpl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d8e69c4b70..e38f757f10 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1101,7 +1101,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column + // the last parameter is the timestamp column + if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) { + pInput->pPTS = pInput->pData[j]; + } ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { // todo avoid case: top(k, 12), 12 is the value parameter. From cdfe9929c8aa8f82ed80213cd2843a7a341c48eb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 30 Apr 2022 14:07:44 +0800 Subject: [PATCH 20/90] feat: fill physical plan --- include/common/taosdef.h | 15 +- include/libs/nodes/plannodes.h | 20 ++- include/libs/nodes/querynodes.h | 9 +- include/util/taoserror.h | 3 + source/libs/executor/src/executorimpl.c | 2 - source/libs/nodes/src/nodesCodeFuncs.c | 42 +++++ source/libs/parser/src/parTranslater.c | 162 +++++++++++++++--- source/libs/parser/src/parUtil.c | 3 +- source/libs/parser/test/parSelectTest.cpp | 45 ++--- source/libs/planner/inc/planInt.h | 3 + source/libs/planner/src/planLogicCreater.c | 1 + source/libs/planner/src/planOptimizer.c | 12 +- source/libs/planner/src/planPhysiCreater.c | 3 + source/libs/planner/src/planUtil.c | 36 ++++ source/libs/planner/test/planBasicTest.cpp | 26 +-- source/libs/planner/test/planIntervalTest.cpp | 7 +- 16 files changed, 295 insertions(+), 94 deletions(-) create mode 100644 source/libs/planner/src/planUtil.c diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 85d11be817..dd1661f871 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -28,15 +28,16 @@ typedef int64_t tb_uid_t; #define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query TSDB_SYSTEM_TABLE = 6, - TSDB_TABLE_MAX = 7 + TSDB_TABLE_MAX = 7 } ETableType; typedef enum { @@ -86,7 +87,7 @@ typedef enum { extern char *qtypeStr[]; -#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_HTTP 11 #undef TD_DEBUG_PRINT_ROW diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 10688ce239..a756217c7a 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -110,10 +110,11 @@ typedef struct SWindowLogicNode { } SWindowLogicNode; typedef struct SFillLogicNode { - SLogicNode node; - EFillMode mode; - SNode* pWStartTs; - SNode* pValues; // SNodeListNode + SLogicNode node; + EFillMode mode; + SNode* pWStartTs; + SNode* pValues; // SNodeListNode + STimeWindow timeRange; } SFillLogicNode; typedef struct SSortLogicNode { @@ -274,11 +275,12 @@ typedef struct SIntervalPhysiNode { } SIntervalPhysiNode; typedef struct SFillPhysiNode { - SPhysiNode node; - EFillMode mode; - SNode* pWStartTs; // SColumnNode - SNode* pValues; // SNodeListNode - SNodeList* pTargets; + SPhysiNode node; + EFillMode mode; + SNode* pWStartTs; // SColumnNode + SNode* pValues; // SNodeListNode + SNodeList* pTargets; + STimeWindow timeRange; } SFillPhysiNode; typedef struct SMultiTableIntervalPhysiNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 29f31f25a0..91c80f6cf5 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -209,10 +209,11 @@ typedef enum EFillMode { } EFillMode; typedef struct SFillNode { - ENodeType type; // QUERY_NODE_FILL - EFillMode mode; - SNode* pValues; // SNodeListNode - SNode* pWStartTs; // _wstartts pseudo column + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode + SNode* pWStartTs; // _wstartts pseudo column + STimeWindow timeRange; } SFillNode; typedef struct SSelectStmt { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 00708f0933..27af7eb27e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -619,9 +619,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) #define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) +#define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) +#define TSDB_CODE_PLAN_EXPECTED_TS_EQUAL TAOS_DEF_ERROR_CODE(0, 0x2701) +#define TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN TAOS_DEF_ERROR_CODE(0, 0x2702) //function #define TSDB_CODE_FUNC_FUNTION_ERROR TAOS_DEF_ERROR_CODE(0, 0x2800) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9f17a4cc7b..a88873dbd5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -41,8 +41,6 @@ #define SET_MAIN_SCAN_FLAG(runtime) ((runtime)->scanFlag = MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) - #define SDATA_BLOCK_INITIALIZER \ (SDataBlockInfo) { {0}, 0 } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e1ea23226b..e1f9a9f6a4 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -571,6 +571,8 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { static const char* jkFillLogicPlanMode = "Mode"; static const char* jkFillLogicPlanWStartTs = "WStartTs"; static const char* jkFillLogicPlanValues = "Values"; +static const char* jkFillLogicPlanStartTime = "StartTime"; +static const char* jkFillLogicPlanEndTime = "EndTime"; static int32_t logicFillNodeToJson(const void* pObj, SJson* pJson) { const SFillLogicNode* pNode = (const SFillLogicNode*)pObj; @@ -585,6 +587,12 @@ static int32_t logicFillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkFillLogicPlanValues, nodeToJson, pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillLogicPlanEndTime, pNode->timeRange.ekey); + } return code; } @@ -602,6 +610,12 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillLogicPlanValues, &pNode->pValues); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillLogicPlanEndTime, &pNode->timeRange.ekey); + } return code; } @@ -1475,6 +1489,8 @@ static const char* jkFillPhysiPlanMode = "Mode"; static const char* jkFillPhysiPlanWStartTs = "WStartTs"; static const char* jkFillPhysiPlanValues = "Values"; static const char* jkFillPhysiPlanTargets = "Targets"; +static const char* jkFillPhysiPlanStartTime = "StartTime"; +static const char* jkFillPhysiPlanEndTime = "EndTime"; static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; @@ -1492,6 +1508,12 @@ static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkFillPhysiPlanTargets, pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillPhysiPlanEndTime, pNode->timeRange.ekey); + } return code; } @@ -1512,6 +1534,12 @@ static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkFillPhysiPlanTargets, &pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillPhysiPlanEndTime, &pNode->timeRange.ekey); + } return code; } @@ -2409,6 +2437,8 @@ static int32_t jsonToNodeListNode(const SJson* pJson, void* pObj) { static const char* jkFillMode = "Mode"; static const char* jkFillValues = "Values"; static const char* jkFillWStartTs = "WStartTs"; +static const char* jkFillStartTime = "StartTime"; +static const char* jkFillEndTime = "EndTime"; static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { const SFillNode* pNode = (const SFillNode*)pObj; @@ -2420,6 +2450,12 @@ static int32_t fillNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkFillWStartTs, nodeToJson, pNode->pWStartTs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillStartTime, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFillEndTime, pNode->timeRange.ekey); + } return code; } @@ -2434,6 +2470,12 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillWStartTs, &pNode->pWStartTs); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillStartTime, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkFillEndTime, &pNode->timeRange.ekey); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2d08342214..d9b250998e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -17,6 +17,7 @@ #include "catalog.h" #include "cmdnodes.h" +#include "filter.h" #include "functionMgt.h" #include "parUtil.h" #include "scalar.h" @@ -468,19 +469,19 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); *(bool*)&pVal->typeData = pVal->datum.b; break; - case TSDB_DATA_TYPE_TINYINT:{ + case TSDB_DATA_TYPE_TINYINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int8_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_SMALLINT:{ + case TSDB_DATA_TYPE_SMALLINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int16_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_INT:{ + case TSDB_DATA_TYPE_INT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); *(int32_t*)&pVal->typeData = pVal->datum.i; @@ -489,43 +490,43 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_BIGINT: { char* endPtr = NULL; pVal->datum.i = strtoll(pVal->literal, &endPtr, 10); - *(int64_t*)&pVal->typeData = pVal->datum.i; + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } - case TSDB_DATA_TYPE_UTINYINT:{ + case TSDB_DATA_TYPE_UTINYINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint8_t*)&pVal->typeData = pVal->datum.u; + *(uint8_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_USMALLINT:{ + case TSDB_DATA_TYPE_USMALLINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint16_t*)&pVal->typeData = pVal->datum.u; + *(uint16_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_UINT:{ + case TSDB_DATA_TYPE_UINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint32_t*)&pVal->typeData = pVal->datum.u; + *(uint32_t*)&pVal->typeData = pVal->datum.u; break; } case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); - *(uint64_t*)&pVal->typeData = pVal->datum.u; + *(uint64_t*)&pVal->typeData = pVal->datum.u; break; } - case TSDB_DATA_TYPE_FLOAT:{ + case TSDB_DATA_TYPE_FLOAT: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); - *(float*)&pVal->typeData = pVal->datum.d; + *(float*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_DOUBLE: { char* endPtr = NULL; pVal->datum.d = strtold(pVal->literal, &endPtr); - *(double*)&pVal->typeData = pVal->datum.d; + *(double*)&pVal->typeData = pVal->datum.d; break; } case TSDB_DATA_TYPE_VARCHAR: @@ -543,7 +544,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } - *(int64_t*)&pVal->typeData = pVal->datum.i; + *(int64_t*)&pVal->typeData = pVal->datum.i; break; } case TSDB_DATA_TYPE_NCHAR: @@ -1244,6 +1245,113 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static EDealRes isPrimaryKeyCondImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool isPrimaryKeyCond(SNode* pNode) { + bool isPrimaryKeyCond = false; + nodesWalkExpr(pNode, isPrimaryKeyCondImpl, &isPrimaryKeyCond); + return isPrimaryKeyCond; +} + +static int32_t getTimeRangeFromLogicCond(STranslateContext* pCxt, SLogicConditionNode* pLogicCond, + STimeWindow* pTimeRange) { + SNodeList* pPrimaryKeyConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (isPrimaryKeyCond(pCond)) { + if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pPrimaryKeyConds, pCond)) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + } + + if (NULL == pPrimaryKeyConds) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + SLogicConditionNode* pPrimaryKeyLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pPrimaryKeyLogicCond) { + nodesClearList(pPrimaryKeyConds); + return TSDB_CODE_OUT_OF_MEMORY; + } + pPrimaryKeyLogicCond->condType = LOGIC_COND_TYPE_AND; + pPrimaryKeyLogicCond->pParameterList = pPrimaryKeyConds; + bool isStrict = false; + int32_t code = filterGetTimeRange((SNode*)pPrimaryKeyLogicCond, pTimeRange, &isStrict); + nodesClearList(pPrimaryKeyConds); + pPrimaryKeyLogicCond->pParameterList = NULL; + nodesDestroyNode(pPrimaryKeyLogicCond); + return code; +} + +static int32_t getTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { + if (NULL == pWhere) { + *pTimeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pWhere)->condType) { + return getTimeRangeFromLogicCond(pCxt, (SLogicConditionNode*)pWhere, pTimeRange); + } + + if (isPrimaryKeyCond(pWhere)) { + bool isStrict = false; + return filterGetTimeRange(pWhere, pTimeRange, &isStrict); + } else { + *pTimeRange = TSWINDOW_INITIALIZER; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t checkFill(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { + SFillNode* pFill = (SFillNode*)pInterval->pFill; + if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || + TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); + int64_t intervalRange = 0; + SValueNode* pInter = (SValueNode*)pInterval->pInterval; + if (TIME_IS_VAR_DURATION(pInter->unit)) { + int64_t f = 1; + if (pInter->unit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pInter->unit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + intervalRange = pInter->datum.i * f; + } else { + intervalRange = pInter->datum.i; + } + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFill(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { + if (NULL == pInterval->pFill) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = getTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange)); + if (TSDB_CODE_SUCCESS == code) { + code = checkFill(pCxt, pInterval); + } + return code; +} + static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); switch (unit) { @@ -1266,7 +1374,7 @@ static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni return -1; } -static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { +static int32_t checkIntervalWindow(STranslateContext* pCxt, SNode* pWhere, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; @@ -1308,7 +1416,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* } } - return TSDB_CODE_SUCCESS; + return translateFill(pCxt, pWhere, pInterval); } static EDealRes checkStateExpr(SNode* pNode, void* pContext) { @@ -1345,28 +1453,28 @@ static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* p return TSDB_CODE_SUCCESS; } -static int32_t checkWindow(STranslateContext* pCxt, SNode* pWindow) { - switch (nodeType(pWindow)) { +static int32_t checkWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return checkStateWindow(pCxt, (SStateWindowNode*)pWindow); + return checkStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); case QUERY_NODE_SESSION_WINDOW: - return checkSessionWindow(pCxt, (SSessionWindowNode*)pWindow); + return checkSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: - return checkIntervalWindow(pCxt, (SIntervalWindowNode*)pWindow); + return checkIntervalWindow(pCxt, pSelect->pWhere, (SIntervalWindowNode*)pSelect->pWindow); default: break; } return TSDB_CODE_SUCCESS; } -static int32_t translateWindow(STranslateContext* pCxt, SNode* pWindow) { - if (NULL == pWindow) { +static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_WINDOW; - int32_t code = translateExpr(pCxt, pWindow); + int32_t code = translateExpr(pCxt, pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { - code = checkWindow(pCxt, pWindow); + code = checkWindow(pCxt, pSelect); } return code; } @@ -1485,7 +1593,7 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { code = translatePartitionBy(pCxt, pSelect->pPartitionByList); } if (TSDB_CODE_SUCCESS == code) { - code = translateWindow(pCxt, pSelect->pWindow); + code = translateWindow(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = translateGroupBy(pCxt, pSelect); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 2bf4071d33..e839536218 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -128,6 +128,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid topic query"; case TSDB_CODE_PAR_INVALID_DROP_STABLE: return "Cannot drop super table in batch"; + case TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE: + return "start(end) time of query range required or time range too large"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -140,7 +142,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...) { va_start(vArgList, errCode); vsnprintf(pBuf->buf, pBuf->len, getSyntaxErrFormat(errCode), vArgList); va_end(vArgList); - terrno = errCode; return errCode; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index e765728cdf..c80a0341d2 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -32,7 +32,7 @@ TEST_F(ParserSelectTest, basic) { run("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); - run("SELECT * FROM t1 tt1, t1 tt2 where tt1.c1 = tt2.c1"); + run("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); } TEST_F(ParserSelectTest, constant) { @@ -43,7 +43,7 @@ TEST_F(ParserSelectTest, constant) { run("SELECT 1234567890123456789012345678901234567890, 20.1234567890123456789012345678901234567890, 'abc', \"wxy\", " "timestamp '2022-02-09 17:30:20', true, false, 15s FROM t1"); - run("SELECT 123 + 45 FROM t1 where 2 - 1"); + run("SELECT 123 + 45 FROM t1 WHERE 2 - 1"); } TEST_F(ParserSelectTest, expression) { @@ -59,9 +59,9 @@ TEST_F(ParserSelectTest, expression) { TEST_F(ParserSelectTest, condition) { useDb("root", "test"); - run("SELECT c1 FROM t1 where ts in (true, false)"); + run("SELECT c1 FROM t1 WHERE ts in (true, false)"); - run("SELECT * FROM t1 where c1 > 10 and c1 is not null"); + run("SELECT * FROM t1 WHERE c1 > 10 and c1 is not null"); } TEST_F(ParserSelectTest, pseudoColumn) { @@ -77,7 +77,7 @@ TEST_F(ParserSelectTest, multiResFunc) { run("SELECT last(c1, c2), first(t1.*), last_row(c3) FROM t1"); - run("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); + run("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); } TEST_F(ParserSelectTest, timelineFunc) { @@ -96,29 +96,29 @@ TEST_F(ParserSelectTest, clause) { useDb("root", "test"); // group by clause - run("SELECT COUNT(*) cnt FROM t1 where c1 > 0"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0"); - run("SELECT COUNT(*), c2 cnt FROM t1 where c1 > 0 group by c2"); + run("SELECT COUNT(*), c2 cnt FROM t1 WHERE c1 > 0 group by c2"); - run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 having COUNT(c1) > 10"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 having COUNT(c1) > 10"); - run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 where c1 > 0 group by c2, c1"); + run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 group by c2, c1"); - run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 where c1 > 0 group by c1 + 10, c2"); + run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 group by c1 + 10, c2"); // order by clause - run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 order by cnt"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 order by cnt"); - run("SELECT COUNT(*) cnt FROM t1 where c1 > 0 group by c2 order by 1"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 order by 1"); // distinct clause - // run("SELECT distinct c1, c2 FROM t1 where c1 > 0 order by c1"); + // run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by c1"); - // run("SELECT distinct c1 + 10, c2 FROM t1 where c1 > 0 order by c1 + 10, c2"); + // run("SELECT distinct c1 + 10, c2 FROM t1 WHERE c1 > 0 order by c1 + 10, c2"); - // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 where c1 > 0 order by cc1, c2"); + // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 order by cc1, c2"); - // run("SELECT distinct COUNT(c2) FROM t1 where c1 > 0 group by c1 order by COUNT(c2)"); + // run("SELECT distinct COUNT(c2) FROM t1 WHERE c1 > 0 group by c1 order by COUNT(c2)"); } // INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] @@ -133,7 +133,8 @@ TEST_F(ParserSelectTest, interval) { // INTERVAL(interval_val, interval_offset) SLIDING (sliding_val) run("SELECT COUNT(*) FROM t1 INTERVAL(10s, 5s) SLIDING(7s)"); // INTERVAL(interval_val) FILL(NONE) - run("SELECT COUNT(*) FROM t1 INTERVAL(10s) FILL(NONE)"); + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(NONE)"); } TEST_F(ParserSelectTest, semanticError) { @@ -152,7 +153,7 @@ TEST_F(ParserSelectTest, semanticError) { run("SELECT t2.c1 FROM t1", TSDB_CODE_PAR_TABLE_NOT_EXIST, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_AMBIGUOUS_COLUMN - run("SELECT c2 FROM t1 tt1, t1 tt2 where tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1", TSDB_CODE_PAR_AMBIGUOUS_COLUMN, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_VALUE_TYPE run("SELECT timestamp '2010' FROM t1", TSDB_CODE_PAR_WRONG_VALUE_TYPE, PARSER_STAGE_TRANSLATE); @@ -161,7 +162,7 @@ TEST_F(ParserSelectTest, semanticError) { run("SELECT c2 FROM t1 tt1 join t1 tt2 on COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("SELECT c2 FROM t1 where COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 WHERE COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); run("SELECT c2 FROM t1 group by COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); @@ -190,13 +191,13 @@ TEST_F(ParserSelectTest, semanticError) { run("SELECT c1 FROM t1 order by COUNT(*)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION - run("SELECT distinct c1, c2 FROM t1 where c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by ts", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("SELECT distinct c1 FROM t1 where c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c1 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("SELECT distinct c2 FROM t1 where c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, + run("SELECT distinct c2 FROM t1 WHERE c1 > 0 order by COUNT(c2)", TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION, PARSER_STAGE_TRANSLATE); } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index e6534f710d..4640ed99bd 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "planner.h" +#include "taoserror.h" #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 @@ -33,6 +34,8 @@ extern "C" { #define planDebug(param, ...) qDebug("PLAN: " param, __VA_ARGS__) #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); + int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index fc0c5d5431..0bc700696b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -597,6 +597,7 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; pFill->pValues = nodesCloneNode(pFillNode->pValues); pFill->pWStartTs = nodesCloneNode(pFillNode->pWStartTs); if ((NULL != pFillNode->pValues && NULL == pFill->pValues) || NULL == pFill->pWStartTs) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 47dcd9ce86..67609355d7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -554,22 +554,19 @@ static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } return TSDB_CODE_SUCCESS; } static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { if (LOGIC_COND_TYPE_AND != pOnCond->condType) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } SNode* pCond = NULL; FOREACH(pCond, pOnCond->pParameterList) { if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_EXPECTED_TS_EQUAL); } } return TSDB_CODE_SUCCESS; @@ -577,8 +574,7 @@ static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join"); - return TSDB_CODE_FAILED; + return generateUsageErrMsg(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 10dac9422a..05b48d2d6a 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1037,6 +1037,9 @@ static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren return TSDB_CODE_OUT_OF_MEMORY; } + pFill->mode = pFillNode->mode; + pFill->timeRange = pFillNode->timeRange; + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); int32_t code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFillNode->node.pTargets, &pFill->pTargets); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c new file mode 100644 index 0000000000..36625b28fb --- /dev/null +++ b/source/libs/planner/src/planUtil.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planInt.h" + +static char* getUsageErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PLAN_EXPECTED_TS_EQUAL: + return "l.ts = r.ts is expected in join expression"; + case TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN: + return "not support cross join"; + default: + break; + } + return "Unknown error"; +} + +int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...) { + va_list vArgList; + va_start(vArgList, errCode); + vsnprintf(pBuf, len, getUsageErrFormat(errCode), vArgList); + va_end(vArgList); + return errCode; +} diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 05a009d79d..639af2875e 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -1,7 +1,7 @@ /* * Copyright (c) 2019 TAOS Data, Inc. * - * This program is free software: you can use, redistribute, and/or modify + * This program is free software: you can use, redistribute, AND/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. * @@ -20,30 +20,32 @@ using namespace std; class PlanBasicTest : public PlannerTestBase {}; -TEST_F(PlanBasicTest, select) { +TEST_F(PlanBasicTest, selectClause) { useDb("root", "test"); - run("select * from t1"); - run("select 1 from t1"); - run("select * from st1"); - run("select 1 from st1"); + run("SELECT * FROM t1"); + run("SELECT 1 FROM t1"); + run("SELECT * FROM st1"); + run("SELECT 1 FROM st1"); } -TEST_F(PlanBasicTest, where) { +TEST_F(PlanBasicTest, whereClause) { useDb("root", "test"); - run("select * from t1 where c1 > 10"); + run("SELECT * FROM t1 WHERE c1 > 10"); + + run("SELECT * FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59'"); } -TEST_F(PlanBasicTest, join) { +TEST_F(PlanBasicTest, joinClause) { useDb("root", "test"); - run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); - run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); + run("SELECT t1.c1, t2.c2 FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts"); } TEST_F(PlanBasicTest, func) { useDb("root", "test"); - run("select diff(c1) from t1"); + run("SELECT DIFF(c1) FROM t1"); } diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index 88810a0eb8..f1bcd747c0 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -35,7 +35,10 @@ TEST_F(PlanIntervalTest, pseudoCol) { TEST_F(PlanIntervalTest, fill) { useDb("root", "test"); - run("SELECT COUNT(*) FROM t1 INTERVAL(10s) FILL(LINEAR)"); + run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(LINEAR)"); - run("SELECT COUNT(*), sum(c1) FROM t1 INTERVAL(10s) FILL(VALUE, 10, 20)"); + run("SELECT COUNT(*), SUM(c1) FROM t1 " + "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(VALUE, 10, 20)"); } From 498cbc62fc4438b15b59e2bb569281405efb21e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 30 Apr 2022 15:13:18 +0800 Subject: [PATCH 21/90] fix(query): update the capacity for ssdatablock when merge new blocks. --- include/common/tcommon.h | 2 +- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 52 +++++++++++++++---------- source/libs/executor/src/executorimpl.c | 7 ++-- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c0eb53d8ca..16653ebfee 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -70,7 +70,7 @@ typedef struct SDataBlockInfo { uint64_t groupId; // no need to serialize int16_t numOfCols; int16_t hasVarCol; - int16_t capacity; + int32_t capacity; } SDataBlockInfo; typedef struct SSDataBlock { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 71e2e4fba3..0d5262fe29 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -183,7 +183,7 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u } int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4a6fef4626..6619f46da0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -168,13 +168,6 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint32_t total = numOfRow1 + numOfRow2; - if (BitmapLen(numOfRow1) < BitmapLen(total)) { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(total)); - uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); - memset(tmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = tmp; - } - uint32_t remindBits = BitPos(numOfRow1); uint32_t shiftBits = 8 - remindBits; @@ -209,10 +202,9 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, - uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); - if (numOfRow2 == 0) { return numOfRow1; } @@ -221,14 +213,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co pColumnInfoData->hasNull = pSource->hasNull; } + uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + if (finalNumOfRows > *capacity) { + char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + *capacity = finalNumOfRows; + pColumnInfoData->varmeta.offset = (int32_t*)p; } - pColumnInfoData->varmeta.offset = (int32_t*)p; for (int32_t i = 0; i < numOfRow2; ++i) { if (pSource->varmeta.offset[i] == -1) { pColumnInfoData->varmeta.offset[i + numOfRow1] = -1; @@ -253,15 +250,27 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + if (finalNumOfRows > *capacity) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows); + if (tmp == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } - int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + pColumnInfoData->pData = tmp; + + if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { + char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); + memset(btmp + BitmapLen(numOfRow1), 0, extend); + + pColumnInfoData->nullbitmap = btmp; + } + + *capacity = finalNumOfRows; } - pColumnInfoData->pData = tmp; + doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + int32_t offset = pColumnInfoData->info.bytes * numOfRow1; memcpy(pColumnInfoData->pData + offset, pSource->pData, pSource->info.bytes * numOfRow2); } @@ -357,6 +366,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd if(pIndexMap) { mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); } + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); @@ -367,7 +377,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd char* tmp = taosMemoryRealloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; - colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); + colDataMergeCol(pCol2, pDest->info.rows, &pDest->info.capacity, pCol1, pSrc->info.rows); } else { return TSDB_CODE_VND_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e38f757f10..9e5e17a053 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1199,7 +1199,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); if (pResult->info.rows > 0 && !createNewColModel) { - colDataMergeCol(pColInfoData, pResult->info.rows, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); } @@ -1227,7 +1227,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1268,7 +1268,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); + colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -7170,7 +7170,6 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) { SJoinOperatorInfo* pJoinInfo = pOperator->info; - // SOptrBasicInfo* pInfo = &pJoinInfo->binfo; SSDataBlock* pRes = pJoinInfo->pRes; blockDataCleanup(pRes); From 912e0df6bb4e23e993f61984dac0dd0dd3a66336 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 30 Apr 2022 15:31:43 +0800 Subject: [PATCH 22/90] fix(query): set the correct memory buffer size when merging two column data. --- source/common/src/tdatablock.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 6619f46da0..e237f61e20 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -361,6 +361,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd assert(pSrc != NULL && pDest != NULL); int32_t numOfCols = pDest->info.numOfCols; + int32_t capacity = pDest->info.capacity; + for (int32_t i = 0; i < numOfCols; ++i) { int32_t mapIndex = i; if(pIndexMap) { @@ -377,12 +379,14 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd char* tmp = taosMemoryRealloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; - colDataMergeCol(pCol2, pDest->info.rows, &pDest->info.capacity, pCol1, pSrc->info.rows); + capacity = pDest->info.capacity; + colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); } else { return TSDB_CODE_VND_OUT_OF_MEMORY; } } + pDest->info.capacity = capacity; pDest->info.rows += pSrc->info.rows; return TSDB_CODE_SUCCESS; } From 2bbeeb5c5b69c3c826868564c683847fdcafc822 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 30 Apr 2022 15:44:06 +0800 Subject: [PATCH 23/90] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e237f61e20..b065ba4fd4 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -251,7 +251,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows); + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -372,18 +372,18 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); - uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); - uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); +// uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); +// uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); - int32_t newSize = oldLen + newLen; - char* tmp = taosMemoryRealloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; +// int32_t newSize = oldLen + newLen; +// char* tmp = taosMemoryRealloc(pCol2->pData, newSize); +// if (tmp != NULL) { +// pCol2->pData = tmp; capacity = pDest->info.capacity; colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); - } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; - } +// } else { +// return TSDB_CODE_VND_OUT_OF_MEMORY; +// } } pDest->info.capacity = capacity; From c52ca4cce8f96d8e3a24b29d95ae48a93110f103 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 30 Apr 2022 15:45:10 +0800 Subject: [PATCH 24/90] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b065ba4fd4..e703063dfa 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -359,31 +359,19 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { // if pIndexMap = NULL, merger one column by on column int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) { assert(pSrc != NULL && pDest != NULL); - - int32_t numOfCols = pDest->info.numOfCols; int32_t capacity = pDest->info.capacity; - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; - if(pIndexMap) { + if (pIndexMap) { mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); } SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); -// uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); -// uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); - -// int32_t newSize = oldLen + newLen; -// char* tmp = taosMemoryRealloc(pCol2->pData, newSize); -// if (tmp != NULL) { -// pCol2->pData = tmp; - capacity = pDest->info.capacity; - colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); -// } else { -// return TSDB_CODE_VND_OUT_OF_MEMORY; -// } + capacity = pDest->info.capacity; + colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); } pDest->info.capacity = capacity; From f99c01e66e756e28f03c425955c5697fd62f4455 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 30 Apr 2022 19:11:25 +0800 Subject: [PATCH 25/90] feat: sql command 'select max(c1), c2 from t' --- source/libs/parser/src/parTranslater.c | 184 ++++++++++------------ source/libs/parser/test/parSelectTest.cpp | 46 ++++-- 2 files changed, 116 insertions(+), 114 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d9b250998e..1e030fdc45 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -624,6 +624,70 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { + *pOutput = pTable; + return TSDB_CODE_SUCCESS; + } + } + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); +} + +static bool isCountStar(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); +} + +// count(*) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { + SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } + return code; +} + +static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } else { + // todo + } + *pPrimaryKey = (SNode*)pCol; + return TSDB_CODE_SUCCESS; +} + +static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); + } + return code; +} + +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + SNode* pPrimaryKey = NULL; + int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); + } + return code; +} + static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, @@ -631,10 +695,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) .pErrBuf = pCxt->msgBuf.buf, .errBufLen = pCxt->msgBuf.len}; pCxt->errCode = fmGetFuncInfo(¶m, pFunc); - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - return DEAL_RES_ERROR; - } - if (fmIsAggFunc(pFunc->funcId)) { + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsAggFunc(pFunc->funcId)) { if (beforeHaving(pCxt->currClause)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION); } @@ -643,11 +704,17 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (haveAggFunc) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } + pCxt->pCurrStmt->hasAggFuncs = true; pCxt->pCurrStmt->isTimeOrderQuery = false; + if (isCountStar(pFunc)) { + pCxt->errCode = rewriteCountStar(pCxt, pFunc); + } } - - return DEAL_RES_CONTINUE; + if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsTimelineFunc(pFunc->funcId)) { + pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { @@ -691,7 +758,13 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } -static bool isAliasColumn(SColumnNode* pCol) { return ('\0' == pCol->tableAlias[0]); } +static bool isAliasColumn(const SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0])); +} + +static bool isAggFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); +} static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); @@ -720,11 +793,10 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; - if (!nodesIsExprNode(pNode) || (QUERY_NODE_COLUMN == nodeType(pNode) && isAliasColumn((SColumnNode*)pNode))) { + if (!nodesIsExprNode(pNode) || isAliasColumn(pNode)) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - !isDistinctOrderBy(pCxt)) { + if (isAggFunc(pNode) && !isDistinctOrderBy(pCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; @@ -733,9 +805,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(pNode) || - (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId) && - isDistinctOrderBy(pCxt))) { + if (QUERY_NODE_COLUMN == nodeType(pNode) || (isAggFunc(pNode) && isDistinctOrderBy(pCxt))) { return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); } return DEAL_RES_CONTINUE; @@ -990,19 +1060,6 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { return (SNode*)pFunc; } -static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput) { - SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); - size_t nums = taosArrayGetSize(pTables); - for (size_t i = 0; i < nums; ++i) { - STableNode* pTable = taosArrayGetP(pTables, i); - if (NULL == pTableAlias || 0 == strcmp(pTable->tableAlias, pTableAlias)) { - *pOutput = pTable; - return TSDB_CODE_SUCCESS; - } - } - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pTableAlias); -} - static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) { STableNode* pTable = NULL; int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); @@ -1511,78 +1568,6 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -static bool isCountStar(SFunctionNode* pFunc) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); -} - -// count(*) is rewritten as count(ts) for scannning optimization -static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { - SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } - return code; -} - -static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { - SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } else { - // todo - } - *pPrimaryKey = (SNode*)pCol; - return TSDB_CODE_SUCCESS; -} - -static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, NULL, &pTable); - if (TSDB_CODE_SUCCESS == code) { - code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); - } - return code; -} - -static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - SNode* pPrimaryKey = NULL; - int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); - } - return code; -} - -EDealRes rewriteFuncForSelectImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - STranslateContext* pCxt = pContext; - SFunctionNode* pFunc = (SFunctionNode*)pNode; - if (isCountStar(pFunc)) { - pCxt->errCode = rewriteCountStar(pCxt, pFunc); - } else if (fmIsTimelineFunc(pFunc->funcId)) { - pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); - } - if (TSDB_CODE_SUCCESS != pCxt->errCode) { - return DEAL_RES_ERROR; - } - } - return DEAL_RES_CONTINUE; -} - -static int32_t rewriteFuncForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteFuncForSelectImpl, pCxt); - return pCxt->errCode; -} - static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1613,9 +1598,6 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } - if (TSDB_CODE_SUCCESS == code) { - code = rewriteFuncForSelect(pCxt, pSelect); - } return code; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index c80a0341d2..19105d4360 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -85,31 +85,45 @@ TEST_F(ParserSelectTest, timelineFunc) { run("SELECT last(*), first(*) FROM t1"); - run("SELECT last(*), first(*) FROM t1 group by c1"); + run("SELECT last(*), first(*) FROM t1 GROUP BY c1"); run("SELECT last(*), first(*) FROM t1 INTERVAL(10s)"); run("SELECT diff(c1) FROM t1"); } +TEST_F(ParserSelectTest, selectFunc) { + useDb("root", "test"); + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), * FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); +} + TEST_F(ParserSelectTest, clause) { useDb("root", "test"); - // group by clause + // GROUP BY clause run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0"); - run("SELECT COUNT(*), c2 cnt FROM t1 WHERE c1 > 0 group by c2"); + run("SELECT COUNT(*), c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2"); - run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 having COUNT(c1) > 10"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 having COUNT(c1) > 10"); - run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 group by c2, c1"); + run("SELECT COUNT(*), c1, c2 + 10, c1 + c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c2, c1"); - run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 group by c1 + 10, c2"); + run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); // order by clause - run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 order by cnt"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by cnt"); - run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 group by c2 order by 1"); + run("SELECT COUNT(*) cnt FROM t1 WHERE c1 > 0 GROUP BY c2 order by 1"); // distinct clause // run("SELECT distinct c1, c2 FROM t1 WHERE c1 > 0 order by c1"); @@ -118,7 +132,7 @@ TEST_F(ParserSelectTest, clause) { // run("SELECT distinct c1 + 10 cc1, c2 cc2 FROM t1 WHERE c1 > 0 order by cc1, c2"); - // run("SELECT distinct COUNT(c2) FROM t1 WHERE c1 > 0 group by c1 order by COUNT(c2)"); + // run("SELECT distinct COUNT(c2) FROM t1 WHERE c1 > 0 GROUP BY c1 order by COUNT(c2)"); } // INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] @@ -137,6 +151,12 @@ TEST_F(ParserSelectTest, interval) { "INTERVAL(10s) FILL(NONE)"); } +TEST_F(ParserSelectTest, intervalSemanticCheck) { + useDb("root", "test"); + + run("SELECT c1 FROM t1 INTERVAL(10s)"); +} + TEST_F(ParserSelectTest, semanticError) { useDb("root", "test"); @@ -164,7 +184,7 @@ TEST_F(ParserSelectTest, semanticError) { run("SELECT c2 FROM t1 WHERE COUNT(*) > 0", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); - run("SELECT c2 FROM t1 group by COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); + run("SELECT c2 FROM t1 GROUP BY COUNT(*)", TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT run("SELECT c2 FROM t1 order by 0", TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT, PARSER_STAGE_TRANSLATE); @@ -174,13 +194,13 @@ TEST_F(ParserSelectTest, semanticError) { // TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION run("SELECT COUNT(*) cnt FROM t1 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("SELECT COUNT(*) cnt FROM t1 group by c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c1 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("SELECT COUNT(*), c1 cnt FROM t1 group by c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*), c1 cnt FROM t1 GROUP BY c2 having c2 > 0", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); - run("SELECT COUNT(*) cnt FROM t1 group by c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, + run("SELECT COUNT(*) cnt FROM t1 GROUP BY c2 having c2 > 0 order by c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION, PARSER_STAGE_TRANSLATE); // TSDB_CODE_PAR_NOT_SINGLE_GROUP From 3d931faa5bc3c530aa65e9ffc10a9f8496c53afb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 1 May 2022 00:23:22 +0800 Subject: [PATCH 26/90] refactor(query): do some internal refactor. --- source/libs/function/src/builtinsimpl.c | 108 ++++++++++++++++++------ 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index bb01af420d..05dddb50aa 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -78,6 +78,8 @@ typedef struct SDiffInfo { int64_t i64; double d64; } prev; + + int64_t prevTs; } SDiffInfo; typedef struct SSpreadInfo { @@ -1196,8 +1198,8 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { bool isFirstBlock = (pDiffInfo->hasPrev == false); int32_t numOfElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - // int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); +// int32_t start = (pCtx->order == TSDB_ORDER_ASC) ? pInput->startRowIndex : pInput->numOfRows + pInput->startRowIndex - 1; SColumnInfoData* pTsOutput = pCtx->pTsOutput; TSKEY* tsList = (int64_t*)pInput->pPTS->pData; @@ -1206,44 +1208,100 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { switch (pInputCol->info.type) { case TSDB_DATA_TYPE_INT: { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { - int32_t pos = startOffset + (isFirstBlock ? (numOfElems - 1) : numOfElems); - if (colDataIsNull_f(pInputCol->nullbitmap, i)) { - if (pDiffInfo->includeNull) { - colDataSetNull_f(pOutput->nullbitmap, pos); - if (tsList != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + if (pCtx->order == TSDB_ORDER_ASC) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t pos = startOffset + (isFirstBlock ? (numOfElems - 1) : numOfElems); + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + if (pDiffInfo->includeNull) { + colDataSetNull_f(pOutput->nullbitmap, pos); + if (tsList != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + + numOfElems += 1; + } + continue; + } + + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + if (pDiffInfo->hasPrev) { + int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); } - numOfElems += 1; + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } } - continue; - } - int32_t v = *(int32_t*)colDataGetData(pInputCol, i); - if (pDiffInfo->hasPrev) { - int32_t delta = (int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null - if (delta < 0 && pDiffInfo->ignoreNegative) { - colDataSetNull_f(pOutput->nullbitmap, pos); - } else { + pDiffInfo->prev.i64 = v; + pDiffInfo->hasPrev = true; + numOfElems++; + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + int32_t pos = startOffset + numOfElems; + + if (pDiffInfo->hasPrev) { + int32_t delta = -(int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + if (delta < 0 && pDiffInfo->ignoreNegative) { + colDataSetNull_f(pOutput->nullbitmap, pos); + } else { + colDataAppendInt32(pOutput, pos, &delta); + } + + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + pDiffInfo->hasPrev = false; + } + + if (i < pInput->numOfRows + pInput->startRowIndex - 1) { + int32_t next = *(int32_t*)colDataGetData(pInputCol, i + 1); + + int32_t delta = v - next; // direct previous may be null colDataAppendInt32(pOutput, pos, &delta); - } - if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + if (pTsOutput != NULL) { + colDataAppendInt64(pTsOutput, pos, &tsList[i]); + } + } else { + pDiffInfo->prev.i64 = v; + if (pTsOutput != NULL) { + pDiffInfo->prevTs = tsList[i]; + } + pDiffInfo->hasPrev = true; } +// if (colDataIsNull_f(pInputCol->nullbitmap, i)) { +// if (pDiffInfo->includeNull) { +// colDataSetNull_f(pOutput->nullbitmap, pos); +// if (tsList != NULL) { +// colDataAppendInt64(pTsOutput, pos, &tsList[i]); +// } +// +// numOfElems += 1; +// } +// continue; +// } + +// int32_t v = *(int32_t*)colDataGetData(pInputCol, i); + +// pDiffInfo->prev.i64 = v; +// pDiffInfo->hasPrev = true; + numOfElems++; } - pDiffInfo->prev.i64 = v; - pDiffInfo->hasPrev = true; - numOfElems++; } break; } case TSDB_DATA_TYPE_BIGINT: { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } From 3b4f9b91cf36b5635a53efb6a941e5bb1778964b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 1 May 2022 08:38:17 +0800 Subject: [PATCH 27/90] feat: sql commadn 'select max(c1), c2 from t2' --- include/libs/function/functionMgt.h | 4 + source/libs/function/inc/functionMgtInt.h | 1 + source/libs/function/src/builtins.c | 19 ++- source/libs/function/src/functionMgt.c | 2 + source/libs/parser/src/parTranslater.c | 127 ++++++++++++++---- .../libs/parser/test/mockCatalogService.cpp | 1 - source/libs/parser/test/parSelectTest.cpp | 11 +- source/libs/planner/test/planGroupByTest.cpp | 37 +++-- source/libs/planner/test/planIntervalTest.cpp | 9 ++ source/libs/planner/test/planSessionTest.cpp | 9 ++ source/libs/planner/test/planStateTest.cpp | 9 ++ 11 files changed, 186 insertions(+), 43 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f6304401fc..1fa9db9927 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -113,6 +113,9 @@ typedef enum EFunctionType { FUNCTION_TYPE_WENDTS, FUNCTION_TYPE_WDURATION, + // internal function + FUNCTION_TYPE_SELECT_VALUE, + // user defined funcion FUNCTION_TYPE_UDF = 10000 } EFunctionType; @@ -141,6 +144,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsNonstandardSQLFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index e19a332e66..3869a5d7b2 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -39,6 +39,7 @@ extern "C" { #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10) #define FUNC_MGT_MULTI_RES_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(11) #define FUNC_MGT_SCAN_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(12) +#define FUNC_MGT_SELECT_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(13) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 8f3c88900d..bd2fb9dca1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -438,6 +438,11 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -465,7 +470,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "min", .type = FUNCTION_TYPE_MIN, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, @@ -476,7 +481,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "max", .type = FUNCTION_TYPE_MAX, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, @@ -974,6 +979,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = NULL, .sprocessFunc = toJsonFunction, .finalizeFunc = NULL + }, + { + .name = "_select_value", + .type = FUNCTION_TYPE_SELECT_VALUE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .translateFunc = translateSelectValue, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL } }; // clang-format on diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index b505f2e8ec..f8ef0f7d20 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -145,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SELECT_FUNC); } + bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1e030fdc45..82f51f70c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -256,6 +256,22 @@ static void destroyTranslateContext(STranslateContext* pCxt) { taosHashCleanup(pCxt->pTables); } +static bool isAliasColumn(const SNode* pNode) { + return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0])); +} + +static bool isAggFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isSelectFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); +} + +static bool isDistinctOrderBy(STranslateContext* pCxt) { + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); +} + static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { int cmp = 0; if ('\0' != pCol->dbName[0]) { @@ -617,7 +633,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveAggFunction(SNode* pNode, void* pContext) { - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + if (isAggFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -758,18 +774,6 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { return pCxt->errCode; } -static bool isAliasColumn(const SNode* pNode) { - return (QUERY_NODE_COLUMN == nodeType(pNode) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0])); -} - -static bool isAggFunc(const SNode* pNode) { - return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)); -} - -static bool isDistinctOrderBy(STranslateContext* pCxt) { - return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); -} - static SNodeList* getGroupByList(STranslateContext* pCxt) { if (isDistinctOrderBy(pCxt)) { return pCxt->pCurrStmt->pProjectionList; @@ -791,28 +795,71 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } -static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { - STranslateContext* pCxt = (STranslateContext*)pContext; - if (!nodesIsExprNode(pNode) || isAliasColumn(pNode)) { +typedef struct SCheckExprForGroupByCxt { + STranslateContext* pTranslateCxt; + int32_t selectFuncNum; + bool hasSelectValFunc; +} SCheckExprForGroupByCxt; + +static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, bool* pHasSelectValFunc, SNode** pNode) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pFunc->functionName, "_select_value"); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + translateFunction(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + if (NULL != pHasSelectValFunc) { + *pHasSelectValFunc = true; + } + } else { + nodesDestroyNode(pFunc); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; +} + +static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { + SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; + if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { return DEAL_RES_CONTINUE; } - if (isAggFunc(pNode) && !isDistinctOrderBy(pCxt)) { + pCxt->selectFuncNum += isSelectFunc(*pNode) ? 1 : 0; + if (pCxt->selectFuncNum > 1 && pCxt->hasSelectValFunc) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } + if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } SNode* pGroupNode; - FOREACH(pGroupNode, getGroupByList(pCxt)) { - if (nodesEqualNode(getGroupByNode(pGroupNode), pNode)) { + FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { + if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { return DEAL_RES_IGNORE_CHILD; } } - if (QUERY_NODE_COLUMN == nodeType(pNode) || (isAggFunc(pNode) && isDistinctOrderBy(pCxt))) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (pCxt->selectFuncNum > 1) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); + } else { + return rewriteColToSelectValFunc(pCxt->pTranslateCxt, &pCxt->hasSelectValFunc, pNode); + } + } + if (isAggFunc(*pNode) && isDistinctOrderBy(pCxt->pTranslateCxt)) { + return generateDealNodeErrMsg(pCxt->pTranslateCxt, getGroupByErrorCode(pCxt->pTranslateCxt)); } return DEAL_RES_CONTINUE; } -static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); +static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode** pNode) { + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExpr(pNode, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } return pCxt->errCode; } @@ -820,7 +867,29 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); + SCheckExprForGroupByCxt cxt = {.pTranslateCxt = pCxt, .selectFuncNum = 0, .hasSelectValFunc = false}; + nodesRewriteExprs(pList, doCheckExprForGroupBy, &cxt); + if (cxt.selectFuncNum != 1 && cxt.hasSelectValFunc) { + return generateSyntaxErrMsg(&pCxt->msgBuf, getGroupByErrorCode(pCxt)); + } + return pCxt->errCode; +} + +static EDealRes rewriteColsToSelectValFuncImpl(SNode** pNode, void* pContext) { + if (isAggFunc(*pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + return rewriteColToSelectValFunc((STranslateContext*)pContext, NULL, pNode); + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteColsToSelectValFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteColsToSelectValFuncImpl, pCxt); + } return pCxt->errCode; } @@ -828,11 +897,13 @@ typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; bool existCol; + int32_t selectFuncNum; } CheckAggColCoexistCxt; static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + pCxt->selectFuncNum += isSelectFunc(pNode) ? 1 : 0; + if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } @@ -851,7 +922,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (!pSelect->isDistinct) { nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } - if ((cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { + if (1 == cxt.selectFuncNum) { + return rewriteColsToSelectValFunc(pCxt, pSelect); + } else if ((cxt.selectFuncNum > 1 || cxt.existAggFunc || NULL != pSelect->pWindow) && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); } return TSDB_CODE_SUCCESS; @@ -1285,7 +1358,7 @@ static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, pSelect->pHaving); if (TSDB_CODE_SUCCESS == code) { - code = checkExprForGroupBy(pCxt, pSelect->pHaving); + code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 2c85bfdf2e..f86cecb9e3 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -103,7 +103,6 @@ class MockCatalogServiceImpl { const char* tname = tNameGetTableName(pTableName); int32_t code = copyTableSchemaMeta(db, tname, &table); if (TSDB_CODE_SUCCESS != code) { - std::cout << "db : " << db << ", table :" << tname << std::endl; return code; } *pTableMeta = table.release(); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 19105d4360..58e517d571 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -94,6 +94,7 @@ TEST_F(ParserSelectTest, timelineFunc) { TEST_F(ParserSelectTest, selectFunc) { useDb("root", "test"); + // select function run("SELECT MAX(c1), MIN(c1) FROM t1"); // select function for GROUP BY clause @@ -102,8 +103,14 @@ TEST_F(ParserSelectTest, selectFunc) { run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); // select function along with the columns of select row run("SELECT MAX(c1), c2 FROM t1"); - run("SELECT MAX(c1), * FROM t1"); run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); + // select function along with the columns of select row, and with window clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); } TEST_F(ParserSelectTest, clause) { @@ -154,7 +161,7 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)"); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); } TEST_F(ParserSelectTest, semanticError) { diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 05ffe41fe7..9ca1001f4c 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -23,30 +23,45 @@ class PlanGroupByTest : public PlannerTestBase {}; TEST_F(PlanGroupByTest, basic) { useDb("root", "test"); - run("select count(*) from t1"); + run("SELECT COUNT(*) FROM t1"); - run("select c1, max(c3), min(c3), count(*) from t1 group by c1"); + run("SELECT c1, MAX(c3), MIN(c3), COUNT(*) FROM t1 GROUP BY c1"); - run("select c1 + c3, c1 + count(*) from t1 where c2 = 'abc' group by c1, c3"); + run("SELECT c1 + c3, c1 + COUNT(*) FROM t1 WHERE c2 = 'abc' GROUP BY c1, c3"); - run("select c1 + c3, sum(c4 * c5) from t1 where concat(c2, 'wwww') = 'abcwww' group by c1 + c3"); + run("SELECT c1 + c3, SUM(c4 * c5) FROM t1 WHERE CONCAT(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); - run("select sum(ceil(c1)) from t1 group by ceil(c1)"); + run("SELECT SUM(CEIL(c1)) FROM t1 GROUP BY CEIL(c1)"); } TEST_F(PlanGroupByTest, withOrderBy) { useDb("root", "test"); - // order by aggfunc - run("select count(*), sum(c1) from t1 order by sum(c1)"); - // order by alias of aggfunc - // run("select count(*), sum(c1) a from t1 order by a"); + // ORDER BY aggfunc + run("SELECT COUNT(*), SUM(c1) FROM t1 ORDER BY SUM(c1)"); + // ORDER BY alias of aggfunc + // run("SELECT COUNT(*), SUM(c1) a FROM t1 ORDER BY a"); } TEST_F(PlanGroupByTest, aggFunc) { useDb("root", "test"); - run("select last(*), first(*) from t1"); + run("SELECT LAST(*), FIRST(*) FROM t1"); - run("select last(*), first(*) from t1 group by c1"); + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); +} + +TEST_F(PlanGroupByTest, selectFunc) { + useDb("root", "test"); + + // select function + run("SELECT MAX(c1), MIN(c1) FROM t1"); + // select function for GROUP BY clause + run("SELECT MAX(c1), MIN(c1) FROM t1 GROUP BY c1"); + // select function along with the columns of select row + run("SELECT MAX(c1), c2 FROM t1"); + run("SELECT MAX(c1), t1.* FROM t1"); + // select function along with the columns of select row, and with GROUP BY clause + run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); + run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index f1bcd747c0..c9bae46ca9 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -42,3 +42,12 @@ TEST_F(PlanIntervalTest, fill) { "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " "INTERVAL(10s) FILL(VALUE, 10, 20)"); } + +TEST_F(PlanIntervalTest, selectFunc) { + useDb("root", "test"); + + // select function for INTERVAL clause + run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); + // select function along with the columns of select row, and with INTERVAL clause + run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); +} \ No newline at end of file diff --git a/source/libs/planner/test/planSessionTest.cpp b/source/libs/planner/test/planSessionTest.cpp index 3ec9c4c387..7d5d826925 100644 --- a/source/libs/planner/test/planSessionTest.cpp +++ b/source/libs/planner/test/planSessionTest.cpp @@ -25,3 +25,12 @@ TEST_F(PlanSessionTest, basic) { run("select count(*) from t1 session(ts, 10s)"); } + +TEST_F(PlanSessionTest, selectFunc) { + useDb("root", "test"); + + // select function for SESSION clause + run("SELECT MAX(c1), MIN(c1) FROM t1 SESSION(ts, 10s)"); + // select function along with the columns of select row, and with SESSION clause + run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 46988d6706..83c9621916 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -31,3 +31,12 @@ TEST_F(PlanStateTest, stateExpr) { run("select count(*) from t1 state_window(c1 + 10)"); } + +TEST_F(PlanStateTest, selectFunc) { + useDb("root", "test"); + + // select function for STATE_WINDOW clause + run("SELECT MAX(c1), MIN(c1) FROM t1 STATE_WINDOW(c3)"); + // select function along with the columns of select row, and with STATE_WINDOW clause + run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); +} From afcf0c8a63110d3bd6c3a41ad4a96e5cf04e14b0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 1 May 2022 08:56:59 +0800 Subject: [PATCH 28/90] feat: sql commadn 'select max(c1), c2 from t' --- tests/system-test/2-query/distinct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index a82f3a6f59..63794a5022 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -111,7 +111,7 @@ class TDTestCase: # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") + # tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") From d6cf688368e23d1b87cfcea44612b2eaf44953d8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 1 May 2022 09:21:36 +0800 Subject: [PATCH 29/90] feat: sql commadn 'select max(c1), c2 from t' --- source/libs/parser/test/parSelectTest.cpp | 2 ++ tests/system-test/2-query/distinct.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 58e517d571..4e0d2851f6 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -162,6 +162,8 @@ TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_SINGLE_GROUP, PARSER_STAGE_TRANSLATE); + run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, + PARSER_STAGE_TRANSLATE); } TEST_F(ParserSelectTest, semanticError) { diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index 63794a5022..a82f3a6f59 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -111,7 +111,7 @@ class TDTestCase: # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - # tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") + tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") From d97f1cc3887f658abc0ea826d6d421917dc25f52 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 1 May 2022 09:53:52 +0800 Subject: [PATCH 30/90] feat: sql commadn 'select max(c1), c2 from t' --- source/libs/parser/src/parTranslater.c | 84 +++++++++++++---------- source/libs/parser/test/parSelectTest.cpp | 16 +++-- tests/system-test/2-query/distinct.py | 4 +- 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 82f51f70c1..4e2d75a79f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -268,6 +268,10 @@ static bool isSelectFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsSelectFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isTimelineFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isDistinctOrderBy(STranslateContext* pCxt) { return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); } @@ -672,38 +676,6 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) return code; } -static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { - SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } else { - // todo - } - *pPrimaryKey = (SNode*)pCol; - return TSDB_CODE_SUCCESS; -} - -static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, NULL, &pTable); - if (TSDB_CODE_SUCCESS == code) { - code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); - } - return code; -} - -static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - SNode* pPrimaryKey = NULL; - int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); - } - return code; -} - static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, @@ -727,9 +699,6 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) pCxt->errCode = rewriteCountStar(pCxt, pFunc); } } - if (TSDB_CODE_SUCCESS == pCxt->errCode && fmIsTimelineFunc(pFunc->funcId)) { - pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); - } return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -1641,6 +1610,48 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } else { + // todo + } + *pPrimaryKey = (SNode*)pCol; + return TSDB_CODE_SUCCESS; +} + +static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); + } + return code; +} + +static EDealRes rewriteTimelineFuncImpl(SNode* pNode, void* pContext) { + STranslateContext* pCxt = pContext; + if (isTimelineFunc(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + SNode* pPrimaryKey = NULL; + pCxt->errCode = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); + } + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); + return pCxt->errCode; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1671,6 +1682,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTimelineFunc(pCxt, pSelect); + } return code; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 4e0d2851f6..5a385ba25e 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -67,27 +67,29 @@ TEST_F(ParserSelectTest, condition) { TEST_F(ParserSelectTest, pseudoColumn) { useDb("root", "test"); - run("SELECT _wstartts, _wendts, COUNT(*) FROM t1 INTERVAL(10s)"); + run("SELECT _WSTARTTS, _WENDTS, COUNT(*) FROM t1 INTERVAL(10s)"); } TEST_F(ParserSelectTest, multiResFunc) { useDb("root", "test"); - run("SELECT last(*), first(*), last_row(*) FROM t1"); + run("SELECT LAST(*), FIRST(*), LAST_ROW(*) FROM t1"); - run("SELECT last(c1, c2), first(t1.*), last_row(c3) FROM t1"); + run("SELECT LAST(c1, c2), FIRST(t1.*), LAST_ROW(c3) FROM t1"); - run("SELECT last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); + run("SELECT LAST(t2.*), FIRST(t1.c1, t2.*), LAST_ROW(t1.*, t2.*) FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts"); } TEST_F(ParserSelectTest, timelineFunc) { useDb("root", "test"); - run("SELECT last(*), first(*) FROM t1"); + run("SELECT LAST(*), FIRST(*) FROM t1"); - run("SELECT last(*), first(*) FROM t1 GROUP BY c1"); + run("SELECT FIRST(ts), FIRST(c1), FIRST(c2), FIRST(c3) FROM t1"); - run("SELECT last(*), first(*) FROM t1 INTERVAL(10s)"); + run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); + + run("SELECT LAST(*), FIRST(*) FROM t1 INTERVAL(10s)"); run("SELECT diff(c1) FROM t1"); } diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index a82f3a6f59..8f27fe4db6 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -109,9 +109,9 @@ class TDTestCase: # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) ") tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) ") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") - tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") + # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") + # tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") From 7cdedc8bc8dddfec1eb69d3ff5d6cd17571ae381 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 1 May 2022 10:24:48 +0800 Subject: [PATCH 31/90] feat: sql commadn 'select max(c1), c2 from t' --- tests/system-test/2-query/distinct.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index 8f27fe4db6..937ff78c71 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -101,7 +101,7 @@ class TDTestCase: # tdSql.error("select distinct c1, ts from stb1 group by c2") tdSql.error("select distinct c1, ts from t1 group by c2") # tdSql.error("select distinct c1, max(c2) from stb1 ") - tdSql.error("select distinct c1, max(c2) from t1 ") + # tdSql.error("select distinct c1, max(c2) from t1 ") # tdSql.error("select max(c2), distinct c1 from stb1 ") tdSql.error("select max(c2), distinct c1 from t1 ") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 group by t0") @@ -109,9 +109,9 @@ class TDTestCase: # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) ") tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) ") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") - # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") + tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - # tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") + tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") From 5a0ae19cbc619b79ba32cc92cfb8f4f902a2f35f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 1 May 2022 10:50:23 +0800 Subject: [PATCH 32/90] feat: submit req msg iter refactor --- include/common/tmsg.h | 19 ++++----- source/common/src/tmsg.c | 23 +++++------ source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tqRead.c | 12 +++--- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 48 +++++----------------- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 ++++++-- source/dnode/vnode/src/tsdb/tsdbSma.c | 12 +++--- source/dnode/vnode/src/tsdb/tsdbWrite.c | 20 ++++----- source/dnode/vnode/src/vnd/vnodeOpen.c | 14 +++---- 11 files changed, 74 insertions(+), 97 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7e049b1dd9..d5d052c80a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -240,21 +240,18 @@ typedef struct { // head of SSubmitBlk const void* pMsg; } SSubmitMsgIter; - +#if 0 +int32_t tInitSubmitMsgIterOrigin(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tGetSubmitMsgNextOrigin(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); +int32_t tInitSubmitBlkIterOrigin(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +STSRow* tGetSubmitBlkNextOrigin(SSubmitBlkIter* pIter); +#endif +// TODO: KEEP one suite of iterator API finally. int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); -// TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter -int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); -STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter); - typedef struct { int32_t index; // index of failed block in submit blocks int32_t vnode; // vnode index of failed block diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 874e09b403..469d336cdd 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,8 +27,8 @@ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" - -int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +#if 0 +int32_t tInitSubmitMsgIterOrigin(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -46,7 +46,7 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return 0; } -int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { +int32_t tGetSubmitMsgNextOrigin(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { ASSERT(pIter->len >= 0); if (pIter->len == 0) { @@ -72,7 +72,7 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { return 0; } -int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { +int32_t tInitSubmitBlkIterOrigin(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pBlock->dataLen <= 0) return -1; pIter->totalLen = pBlock->dataLen; pIter->len = 0; @@ -80,7 +80,7 @@ int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { return 0; } -STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { +STSRow *tGetSubmitBlkNextOrigin(SSubmitBlkIter *pIter) { STSRow *row = pIter->row; if (pIter->len >= pIter->totalLen) { @@ -93,13 +93,10 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { return row; } } +#endif // TODO: KEEP one suite of iterator API finally. -// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts -// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later -// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter - -int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -117,7 +114,7 @@ int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { return 0; } -int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { +int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { ASSERT(pIter->len >= 0); if (pIter->len == 0) { @@ -152,7 +149,7 @@ int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { return 0; } -int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { +int32_t tInitSubmitBlkIter(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { if (pMsgIter->dataLen <= 0) return -1; pIter->totalLen = pMsgIter->dataLen; pIter->len = 0; @@ -160,7 +157,7 @@ int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubm return 0; } -STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { +STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { STSRow *row = pIter->row; if (pIter->len >= pIter->totalLen) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9ac8434949..427ce8c1b6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,7 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows); +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5930dcaf96..92923d4876 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -188,7 +188,7 @@ struct STbUidStore { #define TD_VID(PVNODE) (PVNODE)->config.vgId -static FORCE_INLINE bool tsdbIsRollup(SVnode* pVnode) { +static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); return (pRetention->freq > 0 && pRetention->keep > 0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 89ec55cca1..511d57ed58 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -37,9 +37,9 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); // iterate and convert - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1; if (pReadHandle->pBlock == NULL) break; // pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid); @@ -50,7 +50,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t // pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows); } - if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; pReadHandle->ver = ver; memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; @@ -58,7 +58,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t bool tqNextDataBlock(STqReadHandle* pHandle) { while (1) { - if (tGetSubmitMsgNextEx(&pHandle->msgIter, &pHandle->pBlock) < 0) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } if (pHandle->pBlock == NULL) return false; @@ -169,8 +169,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterInit(&iter, pTschema); STSRow* row; int32_t curRow = 0; - tInitSubmitBlkIterEx(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - while ((row = tGetSubmitBlkNextEx(&pHandle->blkIter)) != NULL) { + tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 48e672d9bc..ff4d99f510 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -190,35 +190,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows) { +int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { // STsdbMeta *pMeta = pRepo->tsdbMeta; // int32_t points = 0; // STable *pTable = NULL; @@ -232,15 +204,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows SSubmitBlk *pBlkCopy; // create container is nedd - tptr = taosHashGet(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid)); + tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid)); if (tptr == NULL) { - pTbData = tsdbNewTbData(pBlock->uid); + pTbData = tsdbNewTbData(pMsgIter->uid); if (pTbData == NULL) { return -1; } // Put into hash - taosHashPut(pMemTable->pHashIdx, &(pBlock->uid), sizeof(pBlock->uid), &(pTbData), sizeof(pTbData)); + taosHashPut(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid), &(pTbData), sizeof(pTbData)); // Put into skiplist tSkipListPut(pMemTable->pSlIdx, pTbData); @@ -249,10 +221,10 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pBlock->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pBlock->dataLen + sizeof(*pBlock)); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pMsgIter->dataLen + sizeof(*pBlock)); + memcpy(pBlkCopy, pBlock, pMsgIter->dataLen + sizeof(*pBlock)); - tInitSubmitBlkIter(pBlkCopy, &blkIter); + tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; keyMin = TD_ROW_KEY(blkIter.row); @@ -261,15 +233,15 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *pAffectedRows // Set statistics keyMax = TD_ROW_KEY(blkIter.row); - pTbData->nrows += pBlock->numOfRows; + pTbData->nrows += pMsgIter->numOfRows; if (pTbData->keyMin > keyMin) pTbData->keyMin = keyMin; if (pTbData->keyMax < keyMax) pTbData->keyMax = keyMax; - pMemTable->nRow += pBlock->numOfRows; + pMemTable->nRow += pMsgIter->numOfRows; if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pBlock->numOfRows; + (*pAffectedRows) += pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 0827ba6eab..da363d9bc9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -67,7 +67,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { goto _err; } - tsdbDebug("vgId: %d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path); + tsdbDebug("vgId:%d tsdb is opened for %s", TD_VID(pVnode), pTsdb->path); *ppTsdb = pTsdb; return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7d3f7cadc9..2b8558c5ac 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -351,14 +351,25 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData } } +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { + if (vnodeIsRollup(pVnode)) { + // for(int32_t i=0; i< TSDB_; ) { + + // } + } + return pVnode->pTsdb; +} + static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; } + STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + pReadHandle->order = pCond->order; - pReadHandle->pTsdb = pVnode->pTsdb; + pReadHandle->pTsdb = pTsdb; pReadHandle->type = TSDB_QUERY_TYPE_ALL; pReadHandle->cur.fid = INT32_MIN; pReadHandle->cur.win = TSWINDOW_INITIALIZER; @@ -376,7 +387,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* snprintf(buf, tListLen(buf), "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, qId); pReadHandle->idStr = strdup(buf); - if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)pVnode->pTsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, pReadHandle->pTsdb) != 0) { goto _end; } @@ -413,7 +424,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* pReadHandle->suppInfo.plist = taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); } - pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->pVnode->config.tsdbCfg.maxRows); + pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); if (pReadHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index d131cef8a2..380e376e24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -702,25 +702,25 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SInterval interval = {0}; TSKEY lastWinSKey = INT64_MIN; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } while (true) { - tGetSubmitMsgNextEx(&msgIter, &pBlock); + tGetSubmitMsgNext(&msgIter, &pBlock); if (!pBlock) break; STSmaWrapper *pSW = NULL; STSma *pTSma = NULL; SSubmitBlkIter blkIter = {0}; - if (tInitSubmitBlkIterEx(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { pSW = tdFreeTSmaWrapper(pSW); break; } while (true) { - STSRow *row = tGetSubmitBlkNextEx(&blkIter); + STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { tdFreeTSmaWrapper(pSW); break; @@ -1966,9 +1966,9 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIterEx(pMsg, &msgIter) < 0) return -1; + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { - if (tGetSubmitMsgNextEx(&msgIter, &pBlock) < 0) return -1; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 9fb6aad472..88b637bc24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -38,11 +38,11 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * while (true) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - if (tsdbInsertTableData(pTsdb, pBlock, &affectedrows) < 0) { + if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &affectedrows) < 0) { return -1; } - numOfRows += pBlock->numOfRows; + numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -66,20 +66,20 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { TSKEY maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days; terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + // pMsg->length = htonl(pMsg->length); + // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; if (pBlock == NULL) break; - pBlock->uid = htobe64(pBlock->uid); - pBlock->suid = htobe64(pBlock->suid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); + // pBlock->uid = htobe64(pBlock->uid); + // pBlock->suid = htobe64(pBlock->suid); + // pBlock->sversion = htonl(pBlock->sversion); + // pBlock->dataLen = htonl(pBlock->dataLen); + // pBlock->schemaLen = htonl(pBlock->schemaLen); + // pBlock->numOfRows = htons(pBlock->numOfRows); #if 0 if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3737bcfe3b..ae134e6496 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -96,24 +96,24 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { } // open tsdb - if (tsdbIsRollup(pVnode)) { + if (vnodeIsRollup(pVnode)) { if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L0) < 0) { - vError("vgId: %d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L1) < 0) { - vError("vgId: %d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } if (tsdbOpen(pVnode, TSDB_TYPE_RSMA_L2) < 0) { - vError("vgId: %d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } } else { if (tsdbOpen(pVnode, TSDB_TYPE_TSDB) < 0) { - vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } } @@ -160,8 +160,8 @@ _err: if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - tsdbClose(VND_RSMA1(pVnode)); - tsdbClose(VND_RSMA2(pVnode)); + tsdbClose(VND_RSMA1(pVnode)); + tsdbClose(VND_RSMA2(pVnode)); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; From b2d6ec14725615b73e97380f4bfaedf25468e5ed Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 1 May 2022 12:18:54 +0800 Subject: [PATCH 33/90] feat: rollup sma retention unit --- include/common/tmsg.h | 12 +-- source/common/src/tmsg.c | 92 +++------------------- source/dnode/mnode/impl/src/mndDb.c | 8 +- source/dnode/vnode/src/inc/tsdbSma.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbSma.c | 102 ++++++++++++------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/libs/parser/src/parTranslater.c | 3 - 8 files changed, 72 insertions(+), 152 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d5d052c80a..a8053d8854 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -181,8 +181,8 @@ typedef struct SField { } SField; typedef struct SRetention { - int32_t freq; - int32_t keep; + int64_t freq; + int64_t keep; int8_t freqUnit; int8_t keepUnit; } SRetention; @@ -240,13 +240,7 @@ typedef struct { // head of SSubmitBlk const void* pMsg; } SSubmitMsgIter; -#if 0 -int32_t tInitSubmitMsgIterOrigin(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); -int32_t tGetSubmitMsgNextOrigin(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int32_t tInitSubmitBlkIterOrigin(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); -STSRow* tGetSubmitBlkNextOrigin(SSubmitBlkIter* pIter); -#endif -// TODO: KEEP one suite of iterator API finally. + int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 469d336cdd..6235dcf895 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,75 +27,7 @@ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -#if 0 -int32_t tInitSubmitMsgIterOrigin(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - pIter->totalLen = pMsg->length; - ASSERT(pIter->totalLen > 0); - pIter->len = 0; - pIter->pMsg = pMsg; - if (pMsg->length <= sizeof(SSubmitReq)) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - return 0; -} - -int32_t tGetSubmitMsgNextOrigin(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - ASSERT(pIter->len >= 0); - - if (pIter->len == 0) { - pIter->len += sizeof(SSubmitReq); - } else { - if (pIter->len >= pIter->totalLen) { - ASSERT(0); - } - - SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); - ASSERT(pIter->len > 0); - } - - if (pIter->len > pIter->totalLen) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - *pPBlock = NULL; - return -1; - } - - *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - - return 0; -} - -int32_t tInitSubmitBlkIterOrigin(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->dataLen <= 0) return -1; - pIter->totalLen = pBlock->dataLen; - pIter->len = 0; - pIter->row = (STSRow *)(pBlock->data + pBlock->schemaLen); - return 0; -} - -STSRow *tGetSubmitBlkNextOrigin(SSubmitBlkIter *pIter) { - STSRow *row = pIter->row; - - if (pIter->len >= pIter->totalLen) { - return NULL; - } else { - pIter->len += TD_ROW_LEN(row); - if (pIter->len < pIter->totalLen) { - pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row)); - } - return row; - } -} -#endif - -// TODO: KEEP one suite of iterator API finally. int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; @@ -1679,8 +1611,8 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -1725,8 +1657,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { @@ -2155,8 +2087,8 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2199,8 +2131,8 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pRsp->pRetensions, &rentension) == NULL) { @@ -2817,8 +2749,8 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pReq->pRetensions, i); - if (tEncodeI32(&encoder, pRetension->freq) < 0) return -1; - if (tEncodeI32(&encoder, pRetension->keep) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->freq) < 0) return -1; + if (tEncodeI64(&encoder, pRetension->keep) < 0) return -1; if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } @@ -2874,8 +2806,8 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { SRetention rentension = {0}; - if (tDecodeI32(&decoder, &rentension.freq) < 0) return -1; - if (tDecodeI32(&decoder, &rentension.keep) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.freq) < 0) return -1; + if (tDecodeI64(&decoder, &rentension.keep) < 0) return -1; if (tDecodeI8(&decoder, &rentension.freqUnit) < 0) return -1; if (tDecodeI8(&decoder, &rentension.keepUnit) < 0) return -1; if (taosArrayPush(pReq->pRetensions, &rentension) == NULL) { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 442bfb5c8a..5861f8d245 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -107,8 +107,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions); SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i); - SDB_SET_INT32(pRaw, dataPos, pRetension->freq, _OVER) - SDB_SET_INT32(pRaw, dataPos, pRetension->keep, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER) + SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } @@ -180,8 +180,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { if (pDb->cfg.pRetensions == NULL) goto _OVER; for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) { SRetention retension = {0}; - SDB_GET_INT32(pRaw, dataPos, &retension.freq, _OVER) - SDB_GET_INT32(pRaw, dataPos, &retension.keep, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.freq, _OVER) + SDB_GET_INT64(pRaw, dataPos, &retension.keep, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.freqUnit, _OVER) SDB_GET_INT8(pRaw, dataPos, &retension.keepUnit, _OVER) if (taosArrayPush(pDb->cfg.pRetensions, &retension) == NULL) { diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 162d733cc3..5215812ac5 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -40,7 +40,6 @@ static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 92923d4876..a9ba2e1de3 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -123,7 +123,7 @@ int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, t int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); -int32_t tsdbTriggerRSma(STsdb* pTsdb, SMeta* pMeta, void* pMsg, int32_t inputType); +int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 380e376e24..d4532cc3ac 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -174,6 +174,8 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -1881,8 +1883,10 @@ int32_t tsdbFetchTbUidList(STsdb *pTsdb, STbUidStore **ppStore, tb_uid_t suid, t return TSDB_CODE_SUCCESS; } + ASSERT(ppStore != NULL); + if (!(*ppStore)) { - if (tsdbUidStoreInit((STbUidStore **)ppStore) != 0) { + if (tsdbUidStoreInit(ppStore) != 0) { return TSDB_CODE_FAILED; } } @@ -1978,7 +1982,44 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { return 0; } -int32_t tsdbExecuteRSma(STsdb *pTsdb, SMeta *pMeta, const void *pMsg, int32_t inputType, tb_uid_t *suid) { +static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, + qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention) { + SArray *pResult = NULL; + tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), retention, taskInfo, + suid); + qSetStreamInput(taskInfo, pMsg, inputType); + while (1) { + SSDataBlock *output; + uint64_t ts; + if (qExecTask(taskInfo, &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + if (!pResult) { + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + taosArrayPush(pResult, output); + } + + if (taosArrayGetSize(pResult) > 0) { + blockDebugShowData(pResult); + } else { + tsdbWarn("vgId:%d no rsma_1 data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); + } + + taosArrayDestroy(pResult); + + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists @@ -1988,65 +2029,22 @@ int32_t tsdbExecuteRSma(STsdb *pTsdb, SMeta *pMeta, const void *pMsg, int32_t in SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), *suid); + tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), suid); return TSDB_CODE_SUCCESS; } - SArray *pResult = NULL; - - pResult = taosArrayInit(0, sizeof(SSDataBlock)); - if (!pResult) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - if (pRSmaInfo->taskInfo[0]) { - tsdbDebug("vgId:%d execute rsma task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), pRSmaInfo->taskInfo[0], - *suid); - qSetStreamInput(pRSmaInfo->taskInfo[0], pMsg, inputType); - while (1) { - SSDataBlock *output; - uint64_t ts; - if (qExecTask(pRSmaInfo->taskInfo[0], &output, &ts) < 0) { - ASSERT(false); - } - if (!output) { - break; - } - taosArrayPush(pResult, output); - } - if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); - } else { - tsdbWarn("vgId:%d no sma data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - } - - // if (pRSmaInfo->taskInfo[1]) { - // qSetStreamInput(pRSmaInfo->taskInfo[1], pMsg, inputType); - // while (1) { - // SSDataBlock *output; - // uint64_t ts; - // if (qExecTask(pRSmaInfo->taskInfo[1], &output, &ts) < 0) { - // ASSERT(false); - // } - // if (!output) { - // break; - // } - // taosArrayPush(pResult, output); - // } - // blockDebugShowData(pResult); - // } + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], suid, TSDB_RSMA_RETENTION_1); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], suid, TSDB_RSMA_RETENTION_2); } return TSDB_CODE_SUCCESS; } -int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType) { +int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists @@ -2058,12 +2056,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputTyp tsdbFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, &uidStore.suid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, pTbSuid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid); pIter = taosHashIterate(uidStore.uidHash, pIter); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4eeba06027..eddd295e2e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -461,7 +461,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in SSubmitRsp rsp = {0}; pRsp->code = 0; - tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); // handle the request if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { pRsp->code = terrno; @@ -470,7 +470,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in // pRsp->msgType = TDMT_VND_SUBMIT_RSP; // vnodeProcessSubmitReq(pVnode, ptr, pRsp); - // tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4e2d75a79f..30e55420d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1947,9 +1947,6 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - if (!TIME_IS_VAR_DURATION(pVal->unit)) { - pVal->datum.i = convertTimeFromPrecisionToUnit(pVal->datum.i, pVal->node.resType.precision, pVal->unit); - } } } From f096f27ce82f894a7953ca56c999ca5d468455da Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 1 May 2022 15:22:28 +0800 Subject: [PATCH 34/90] refactor: do some internal refactor. --- source/libs/executor/src/executorimpl.c | 5 +--- source/libs/function/src/builtinsimpl.c | 37 ++++--------------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4fabe128a8..7e9c6b6fe3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1235,9 +1235,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc if (fmIsPseudoColumnFunc(pfCtx->functionId)) { // do nothing } else if (fmIsNonstandardSQLFunc(pfCtx->functionId)) { - // todo set the correct timestamp column - pfCtx->input.pPTS = taosArrayGet(pSrcBlock->pDataBlock, 1); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); pfCtx->fpSet.init(&pCtx[k], pResInfo); @@ -2490,7 +2487,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // if (pQueryAttr->pFilters != NULL) { // filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); // } - + // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { // filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); // } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 05dddb50aa..0eba442e66 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1198,9 +1198,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { bool isFirstBlock = (pDiffInfo->hasPrev == false); int32_t numOfElems = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); -// int32_t start = (pCtx->order == TSDB_ORDER_ASC) ? pInput->startRowIndex : pInput->numOfRows + pInput->startRowIndex - 1; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; TSKEY* tsList = (int64_t*)pInput->pPTS->pData; @@ -1246,8 +1243,9 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { int32_t v = *(int32_t*)colDataGetData(pInputCol, i); int32_t pos = startOffset + numOfElems; + // there is a row of previous data block to be handled in the first place. if (pDiffInfo->hasPrev) { - int32_t delta = -(int32_t)(v - pDiffInfo->prev.i64); // direct previous may be null + int32_t delta = (int32_t)(pDiffInfo->prev.i64 - v); // direct previous may be null if (delta < 0 && pDiffInfo->ignoreNegative) { colDataSetNull_f(pOutput->nullbitmap, pos); } else { @@ -1255,11 +1253,12 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { } if (pTsOutput != NULL) { - colDataAppendInt64(pTsOutput, pos, &tsList[i]); + colDataAppendInt64(pTsOutput, pos, &pDiffInfo->prevTs); } pDiffInfo->hasPrev = false; } + // it is not the last row of current block if (i < pInput->numOfRows + pInput->startRowIndex - 1) { int32_t next = *(int32_t*)colDataGetData(pInputCol, i + 1); @@ -1276,22 +1275,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { } pDiffInfo->hasPrev = true; } -// if (colDataIsNull_f(pInputCol->nullbitmap, i)) { -// if (pDiffInfo->includeNull) { -// colDataSetNull_f(pOutput->nullbitmap, pos); -// if (tsList != NULL) { -// colDataAppendInt64(pTsOutput, pos, &tsList[i]); -// } -// -// numOfElems += 1; -// } -// continue; -// } - -// int32_t v = *(int32_t*)colDataGetData(pInputCol, i); - -// pDiffInfo->prev.i64 = v; -// pDiffInfo->hasPrev = true; numOfElems++; } @@ -1436,7 +1419,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { } // initial value is not set yet - if (!pDiffInfo->hasPrev || numOfElems <= 0) { + if (numOfElems <= 0) { /* * 1. current block and blocks before are full of null * 2. current block may be null value @@ -1444,15 +1427,7 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) { assert(pCtx->hasNull); return 0; } else { - // for (int t = 0; t < pCtx->tagInfo.numOfTagCols; ++t) { - // SqlFunctionCtx* tagCtx = pCtx->tagInfo.pTagCtxList[t]; - // if (tagCtx->functionId == TSDB_FUNC_TAG_DUMMY) { - // aAggs[TSDB_FUNC_TAGPRJ].xFunction(tagCtx); - // } - // } - - int32_t forwardStep = (isFirstBlock) ? numOfElems - 1 : numOfElems; - return forwardStep; + return (isFirstBlock) ? numOfElems - 1 : numOfElems; } } From ee5b67a965c4d30ce9415193050273f0f61bf1f7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 1 May 2022 22:20:24 +0800 Subject: [PATCH 35/90] enh(index): support index filter --- source/libs/executor/CMakeLists.txt | 4 +- source/libs/executor/src/indexoperator.c | 181 ++++++++++++++++------- source/libs/index/src/index.c | 35 ++--- 3 files changed, 147 insertions(+), 73 deletions(-) diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 3dfef9b59f..bfa54be71f 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode scalar nodes + PRIVATE os util common function parser planner qcom vnode scalar nodes index ) target_include_directories( @@ -19,4 +19,4 @@ target_include_directories( #if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +#endif(${BUILD_TEST}) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 1f8e73b80a..911ba22fe6 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -15,7 +15,9 @@ #include "indexoperator.h" #include "executorimpl.h" +#include "index.h" #include "nodes.h" +#include "tdatablock.h" typedef struct SIFCtx { int32_t code; @@ -48,11 +50,19 @@ typedef struct SIFCtx { } while (0) typedef struct SIFParam { - SArray * result; SHashObj *pFilter; + + SArray *result; + char * condValue; + + col_id_t colId; + + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; -typedef int32_t (*sif_func_t)(SNode *left, SNode *rigth, SIFParam *output); +typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; @@ -60,7 +70,10 @@ static void destroyTagFilterOperatorInfo(void *param) { static void sifFreeParam(SIFParam *param) { if (param == NULL) return; + taosArrayDestroy(param->result); + taosMemoryFree(param->condValue); + taosHashCleanup(param->pFilter); } static int32_t sifGetOperParamNum(EOperatorType ty) { @@ -71,15 +84,70 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } +static int32_t sifValidateColumn(SColumnNode *cn) { + // add more check + if (cn == NULL) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + if (cn->colType != COLUMN_TYPE_TAG) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t sifGetValueFromNode(SNode *node, char **value) { + // covert data From snode; + SValueNode *vn = (SValueNode *)node; + + char * pData = nodesGetValueFromNode(vn); + SDataType *pType = &vn->node.resType; + int32_t type = pType->type; + int32_t valLen = 0; + + if (IS_VAR_DATA_TYPE(type)) { + int32_t dataLen = varDataTLen(pData); + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + valLen = dataLen; + } else { + valLen = pType->bytes; + } + char *tv = taosMemoryCalloc(1, valLen + 1); + if (tv == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(tv, pData, valLen); + *value = tv; + + return TSDB_CODE_SUCCESS; +} + static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; - + SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); + param->colId = -1; break; } case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; + /*only support tag column*/ + SIF_ERR_RET(sifValidateColumn(cn)); + param->colId = cn->colId; + memcpy(param->dbName, cn->dbName, sizeof(cn->dbName)); + memcpy(param->colName, cn->colName, sizeof(cn->colName)); break; } @@ -89,7 +157,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->dataType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); @@ -163,58 +231,63 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu qError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } - -static int32_t sifLessThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLessEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterThanFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifGreaterEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later +static int32_t sifIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + SIndexMultiTermQuery *mq = indexMultiTermQueryCreate(MUST); return TSDB_CODE_SUCCESS; } -static int32_t sifEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_THAN; + return sifIndex(left, right, id, output); } -static int32_t sifNotEqualFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifInFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotInFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; -} -static int32_t sifNotLikeFunc(SNode *left, SNode *right, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LOWER_EQUAL; + return sifIndex(left, right, id, output); } -static int32_t sifMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_THAN; + return sifIndex(left, right, id, output); } -static int32_t sifNotMatchFunc(SNode *left, SNode *rigth, SIFParam *output) { - // impl later - return TSDB_CODE_SUCCESS; +static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_GREATER_EQUAL; + return sifIndex(left, right, id, output); } -static int32_t sifDefaultFunc(SNode *left, SNode *rigth, SIFParam *output) { + +static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_EQUAL; + return sifIndex(left, right, id, output); +} +static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_EQUAL; + return sifIndex(left, right, id, output); +} +static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_IN; + return sifIndex(left, right, id, output); +} +static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_IN; + return sifIndex(left, right, id, output); +} +static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_LIKE; + return sifIndex(left, right, id, output); +} +static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NOT_LIKE; + return sifIndex(left, right, id, output); +} + +static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_MATCH; + return sifIndex(left, right, id, output); +} +static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { + int id = OP_TYPE_NMATCH; + return sifIndex(left, right, id, output); +} +static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } @@ -252,17 +325,18 @@ static sif_func_t sifGetOperFn(int32_t funcId) { return sifDefaultFunc; } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { - int32_t code = 0; - SIFParam *params = NULL; - SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); - + int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + + SIFParam *params = NULL; + SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + sif_func_t operFn = sifGetOperFn(node->opType); - return operFn(node->pLeft, node->pRight, output); + return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); _return: taosMemoryFree(params); SIF_RET(code); @@ -335,7 +409,6 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { if (ctx->code) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 7d52abcd1b..aeb5e01175 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -258,13 +258,13 @@ void indexOptsDestroy(SIndexOpts* opts) { * */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { + SIndexMultiTermQuery* mtq = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery)); + if (mtq == NULL) { return NULL; } - p->opera = opera; - p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); - return p; + mtq->opera = opera; + mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + return mtq; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { @@ -282,23 +282,24 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { - SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { + SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); + if (tm == NULL) { return NULL; } - t->suid = suid; - t->operType = oper; - t->colType = colType; + tm->suid = suid; + tm->operType = oper; + tm->colType = colType; - t->colName = (char*)taosMemoryCalloc(1, nColName + 1); - memcpy(t->colName, colName, nColName); - t->nColName = nColName; + tm->colName = (char*)taosMemoryCalloc(1, nColName + 1); + memcpy(tm->colName, colName, nColName); + tm->nColName = nColName; - t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); - memcpy(t->colVal, colVal, nColVal); - t->nColVal = nColVal; - return t; + tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); + memcpy(tm->colVal, colVal, nColVal); + tm->nColVal = nColVal; + + return tm; } void indexTermDestroy(SIndexTerm* p) { taosMemoryFree(p->colName); From c7ca57f5573bf949e38c797b07930f178cd9a482 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 00:30:47 +0800 Subject: [PATCH 36/90] feat: rollup data submit --- include/common/taosdef.h | 10 +- include/common/tdatablock.h | 3 + include/common/tmsgdef.h | 1 + source/common/src/tdatablock.c | 155 +++++++++++++++++--- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 2 + source/dnode/vnode/src/tsdb/tsdbFile.c | 17 ++- source/dnode/vnode/src/tsdb/tsdbOpen.c | 24 ++- source/dnode/vnode/src/tsdb/tsdbSma.c | 38 +++-- source/dnode/vnode/src/vnd/vnodeCfg.c | 11 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 23 ++- source/dnode/vnode/src/vnd/vnodeSvr.c | 20 ++- 14 files changed, 249 insertions(+), 61 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index dd1661f871..e1f8832edf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -79,11 +79,11 @@ typedef enum { } ETsdbSmaType; typedef enum { - TSDB_RSMA_RETENTION_0 = 0, - TSDB_RSMA_RETENTION_1 = 1, - TSDB_RSMA_RETENTION_2 = 2, - TSDB_RSMA_RETENTION_MAX = 3 -} ERSmaRetention; + TSDB_RETENTION_L0 = 0, + TSDB_RETENTION_L1 = 1, + TSDB_RETENTION_L2 = 2, + TSDB_RETENTION_MAX = 3 +} ERetentionLevel; extern char *qtypeStr[]; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 0d5262fe29..d86586a378 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -225,6 +225,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); +void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid); + static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b0439c722c..36bef5e85a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) // sync integration TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e703063dfa..e4f344b88f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -187,17 +187,17 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c int32_t i = 0; uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; - int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); - while (i < len) { // size limit of pSource->nullbitmap + int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { - start[i - 1] |= (p[i] >> remindBits); //copy remind bits + start[i - 1] |= (p[i] >> remindBits); // copy remind bits } - if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap + if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap return; } - start[i] |= (p[i] << shiftBits); //copy shift bits + start[i] |= (p[i] << shiftBits); // copy shift bits i += 1; } } @@ -453,7 +453,6 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // all fit in *stopIndex = numOfRows - 1; return TSDB_CODE_SUCCESS; - } SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) { @@ -558,7 +557,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); - char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small + char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -940,8 +939,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, - p3 - p2, p4 - p3, rows); + uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 + ", rows:%d\n", + p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -1178,7 +1178,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { - if(pDataBlock == NULL){ + if (pDataBlock == NULL) { return NULL; } @@ -1189,7 +1189,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { pBlock->info.numOfCols = numOfCols; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rows; + pBlock->info.rowSize = pDataBlock->info.rows; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; @@ -1219,7 +1219,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { } size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock)); + return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock)); } void colDataDestroy(SColumnInfoData* pColData) { @@ -1236,14 +1236,14 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); int32_t newLen = BitmapLen(total - n); - if (n%8 == 0) { - memmove(nullBitmap, nullBitmap + n/8, newLen); + if (n % 8 == 0) { + memmove(nullBitmap, nullBitmap + n / 8, newLen); } else { int32_t tail = n % 8; int32_t i = 0; - uint8_t* p = (uint8_t*) nullBitmap; - while(i < len) { + uint8_t* p = (uint8_t*)nullBitmap; + while (i < len) { uint8_t v = p[i]; p[i] = 0; @@ -1270,7 +1270,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ } } -int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { +int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { return TSDB_CODE_SUCCESS; } @@ -1278,7 +1278,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1464,3 +1464,122 @@ void blockDebugShowData(const SArray* dataBlocks) { } } +/** + * @brief TODO: Assume that the final generated result it less than 3M + * + * @param pReq + * @param pDataBlocks + * @param vgId + * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in + * SDataBlock->info.uid + * @param suid // TODO: check with Liao whether suid response is reasonable + * + * TODO: colId should be set + */ +void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, + tb_uid_t suid) { + int32_t sz = taosArrayGetSize(pDataBlocks); + int32_t bufSize = sizeof(SSubmitReq); + for (int32_t i = 0; i < sz; ++i) { + SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + bufSize += sizeof(SSubmitBlk); + } + + ASSERT(bufSize < 3 * 1024 * 1024); + + *pReq = taosMemoryCalloc(1, bufSize); + void* pDataBuf = *pReq; + + int32_t msgLen = sizeof(SSubmitReq); + int32_t numOfBlks = 0; + SRowBuilder rb = {0}; + tdSRowInit(&rb, 0); // TODO: use the latest version + + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + + if(rb.nCols != colNum) { + tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); + } + + SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); + pSubmitBlk->suid = suid; + pSubmitBlk->uid = uid; + pSubmitBlk->numOfRows = rows; + + ++numOfBlks; + + msgLen += sizeof(SSubmitBlk); + int32_t dataLen = 0; + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + printf("|"); + bool isStartKey = false; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + } else { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); + break; + } + break; + case TSDB_DATA_TYPE_NCHAR: { + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + TASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k); + } else { + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + } + break; + } + } + dataLen += TD_ROW_LEN(rb.pBuf); + } + pSubmitBlk->dataLen = dataLen; + msgLen += pSubmitBlk->dataLen; + } + + (*pReq)->length = msgLen; + + (*pReq)->header.vgId = htonl(vgId); + (*pReq)->header.contLen = htonl(msgLen); + (*pReq)->length = (*pReq)->header.contLen; + (*pReq)->numOfBlocks = htonl(numOfBlks); + SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); + while (numOfBlks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b63a7f5b32..e3816396c9 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -284,6 +284,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f4fccc2f49..62deb200c9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,7 +37,7 @@ #ifdef __cplusplus extern "C" { #endif -#define TSDB_VNODE_SMA_DEBUG // TODO: evaluate to remove the macro and the relative codes + // vnode typedef struct SVnode SVnode; typedef struct STsdbCfg STsdbCfg; // todo: remove @@ -145,7 +145,7 @@ struct STsdbCfg { int32_t keep2; // TODO: save to tsdb cfg file int8_t type; // ETsdbType - SRetention retentions[TSDB_RSMA_RETENTION_MAX]; + SRetention retentions[TSDB_RETENTION_MAX]; }; typedef enum { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 427ce8c1b6..33ffe7eda6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -72,6 +72,7 @@ struct STsdb { char *path; SVnode *pVnode; bool repoLocked; + int8_t level; // retention level TdThreadMutex mutex; STsdbMemTable *mem; STsdbMemTable *imem; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a9ba2e1de3..1c7f6034ee 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -124,6 +124,7 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); +int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other @@ -181,6 +182,7 @@ struct SVnode { struct STbUidStore { tb_uid_t suid; + tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index b1af3e0461..25566d6dad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,13 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +static const char *TSDB_DIR_NAME[] = { + "tsdb", + "rsma1", + "rsma2", +}; + +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); @@ -45,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_DIR_NAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } @@ -431,14 +437,15 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype return 0; } -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) { ASSERT(ftype != TSDB_FILE_MAX); if (ftype < TSDB_FILE_MAX) { if (ver == 0) { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid, + TSDB_FNAME_SUFFIX[ftype]); } else { - snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid, TSDB_FNAME_SUFFIX[ftype], ver); } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index da363d9bc9..e18c01dc01 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -15,21 +15,21 @@ #include "tsdb.h" -static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir); +static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level); int tsdbOpen(SVnode *pVnode, int8_t type) { switch (type) { case TSDB_TYPE_TSDB: - return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_TSMA: ASSERT(0); break; case TSDB_TYPE_RSMA_L0: - return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0); case TSDB_TYPE_RSMA_L1: - return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR, TSDB_RETENTION_L1); case TSDB_TYPE_RSMA_L2: - return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR); + return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR, TSDB_RETENTION_L2); default: ASSERT(0); break; @@ -37,7 +37,17 @@ int tsdbOpen(SVnode *pVnode, int8_t type) { return 0; } -int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { +/** + * @brief + * + * @param pVnode + * @param type + * @param ppTsdb + * @param dir + * @param level retention level + * @return int + */ +int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) { STsdb *pTsdb = NULL; int slen = 0; @@ -55,6 +65,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) { sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, dir); pTsdb->pVnode = pVnode; + pTsdb->level = level; pTsdb->repoLocked = false; taosThreadMutexInit(&pTsdb->mutex, NULL); pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb)); @@ -79,6 +90,7 @@ _err: int tsdbClose(STsdb *pTsdb) { if (pTsdb) { + // TODO: destroy mem/imem tsdbCloseFS(pTsdb); tsdbFreeFS(pTsdb->fs); taosMemoryFree(pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index d4532cc3ac..0e38deb17d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -105,7 +105,7 @@ typedef struct { #define RSMA_TASK_INFO_HASH_SLOT 8 struct SRSmaInfo { - void *taskInfo[TSDB_RSMA_RETENTION_2]; // qTaskInfo_t + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t }; struct SSmaStat { @@ -127,7 +127,7 @@ static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { } static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { - for (int32_t i = 0; i < TSDB_RSMA_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (pInfo->taskInfo[i]) { tsdbFreeTaskHandle(pInfo->taskInfo[i]); } @@ -175,7 +175,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention); + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -1976,6 +1976,7 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; @@ -1983,13 +1984,15 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { } static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention) { + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, + tb_uid_t uid, int8_t level) { SArray *pResult = NULL; - tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), retention, taskInfo, + tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, suid); + qSetStreamInput(taskInfo, pMsg, inputType); while (1) { - SSDataBlock *output; + SSDataBlock *output = NULL; uint64_t ts; if (qExecTask(taskInfo, &output, &ts) < 0) { ASSERT(false); @@ -2010,22 +2013,28 @@ static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, if (taosArrayGetSize(pResult) > 0) { blockDebugShowData(pResult); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + SSubmitReq *pReq = NULL; + buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode),uid, suid); + tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq); } else { - tsdbWarn("vgId:%d no rsma_1 data generated since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); } - + taosArrayDestroy(pResult); return TSDB_CODE_SUCCESS; } -int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) { SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + ASSERT(uid != 0); // TODO: remove later + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; @@ -2037,8 +2046,11 @@ int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_ui } if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], suid, TSDB_RSMA_RETENTION_1); - tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], suid, TSDB_RSMA_RETENTION_2); + // TODO: use the proper schema instead of 0, and cache STSchema in cache + STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1); + tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2); + taosMemoryFree(pTSchema); } return TSDB_CODE_SUCCESS; @@ -2056,12 +2068,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) { tsdbFetchSubmitReqSuids(pMsg, &uidStore); if (uidStore.suid != 0) { - tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid); void *pIter = taosHashIterate(uidStore.uidHash, NULL); while (pIter) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid); + tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0); pIter = taosHashIterate(uidStore.uidHash, pIter); } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 8130a00e04..7eac0389d2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -66,7 +66,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG if (pCfg->tsdbCfg.retentions[0].freq > 0) { int32_t nRetention = 1; if (pCfg->tsdbCfg.retentions[1].freq > 0) { @@ -87,7 +86,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeRetentions, pNodeRetention); } } -#endif if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -135,11 +133,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; -#ifdef TSDB_VNODE_SMA_DEBUG SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); - int nRetention = tjsonGetArraySize(pNodeRetentions); - ASSERT(nRetention <= TSDB_RSMA_RETENTION_MAX); - + int32_t nRetention = tjsonGetArraySize(pNodeRetentions); + if (nRetention > TSDB_RETENTION_MAX) { + nRetention = TSDB_RETENTION_MAX; + } for (int32_t i = 0; i < nRetention; ++i) { SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); ASSERT(pNodeRetention != NULL); @@ -148,7 +146,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit); } -#endif if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 7026673231..8bf68c5d29 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -47,9 +47,26 @@ int vnodeBegin(SVnode *pVnode) { } // begin tsdb - if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + if (vnodeIsRollup(pVnode)) { + if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA1(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + + if (tsdbBegin(VND_RSMA2(pVnode)) < 0) { + vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } + } else { + if (tsdbBegin(pVnode->pTsdb) < 0) { + vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index eddd295e2e..91a470b118 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -461,7 +461,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in SSubmitRsp rsp = {0}; pRsp->code = 0; - tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + // handle the request if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { pRsp->code = terrno; @@ -470,12 +470,28 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in // pRsp->msgType = TDMT_VND_SUBMIT_RSP; // vnodeProcessSubmitReq(pVnode, ptr, pRsp); - // tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); pRsp->contLen = sizeof(SSubmitRsp); + tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + return 0; } + +int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if(!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} From 5b7117884447e96964c34f12d300b4da328c956d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 09:37:24 +0800 Subject: [PATCH 37/90] feat: rollup data storage --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 8 +++++++- source/dnode/vnode/src/tsdb/tsdbFile.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbSma.c | 25 +++++++++++++++++-------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index d86586a378..3a98102661 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -225,7 +225,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); void blockDebugShowData(const SArray* dataBlocks); -void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t uid, tb_uid_t suid); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e4f344b88f..dcea167e81 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1476,7 +1476,7 @@ void blockDebugShowData(const SArray* dataBlocks) { * * TODO: colId should be set */ -void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, tb_uid_t suid) { int32_t sz = taosArrayGetSize(pDataBlocks); int32_t bufSize = sizeof(SSubmitReq); @@ -1489,6 +1489,10 @@ void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, S ASSERT(bufSize < 3 * 1024 * 1024); *pReq = taosMemoryCalloc(1, bufSize); + if(!(*pReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } void* pDataBuf = *pReq; int32_t msgLen = sizeof(SSubmitReq); @@ -1582,4 +1586,6 @@ void buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, S blk->numOfRows = htons(blk->numOfRows); blk = (SSubmitBlk*)(blk->data + dataLen); } + + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 25566d6dad..2f40698e49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static const char *TSDB_DIR_NAME[] = { +static const char *TSDB_LEVEL_DNAME[] = { "tsdb", "rsma1", "rsma2", @@ -51,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_DIR_NAME[pRepo->level], fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 0e38deb17d..362e31ff61 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -175,7 +175,8 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, int8_t level); + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -1976,7 +1977,7 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tsdbUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging + pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; @@ -1984,8 +1985,8 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { } static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType, - qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, - tb_uid_t uid, int8_t level) { + qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid, + int8_t level) { SArray *pResult = NULL; tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo, suid); @@ -2013,10 +2014,18 @@ static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, if (taosArrayGetSize(pResult) > 0) { blockDebugShowData(pResult); - STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); + STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2); SSubmitReq *pReq = NULL; - buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode),uid, suid); - tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq); + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), uid, suid) != 0) { + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; + } + if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + taosArrayDestroy(pResult); + taosMemoryFreeClear(pReq); + return TSDB_CODE_FAILED; + } + taosMemoryFreeClear(pReq); } else { tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno)); } @@ -2033,7 +2042,7 @@ static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType return TSDB_CODE_SUCCESS; } - ASSERT(uid != 0); // TODO: remove later + ASSERT(uid != 0); // TODO: remove later SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; From 9c02b1b2731bfb82ea960594d9ded8d09c62c81c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 10:44:31 +0800 Subject: [PATCH 38/90] fix(rpc): avoid invalid write of sync api --- source/libs/transport/src/transCli.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b4d23961b2..718be6aa64 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -949,9 +949,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } if (pCtx->pSem != NULL) { - tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); - memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + tTrace("%s cli conn %p(sync) handle resp", pTransInst->label, pConn); + if (pCtx->pRsp == NULL) { + tTrace("%s cli conn %p(sync) failed to resp, ignore", pTransInst->label, pConn); + } else { + memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); + } tsem_post(pCtx->pSem); + pCtx->pRsp = NULL; } else { tTrace("%s cli conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, pEpSet); From 47410b16d23f53574fcfac67bbee0b62a9ee7f35 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 03:02:34 +0000 Subject: [PATCH 39/90] refact TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 16 +++---- source/dnode/vnode/src/meta/metaTable.c | 12 ++--- source/dnode/vnode/src/tsdb/tsdbTDBImpl.c | 2 +- source/libs/tdb/inc/tdb.h | 4 +- source/libs/tdb/src/db/tdbBtree.c | 57 +++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 9 +++- source/libs/tdb/src/inc/tdbInt.h | 2 + source/libs/tdb/test/tdbTest.cpp | 4 +- 8 files changed, 87 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index bba707076c..cb556e8630 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeTbInfo(&pBuf, pTbCfg); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { pVal = pBuf = buf; metaEncodeSchemaEx(&pBuf, &schemaWrapper); vLen = POINTER_DISTANCE(pBuf, buf); - ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn); if (ret < 0) { return -1; } @@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = nameLen + 1 + sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(ctbIdxKey); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { kLen = sizeof(uid); pVal = NULL; vLen = 0; - ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn); if (ret < 0) { return -1; } @@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { int32_t kLen = sizeof(pSmaCfg->indexUid); int32_t vLen = POINTER_DISTANCE(qBuf, pBuf); - ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; @@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { val = NULL; vLen = 0; - ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); + ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn); if (ret < 0) { taosMemoryFreeClear(pBuf); return -1; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cb0b3dc81c..41ef231dcf 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -218,7 +218,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderClear(&coder); // write to table.db - if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -231,11 +231,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -258,12 +258,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -304,7 +304,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index 74878e817f..8a553e94fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 49da6fd8ed..dae085c672 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -40,7 +40,9 @@ int tdbCommit(TENV *pEnv, TXN *pTxn); int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); +int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2665dd1cf6..d711b47838 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -202,6 +202,36 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in return 0; } +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); + if (ret < 0) { + tdbBtcClose(&btc); + ASSERT(0); + return -1; + } + + if (btc.idx < 0 || c != 0) { + tdbBtcClose(&btc); + return -1; + } + + // delete the key + if (tdbBtcDelete(&btc) < 0) { + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen); } @@ -1363,6 +1393,33 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int return 0; } +int tdbBtcDelete(SBTC *pBtc) { +#if 0 + // check transaction + if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) { + return -1; + } + + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + + tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + + tdbPageDropCell(pBtc->pPage, idx); + + if (idx == nCells - 1) { + // drop cells above + for (;;) { + /* code */ + } + + // do balance + } +#endif + + return 0; +} + int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; int nCells; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index aadb9238b0..f9ef2e7901 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -75,10 +75,17 @@ int tdbDbDrop(TDB *pDb) { return 0; } -int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { +int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); } +int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } + +int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + // TODO + return 0; +} + int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 4c19dee03e..ec0ad1fa55 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -128,6 +128,7 @@ struct SBTC { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -141,6 +142,7 @@ int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); +int tdbBtcDelete(SBTC *pBtc); // tdbPager.c ==================================== diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1f6f28261f..9b4bc46ea9 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -152,7 +152,7 @@ TEST(tdb_test, simple_test) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -269,7 +269,7 @@ TEST(tdb_test, simple_test2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbDbPut(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } From 52d157718b588df2f50b0db3d13c4799b4091108 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 07:55:30 +0000 Subject: [PATCH 40/90] more tdb delte --- source/libs/tdb/src/db/tdbBtree.c | 75 +++++++++++++++++++++++++------ source/libs/tdb/src/db/tdbPage.c | 5 +++ source/libs/tdb/src/inc/tdbInt.h | 2 + 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d711b47838..1d697e0035 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1394,32 +1394,79 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int } int tdbBtcDelete(SBTC *pBtc) { -#if 0 - // check transaction - if (!TDB_TXN_IS_WRITE(pBtc->pTxn)) { + int idx = pBtc->idx; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + SPager *pPager = pBtc->pBt->pPager; + const void *pKey; + i8 iPage; + SPage *pPage; + SPgno pgno; + SCell *pCell; + int szCell; + int nKey; + int ret; + + ASSERT(idx >= 0 && idx < nCells); + + // drop the cell on the leaf + ret = tdbPagerWrite(pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); return -1; } - int idx = pBtc->idx; - int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); - - tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); - tdbPageDropCell(pBtc->pPage, idx); + // update interior page or do balance if (idx == nCells - 1) { - // drop cells above - for (;;) { - /* code */ + if (idx) { + pBtc->idx--; + tdbBtcGet(pBtc, &pKey, &nKey, NULL, NULL); + + // loop to update the interial page + pgno = TDB_PAGE_PGNO(pBtc->pPage); + for (iPage = pBtc->iPage - 1; iPage >= 0; iPage--) { + pPage = pBtc->pgStack[iPage]; + idx = pBtc->idxStack[iPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + + if (idx < nCells) { + ret = tdbPagerWrite(pPager, pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // update the cell with new key + pCell = tdbOsMalloc(nKey + 9); + tdbBtreeEncodeCell(pPage, pKey, nKey, &pgno, sizeof(pgno), pCell, &szCell); + + ret = tdbPageUpdateCell(pPage, idx, pCell, szCell); + if (ret < 0) { + tdbOsFree(pCell); + ASSERT(0); + return -1; + } + tdbOsFree(pCell); + break; + } else { + pgno = TDB_PAGE_PGNO(pPage); + } + } + } else { + // delete the leaf page and do balance (TODO) } - - // do balance } -#endif return 0; } +int tdbBtcUpsert(SBTC *pBtc) { + ASSERT(0); + // TODO + return 0; +} + int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; int nCells; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index f0fdd82944..92272fb438 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl return 0; } +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + tdbPageDropCell(pPage, idx); + return tdbPageInsertCell(pPage, idx, pCell, szCell, 0); +} + int tdbPageDropCell(SPage *pPage, int idx) { int lidx; SCell *pCell; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index ec0ad1fa55..d1a1109381 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -143,6 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc); // tdbPager.c ==================================== @@ -280,6 +281,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); int tdbPageCapacity(int pageSize, int amHdrSize); From 9d0d5295f40517fd474eea94765301e8baf187e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 10:13:45 +0000 Subject: [PATCH 41/90] more TDB delte --- source/libs/tdb/src/db/tdbBtree.c | 4 +++- source/libs/tdb/src/inc/tdbInt.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1d697e0035..887ab10ec0 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1455,13 +1455,15 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance (TODO) + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + ASSERT(0); } } return 0; } -int tdbBtcUpsert(SBTC *pBtc) { +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) { ASSERT(0); // TODO return 0; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index d1a1109381..e18952505c 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -143,7 +143,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); -int tdbBtcUpsert(SBTC *pBtc); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen); // tdbPager.c ==================================== From 4ca81bec897c5951b2d3c95cbe702259c7cdcf21 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 18:16:43 +0800 Subject: [PATCH 42/90] feat: rollup data query --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS.c | 20 ++-- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 113 ++++++++++++++--------- source/dnode/vnode/src/vnd/vnodeCommit.c | 24 ++++- 5 files changed, 105 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 33ffe7eda6..078a493773 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -186,6 +186,7 @@ struct STsdbFS { #define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->pVnode->config.tsdbCfg) +#define REPO_LEVEL(r) ((r)->level) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) #define REPO_TFS(r) ((r)->pVnode->pTfs) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4ff3227e08..6eda476b65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -15,6 +15,8 @@ #include "tsdb.h" +extern const char *TSDB_LEVEL_DNAME[]; + typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) @@ -35,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]); } -static void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]); } // For backward compatibility @@ -588,8 +590,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), - tsdbTxnFname[ftype]); + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), + TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]); } static int tsdbOpenFSFromCurrent(STsdb *pRepo) { @@ -719,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetRootDir(REPO_ID(pRepo), rootDir); + tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); @@ -753,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); const STfsFile *pf; - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); @@ -801,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { regex_t regex; STsdbFS *pfs = REPO_FS(pRepo); - tsdbGetDataDir(REPO_ID(pRepo), dataDir); + tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir); // Resource allocation and init regcomp(®ex, pattern, REG_EXTENDED); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 2f40698e49..7f024786de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -27,7 +27,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "rsma", // TSDB_FILE_RSMA }; -static const char *TSDB_LEVEL_DNAME[] = { +const char *TSDB_LEVEL_DNAME[] = { "tsdb", "rsma1", "rsma2", diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2b8558c5ac..c597c642f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -98,46 +98,46 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SColumnDataAgg *pstatis; - SColumnDataAgg **plist; - SArray *defaultLoadColumn; // default load column - int32_t *slotIds; // colId to slotId + SColumnDataAgg* pstatis; + SColumnDataAgg** plist; + SArray* defaultLoadColumn; // default load column + int32_t* slotIds; // colId to slotId } SBlockLoadSuppInfo; typedef struct STsdbReadHandle { - STsdb* pTsdb; - SQueryFilePos cur; // current position - int16_t order; - STimeWindow window; // the primary query time window that applies to all queries -// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time -// SColumnDataAgg** pstatis;// the ptr array list to return to caller - int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list - bool locateStart; - int32_t outputCapacity; - int32_t realNumOfRows; - SArray* pTableCheckInfo; // SArray - int32_t activeIndex; - bool checkFiles; // check file stage - int8_t cachelastrow; // check if last row cached - bool loadExternalRow; // load time window external data rows - bool currentLoadExternalRows; // current load external rows - int32_t loadType; // block load type - char* idStr; // query info handle, for debug purpose + STsdb* pTsdb; + SQueryFilePos cur; // current position + int16_t order; + STimeWindow window; // the primary query time window that applies to all queries + // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time + // SColumnDataAgg** pstatis;// the ptr array list to return to caller + int32_t numOfBlocks; + SArray* pColumns; // column list, SColumnInfoData array list + bool locateStart; + int32_t outputCapacity; + int32_t realNumOfRows; + SArray* pTableCheckInfo; // SArray + int32_t activeIndex; + bool checkFiles; // check file stage + int8_t cachelastrow; // check if last row cached + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + char* idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; SReadH rhelper; STableBlockInfo* pDataBlockInfo; - SDataCols* pDataCols; // in order to hold current file data block - int32_t allocSize; // allocated data block size - SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ - SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ + SDataCols* pDataCols; // in order to hold current file data block + int32_t allocSize; // allocated data block size + SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ + SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ SBlockLoadSuppInfo suppInfo; - SArray* prev; // previous row which is before than time window - SArray* next; // next row which is after the query time window - SIOCostSummary cost; - STSchema* pSchema; + SArray* prev; // previous row which is before than time window + SArray* next; // next row which is after the query time window + SIOCostSummary cost; + STSchema* pSchema; } STsdbReadHandle; typedef struct STableGroupSupporter { @@ -164,6 +164,8 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); +static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->uid = 0; @@ -350,12 +352,38 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } - +#if 0 +int nQUERY = 0; +#endif static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - // for(int32_t i=0; i< TSDB_; ) { - - // } + int level = 0; +#if 1 + int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + SRetention* pRetention = retentions + i; + if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { + break; + } + } +#endif +#if 0 + ++nQUERY; + if(nQUERY%3 == 0) { + level = 2; + } else if(nQUERY%2 == 0) { + level = 1; + } else { + level = 0; + } +#endif + if (level == TSDB_RETENTION_L0) { + return VND_RSMA0(pVnode); + } else if (level == TSDB_RETENTION_L1) { + return VND_RSMA1(pVnode); + } else { + return VND_RSMA2(pVnode); + } } return pVnode->pTsdb; } @@ -420,8 +448,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); - pReadHandle->suppInfo.slotIds = taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); - pReadHandle->suppInfo.plist = taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); + pReadHandle->suppInfo.slotIds = + taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn)); + pReadHandle->suppInfo.plist = + taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES); } pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows); @@ -444,7 +474,6 @@ _end: tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { - STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; @@ -462,16 +491,16 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG return NULL; } - STableCheckInfo *pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0); pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0); - int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); + int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn); int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData; STSchema* pSchema = pTsdbReadHandle->pSchema; int32_t i = 0, j = 0; - while(i < numOfCols && j < pSchema->numOfCols) { + while (i < numOfCols && j < pSchema->numOfCols) { if (ids[i] == pSchema->columns[j].colId) { pTsdbReadHandle->suppInfo.slotIds[i] = j; i++; @@ -1137,7 +1166,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 8bf68c5d29..6d8bcb35c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -229,10 +229,28 @@ int vnodeCommit(SVnode *pVnode) { ASSERT(0); return -1; } - if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); - return -1; + + if(vnodeIsRollup(pVnode)) { + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { + ASSERT(0); + return -1; + } + if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { + ASSERT(0); + return -1; + } + } else { + if (tsdbCommit(pVnode->pTsdb) < 0) { + ASSERT(0); + return -1; + } } + + if (tqCommit(pVnode->pTq) < 0) { ASSERT(0); return -1; From cca06e2b714f6faf54538f6921d9e10da7c26f58 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 2 May 2022 18:18:47 +0800 Subject: [PATCH 43/90] feat: rollup data query --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c597c642f7..72668a70f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -357,7 +357,7 @@ int nQUERY = 0; #endif static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { - int level = 0; + int level = 0; #if 1 int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { From 26f2d342b03b09d9d2224dc02367067add49ad6c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 22:15:51 +0800 Subject: [PATCH 44/90] enh(index): add prefix query --- include/libs/index/index.h | 4 +- source/libs/executor/src/indexoperator.c | 49 +++--- source/libs/index/src/index.c | 50 +----- source/libs/index/src/indexCache.c | 29 ++++ source/libs/index/src/indexTfile.c | 199 +++++++++++++++++------ 5 files changed, 220 insertions(+), 111 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 453b49e4c6..c33f95cb08 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -17,6 +17,7 @@ #define _TD_INDEX_H_ #include "os.h" +#include "taoserror.h" #include "tarray.h" #ifdef __cplusplus @@ -41,7 +42,8 @@ typedef enum { UPDATE_VALUE, // update index column value ADD_INDEX, // add index on specify column DROP_INDEX, // drop existed index - DROP_SATBLE // drop stable + DROP_SATBLE, // drop stable + DEFAULT // query } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 911ba22fe6..438c4254f7 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -55,11 +55,11 @@ typedef struct SIFParam { SArray *result; char * condValue; + uint8_t colValType; col_id_t colId; - - int64_t suid; // add later - char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + int64_t suid; // add later + char dbName[TSDB_DB_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; } SIFParam; typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); @@ -145,10 +145,11 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ SIF_ERR_RET(sifValidateColumn(cn)); + param->colId = cn->colId; + param->colValType = cn->node.resType.type; memcpy(param->dbName, cn->dbName, sizeof(cn->dbName)); memcpy(param->colName, cn->colName, sizeof(cn->colName)); - break; } case QUERY_NODE_NODE_LIST: { @@ -231,61 +232,68 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu qError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } -static int32_t sifIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexMultiTermQuery *mq = indexMultiTermQueryCreate(MUST); - return TSDB_CODE_SUCCESS; +static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); + if (tm == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); + indexMultiTermQueryAdd(mtm, tm, QUERY_TERM); + return indexSearch(NULL, mtm, output->result); } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_THAN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_THAN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_EQUAL; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_IN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_IN; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LIKE; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_LIKE; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_MATCH; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NMATCH; - return sifIndex(left, right, id, output); + return sifDoIndex(left, right, id, output); } static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except @@ -460,6 +468,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { qError("index-filter failed to taosHashInit"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); SIF_ERR_RET(ctx.code); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index aeb5e01175..e0c24ac3bd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -175,55 +175,19 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { -#ifdef USE_LUCENE - EIndexOperatorType opera = multiQuerys->opera; - - int nQuery = taosArrayGetSize(multiQuerys->query); - char** fields = taosMemoryMalloc(sizeof(char*) * nQuery); - char** keys = taosMemoryMalloc(sizeof(char*) * nQuery); - int* types = taosMemoryMalloc(sizeof(int) * nQuery); - - for (int i = 0; i < nQuery; i++) { - SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); - SIndexTerm* term = p->field_value; - - fields[i] = taosMemoryCalloc(1, term->nKey + 1); - keys[i] = taosMemoryCalloc(1, term->nVal + 1); - - memcpy(fields[i], term->key, term->nKey); - memcpy(keys[i], term->val, term->nVal); - types[i] = (int)(p->type); - } - int* tResult = NULL; - int tsz = 0; - index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } - - for (int i = 0; i < nQuery; i++) { - taosMemoryFree(fields[i]); - taosMemoryFree(keys[i]); - } - taosMemoryFree(fields); - taosMemoryFree(keys); - taosMemoryFree(types); -#endif - #ifdef USE_INVERTED_INDEX EIndexOperatorType opera = multiQuerys->opera; // relation of querys - SArray* interResults = taosArrayInit(4, POINTER_BYTES); + SArray* iRslts = taosArrayInit(4, POINTER_BYTES); int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); - SArray* tResult = NULL; - indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void*)&tResult); + SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i); + SArray* trslt = NULL; + indexTermSearch(index, qterm, &trslt); + taosArrayPush(iRslts, (void*)&trslt); } - indexMergeFinalResults(interResults, opera, result); - indexInterResultsDestroy(interResults); + indexMergeFinalResults(iRslts, opera, result); + indexInterResultsDestroy(iRslts); #endif return 0; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index df3c0b6e7b..eaa3c6e626 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,9 +34,38 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); + +static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchRange}; + static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); +static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} static IterateValue* indexCacheIteratorGetValue(Iterate* iter); IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 554b8092a2..7072baf574 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -57,6 +57,17 @@ static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version); static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version); static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); +/* + * search from tfile + */ +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); + +static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { + tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchRange}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -183,59 +194,153 @@ void tfileReaderDestroy(TFileReader* reader) { writerCtxDestroy(reader->ctx, reader->remove); taosMemoryFree(reader); } +static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + uint64_t offset; + if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + int64_t et = taosGetTimestampUs(); + int64_t cost = et - st; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + tem->suid, tem->colName, tem->colVal, cost); + + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + cost = taosGetTimestampUs() - et; + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + tem->colName, tem->colVal, cost); + } + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} + +static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + + int32_t ret = 0; + for (int i = 0; i < taosArrayGetSize(offsets); i++) { + uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); + ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + if (ret != 0) { + indexError("failed to find target tablelist"); + return TSDB_CODE_TDB_FILE_CORRUPTED; + } + } + if (hasJson) { + taosMemoryFree(p); + } + return 0; +} +static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} +static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + /*impl later*/ + + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} +static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + int64_t st = taosGetTimestampUs(); + FstSlice key = fstSliceCreate(p, sz); + // uint64_t offset; + // if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) { + // int64_t et = taosGetTimestampUs(); + // int64_t cost = et - st; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", + // tem->suid, tem->colName, tem->colVal, cost); + + // ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); + // cost = taosGetTimestampUs() - et; + // indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid, + // tem->colName, tem->colVal, cost); + //} + if (hasJson) { + taosMemoryFree(p); + } + fstSliceDestroy(&key); + return 0; +} int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); EIndexQueryType qtype = query->qType; - - // SArray* result = taosArrayInit(16, sizeof(uint64_t)); - int ret = -1; - // refactor to callback later - if (qtype == QUERY_TERM) { - uint64_t offset; - char* p = term->colVal; - uint64_t sz = term->nColVal; - if (hasJson) { - p = indexPackJsonData(term); - sz = strlen(p); - } - int64_t st = taosGetTimestampUs(); - FstSlice key = fstSliceCreate(p, sz); - if (fstGet(reader->fst, &key, &offset)) { - int64_t et = taosGetTimestampUs(); - int64_t cost = et - st; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us", - term->suid, term->colName, term->colVal, cost); - - ret = tfileReaderLoadTableIds(reader, offset, tr->total); - cost = taosGetTimestampUs() - et; - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", term->suid, - term->colName, term->colVal, cost); - } else { - indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, - term->colVal); - } - fstSliceDestroy(&key); - if (hasJson) { - taosMemoryFree(p); - } - } else if (qtype == QUERY_PREFIX) { - // handle later - // - } else if (qtype == QUERY_SUFFIX) { - // handle later - } else if (qtype == QUERY_REGEX) { - // handle later - } else if (qtype == QUERY_RANGE) { - // handle later + if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, + term->colVal); + return -1; + } else { + return tfSearch[qtype](reader, term, tr); } tfileReaderUnRef(reader); - - // taosArrayAddAll(tr->total, result); - // taosArrayDestroy(result); - - return ret; + return 0; } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { From f6e51532eda59619afb8f0dd0d4c85081d2791ed Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 May 2022 22:46:59 +0800 Subject: [PATCH 45/90] enh(index): add prefix query --- source/libs/index/src/indexCache.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index eaa3c6e626..2035766f26 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -47,7 +47,33 @@ static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later + if (cache == NULL) { + return 0; + } + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, ct->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + tSkipListDestroyIter(iter); return 0; } static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { From 05f06e04c1403a2e8889cfc8795ddb5a1507d2d2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 2 May 2022 15:17:43 +0000 Subject: [PATCH 46/90] refact sl --- include/util/tskiplist2.h | 159 +++++++++++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 4 +- source/util/src/tskiplist2.c | 139 +++++++++++++++++++++++++++ 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 include/util/tskiplist2.h create mode 100644 source/util/src/tskiplist2.c diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h new file mode 100644 index 0000000000..ce44ff96eb --- /dev/null +++ b/include/util/tskiplist2.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef _TD_UTIL_SKIPLIST2_H_ +#define _TD_UTIL_SKIPLIST2_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define SL_MAX_LEVEL 15 + +typedef struct SSkipList2 SSkipList2; +typedef struct SSLCursor SSLCursor; +typedef struct SSLCfg SSLCfg; + +typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); + +// SSkipList2 +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); +int32_t slClose(SSkipList2 *pSl); + +// SSLCursor +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); +int32_t slcClose(SSLCursor *pSlc); +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey); +int32_t slcMoveToNext(SSLCursor *pSlc); +int32_t slcMoveToPrev(SSLCursor *pSlc); +int32_t slcMoveToFirst(SSLCursor *pSlc); +int32_t slcMoveToLast(SSLCursor *pSlc); +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData); +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData); +int32_t slcDrop(SSLCursor *pSlc); + +// struct +struct SSLCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; + tslCmprFn cmprFn; + void *pPool; + void *(*xMalloc)(void *, int32_t size); + void (*xFree)(void *, void *); +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKIPLIST2_H_*/ + +#if 0 +#ifndef _TD_UTIL_SKILIST_H +#define _TD_UTIL_SKILIST_H + +#include "os.h" +#include "taos.h" +#include "tarray.h" +#include "tfunctional.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SKIP_LIST_LEVEL 15 +#define SKIP_LIST_RECORD_PERFORMANCE 0 + +// For key property setting +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) + +// For thread safety setting +#define SL_THREAD_SAFE (uint8_t)0x4 + +typedef char *SSkipListKey; +typedef char *(*__sl_key_fn_t)(const void *); + +typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); +typedef void *(*iter_next_fn_t)(void *iter); + +typedef struct SSkipListNode { + uint8_t level; + void *pData; + struct SSkipListNode *forwards[]; +} SSkipListNode; + +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] + +typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; + +typedef struct SSkipList { + uint32_t seed; + __compar_fn_t comparFn; + __sl_key_fn_t keyFn; + TdThreadRwlock *lock; + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element + tGenericSavedFunc *insertHandleFn; +} SSkipList; + +typedef struct SSkipListIterator { + SSkipList *pSkipList; + SSkipListNode *cur; + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator + SSkipListNode *next; // next points to the true qualified node in skiplist +} SSkipListIterator; + +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); +void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); +bool tSkipListIterNext(SSkipListIterator *iter); +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_SKILIST_H*/ + +#endif \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 9b4bc46ea9..4f20efd904 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -316,4 +316,6 @@ TEST(tdb_test, simple_test2) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); -} \ No newline at end of file +} + +TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); } \ No newline at end of file diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c new file mode 100644 index 0000000000..6657896131 --- /dev/null +++ b/source/util/src/tskiplist2.c @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tskiplist2.h" + +typedef struct SSLNode SSLNode; +struct SSLNode { + int8_t level; + SSLNode *forwards[]; +}; + +struct SSkipList2 { + int8_t level; + uint32_t seed; + int32_t size; + const SSLCfg *pCfg; +}; + +struct SSLCursor { + SSkipList2 *pSl; + SSLNode *forwards[SL_MAX_LEVEL]; +}; + +static void *slMalloc(void *pPool, int32_t size); +static void slFree(void *pPool, void *p); +static int32_t slCmprFn(const void *pKey, int32_t nKey, const void *pData, int32_t nData); + +const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, + .nKey = -1, + .nData = -1, + .cmprFn = slCmprFn, + .pPool = NULL, + .xMalloc = slMalloc, + .xFree = slFree}; + +int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { + SSkipList2 *pSl = NULL; + + *ppSl = NULL; + if (pCfg == NULL) pCfg = &slDefaultCfg; + + // check cfg (TODO) + + // create handle + pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl)); + // (TODO) + *ppSl = pSl; + return 0; +} + +int32_t slClose(SSkipList2 *pSl) { + // TODO + return 0; +} + +int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcClose(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey) { + // TODO + return 0; +} + +int32_t slcMoveToNext(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToPrev(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToFirst(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcMoveToLast(SSLCursor *pSlc) { + // TODO + return 0; +} + +int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData) { + // TODO + return 0; +} + +int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData) { + // TODO + return 0; +} + +int32_t slcDrop(SSLCursor *pSlc) { + // TODO + return 0; +} + +static FORCE_INLINE void *slMalloc(void *pPool, int32_t size) { return taosMemoryMalloc(size); } + +static FORCE_INLINE void slFree(void *pPool, void *p) { taosMemoryFree(p); } + +static int32_t slCmprFn(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2) { + ASSERT(nKey1 >= 0 && nKey2 >= 0); + + int32_t nKey = nKey1 > nKey2 ? nKey2 : nKey1; + int32_t c; + + c = memcmp(pKey1, pKey2, nKey); + if (c == 0) { + if (nKey1 > nKey2) { + c = 1; + } else if (nKey1 < nKey2) { + c = -1; + } + } + + return c; +} \ No newline at end of file From ffbf455fee2b148341a1f8b54e234cbaf1a5bde0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 May 2022 23:52:32 +0800 Subject: [PATCH 47/90] enh(query): time window interpolation for interval query. --- source/libs/executor/inc/executorInt.h | 6 + source/libs/executor/inc/executorimpl.h | 11 +- source/libs/executor/inc/tfill.h | 24 +- source/libs/executor/src/executorimpl.c | 59 ++-- source/libs/executor/src/tfill.c | 421 +++++++++++++----------- source/libs/nodes/src/nodesCodeFuncs.c | 2 +- source/libs/qworker/src/qworker.c | 3 +- 7 files changed, 295 insertions(+), 231 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 59db66becc..f2f0bc2055 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,12 @@ extern "C" { #endif +typedef struct { + char* pData; + bool isNull; + int16_t type; + int32_t bytes; +} SGroupKeys, SStateKeys; #ifdef __cplusplus } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ee8cdc6b1b..a9420cf886 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -40,6 +40,7 @@ extern "C" { #include "tpagedbuf.h" #include "vnode.h" +#include "executorInt.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -478,16 +479,8 @@ typedef struct SFillOperatorInfo { void** p; SSDataBlock* existNewGroupBlock; bool multigroupResult; - SInterval intervalInfo; } SFillOperatorInfo; -typedef struct { - char* pData; - bool isNull; - int16_t type; - int32_t bytes; -} SGroupKeys, SStateKeys; - typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; SArray* pGroupCols; // group by columns, SArray @@ -676,7 +669,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* SArray* pTableIdList, SExecTaskInfo* pTaskInfo, SNode* pConditions); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 26d066d9a9..a1f45fd665 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -27,13 +27,12 @@ extern "C" { struct SSDataBlock; typedef struct SFillColInfo { -// STColumn col; // column info - SResSchema col; - int16_t functionId; // sql function id - int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN - int16_t tagIndex; // index of current tag in SFillTagColInfo array list - int32_t offset; - union {int64_t i; double d;} val; + SExprInfo *pExpr; +// SResSchema schema; +// int16_t functionId; // sql function id + int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN + int16_t tagIndex; // index of current tag in SFillTagColInfo array list + SVariant fillVal; } SFillColInfo; typedef struct { @@ -56,9 +55,10 @@ typedef struct SFillInfo { int32_t numOfCols; // number of columns, including the tags columns int32_t rowSize; // size of each row SInterval interval; - char * prevValues; // previous row of data, to generate the interpolation results - char * nextValues; // next row of data - char** pData; // original result data block involved in filling data + + SArray *prev; + SArray *next; + SSDataBlock *pSrcBlock; int32_t alloc; // data buffer size in rows SFillColInfo* pFillCol; // column info for fill operations @@ -72,7 +72,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, @@ -80,7 +80,7 @@ SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int3 struct SFillColInfo* pCol, const char* id); void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); int64_t getFillInfoStart(struct SFillInfo *pFillInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7e9c6b6fe3..0b3343e074 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3201,16 +3201,16 @@ static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pOutput, int32_t capacity, void** p) { +int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t capacity) { // for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { // SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); // p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); // } - int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); - pOutput->info.rows += numOfRows; + int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, pBlock, capacity - pBlock->info.rows); + pBlock->info.rows += numOfRows; - return pOutput->info.rows; + return pBlock->info.rows; } void publishOperatorProfEvent(SOperatorInfo* pOperator, EQueryProfEventType eventType) { @@ -5559,7 +5559,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResult taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pResultInfo->capacity); pInfo->existNewGroupBlock = NULL; *newgroup = true; } @@ -5568,7 +5568,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf SExecTaskInfo* pTaskInfo) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity, pInfo->p); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pResultInfo->capacity); if (pInfo->pRes->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult)) { return; } @@ -5629,7 +5629,8 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { } } - doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity, pInfo->p); + blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pResBlock, pOperator->resultInfo.capacity); // current group has no more result to return if (pResBlock->info.rows > 0) { @@ -6203,19 +6204,17 @@ _error: return NULL; } -static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, int64_t* fillVal, +static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { - SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, NULL); + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); - // TODO set correct time precision STimeWindow w = TSWINDOW_INITIALIZER; - getAlignQueryTimeWindow(pInterval, TSDB_TIME_PRECISION_MILLI, win.skey, &w); + getAlignQueryTimeWindow(pInterval, pInterval->precision, win.skey, &w); int32_t order = TSDB_ORDER_ASC; pInfo->pFillInfo = taosCreateFillInfo(order, w.skey, 0, capacity, numOfCols, pInterval, fillType, pColInfo, id); pInfo->p = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pInfo->pFillInfo == NULL || pInfo->p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } else { @@ -6223,15 +6222,29 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } +static SArray* getFillValue(SNodeListNode* pNodeList) { + SArray* pList = taosArrayInit(4, sizeof(SVariant)); + + size_t len = LIST_LENGTH(pNodeList->pNodeList); + for(int32_t i = 0; i < len; ++i) { + SValueNode* pvalue = (SValueNode*)nodesListGetNode(pNodeList->pNodeList, i); + + SVariant v = {0}; + valueNodeToVariant(pvalue, &v); + taosArrayPush(pList, &v); + } + + return pList; +} + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, + SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pInfo->pRes = pResBlock; pInfo->multigroupResult = multigroupResult; - pInfo->intervalInfo = *pInterval; int32_t type = TSDB_FILL_NONE; switch (fillType) { @@ -6260,7 +6273,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SResultInfo* pResultInfo = &pOperator->resultInfo; initResultSizeInfo(pOperator, 4096); - int32_t code = initFillInfo(pInfo, pExpr, numOfCols, (int64_t*)fillVal, pTaskInfo->window, pResultInfo->capacity, + int32_t code = initFillInfo(pInfo, pExpr, numOfCols, pValueNode, *pWindow, pResultInfo->capacity, pTaskInfo->id.str, pInterval, type); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -6269,7 +6282,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp pOperator->name = "FillOperator"; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; - // pOperator->operatorType = OP_Fill; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; @@ -6618,13 +6631,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t primaryTsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, pTableGroupInfo, pTaskInfo); - - // if (pIntervalPhyNode->pFill != NULL) { - // pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, - // NULL, - // false, pTaskInfo); - // } - } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; @@ -6662,6 +6668,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &num); pOptr = createJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { + SFillPhysiNode* pFillNode = (SFillPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); + + SInterval* pInterval = &((STableIntervalOperatorInfo*)ops[0]->info)->interval; + pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); } else { ASSERT(0); } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index aeed07c636..d6f69c424d 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -13,20 +13,21 @@ * along with this program. If not, see . */ -#include "function.h" #include "os.h" -#include "querynodes.h" - #include "taosdef.h" #include "tmsg.h" #include "ttypes.h" -#include "tfill.h" -#include "function.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" +#include "function.h" +#include "tdatablock.h" +#include "executorInt.h" +#include "querynodes.h" +#include "tfill.h" + #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) @@ -37,168 +38,210 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { continue; } - char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + char* val1 = elePtrAt(data[j], pSchema->bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - -// assert (pTag->col.colId == pCol->col.colId); - assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type); + assignVal(val1, pTag->tagVal, pSchema->bytes, pSchema->type); } } -static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { +static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < numOfCol; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); - setNull(output, pCol->col.type, pCol->col.bytes); + for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); + colDataAppendNULL(p, rowIndex); } } -static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { - char* prev = pFillInfo->prevValues; - char* next = pFillInfo->nextValues; +#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) +#define GET_SRC_SLOT_ID(_p) ((_p)->pExpr->base.pParam[0].pCol->slotId) +static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); + +static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSDataBlock* pSrcBlock, int64_t ts, bool outOfBound) { SPoint point1, point2, point; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); + SColumnInfoData *pCol0 = taosArrayGet(pBlock->pDataBlock, 0); + char* val = colDataGetData(pCol0, index); + *(TSKEY*) val = pFillInfo->currentKey; // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { - char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev : pFillInfo->next; - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_NEXT) { - char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; - - if (p != NULL) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - char* output = elePtrAt(data[i], pCol->col.bytes, index); -// assignVal(output, p + pCol->offset, pCol->col.bytes, pCol->col.type); - } - } else { // no prev value yet, set the value for NULL - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - // TODO : linear interpolation supports NULL value - if (prev != NULL && !outOfBound) { - for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)) { - continue; - } - - int16_t type = pCol->col.type; - int16_t bytes = pCol->col.bytes; - - char *val1 = elePtrAt(data[i], pCol->col.bytes, index); - if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { - setNull(val1, pCol->col.type, bytes); - continue; - } - - point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->offset}; - point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; - point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); - } - } else { - setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); - } - } else { // fill the default value */ for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)) { continue; } - char* val1 = elePtrAt(data[i], pCol->col.bytes, index); - assignVal(val1, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); + } + } else if (pFillInfo->type == TSDB_FILL_NEXT) { + SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next : pFillInfo->prev; + + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + SGroupKeys* pKey = taosArrayGet(p, i); + SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); + doSetVal(pDstColInfoData, index, pKey); + } + } else if (pFillInfo->type == TSDB_FILL_LINEAR) { + // TODO : linear interpolation supports NULL value + if (outOfBound) { + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); + + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + + int16_t type = pCol->pExpr->base.resSchema.type; + SGroupKeys* pKey = taosArrayGet(pFillInfo->prev, i); + if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { + colDataAppendNULL(pDstCol, index); + continue; + } + + SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev, 0); + int64_t prevTs = *(int64_t*)pKey1->pData; + + SColumnInfoData* pSrcCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); + char* data = colDataGetData(pSrcCol, pFillInfo->index); + + point1 = (SPoint){.key = prevTs, .val = pKey->pData}; + point2 = (SPoint){.key = ts, .val = data}; + + int64_t out = 0; + point = (SPoint){.key = pFillInfo->currentKey, .val = &out}; + taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); + + colDataAppend(pDstCol, index, (const char*)&out, false); + } + } + } else if (pFillInfo->type == TSDB_FILL_NULL) { // fill with NULL + setNullRow(pBlock, pFillInfo->numOfCols, index); + } else { // fill with user specified value for each column + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { + continue; + } + + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); + if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + colDataAppend(pDst, index, (char*)&v, false); + } + +// colDataAppend(pDst, index, (char*)&pVar->i, false); } } - setTagsValue(pFillInfo, data, index); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, - pFillInfo->interval.precision); +// setTagsValue(pFillInfo, data, index); + SInterval* pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); pFillInfo->numOfCurrent++; } -static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { - if (*next != NULL) { +void doSetVal(SColumnInfoData* pDstCol, int32_t rowIndex, const SGroupKeys* pKey) { + if (pKey->isNull) { + colDataAppendNULL(pDstCol, rowIndex); + } else { + colDataAppend(pDstCol, rowIndex, pKey->pData, false); + } +} + +static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { + if (taosArrayGetSize(pFillInfo->next) > 0) { return; } - *next = taosMemoryCalloc(1, pFillInfo->rowSize); - for (int i = 1; i < pFillInfo->numOfCols; i++) { + for (int i = 0; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - setNull(*next + pCol->offset, pCol->col.type, pCol->col.bytes); + + SGroupKeys key = {0}; + SResSchema* pSchema = &pCol->pExpr->base.resSchema; + key.pData = taosMemoryMalloc(pSchema->bytes); + key.isNull = true; + key.bytes = pSchema->bytes; + key.type = pSchema->type; + + taosArrayPush(pFillInfo->next, &key); + + key.pData = taosMemoryMalloc(pSchema->bytes); + taosArrayPush(pFillInfo->prev, &key); } } -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) { - int32_t rowIndex = pFillInfo->index; +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); + +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - memcpy(buf + pCol->offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes); + int32_t srcSlotId = GET_SRC_SLOT_ID(&pFillInfo->pFillCol[i]); + + SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + bool isNull = colDataIsNull_s(pSrcCol, rowIndex); + char* p = colDataGetData(pSrcCol, rowIndex); + saveColData(pRow, i, p, isNull); } } -static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { +static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t outputRows) { pFillInfo->numOfCurrent = 0; - char** srcData = pFillInfo->pData; - char** prev = &pFillInfo->prevValues; - char** next = &pFillInfo->nextValues; + // todo make sure the first column is always the primary timestamp column? + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); + bool ascFill = FILL_IS_ASC_FILL(pFillInfo); - if (FILL_IS_ASC_FILL(pFillInfo)) { - assert(pFillInfo->currentKey >= pFillInfo->start); - } else { - assert(pFillInfo->currentKey <= pFillInfo->start); - } +#if 0 + ASSERT(ascFill && (pFillInfo->currentKey >= pFillInfo->start) || (!ascFill && (pFillInfo->currentKey <= pFillInfo->start))); +#endif while (pFillInfo->numOfCurrent < outputRows) { - int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; + int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index]; // set the next value for interpolation - if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - initBeforeAfterDataBuf(pFillInfo, next); - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); } - if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - - // fill the gap between two actual input rows - while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || - (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) && - pFillInfo->numOfCurrent < outputRows) { - doFillOneRowResult(pFillInfo, data, srcData, ts, false); + if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + // fill the gap between two input rows + while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); } // output buffer is full, abort @@ -208,61 +251,66 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR } } else { assert(pFillInfo->currentKey == ts); - initBeforeAfterDataBuf(pFillInfo, prev); + if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { - initBeforeAfterDataBuf(pFillInfo, next); ++pFillInfo->index; - copyCurrentRowIntoBuf(pFillInfo, srcData, *next); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); --pFillInfo->index; } // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } - char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); - char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); + int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); - if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) || - (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlotId); + SColumnInfoData* pSrc = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); + + char* src = colDataGetData(pSrc, pFillInfo->index); + if (i == 0 || (/*pCol->functionId != FUNCTION_COUNT &&*/ !colDataIsNull_s(pSrc, pFillInfo->index)) /*|| + (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)*/) { + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { - assignVal(output, *prev + pCol->offset, pCol->col.bytes, pCol->col.type); + SGroupKeys *pKey = taosArrayGet(pFillInfo->prev, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { - assignVal(output, src, pCol->col.bytes, pCol->col.type); - memcpy(*prev + pCol->offset, src, pCol->col.bytes); + bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); + colDataAppend(pDst, pFillInfo->numOfCurrent, src, isNull); + saveColData(pFillInfo->prev, i, src, isNull); + } else if (pFillInfo->type == TSDB_FILL_NULL) { + colDataAppendNULL(pDst, pFillInfo->numOfCurrent); } else if (pFillInfo->type == TSDB_FILL_NEXT) { - if (*next) { - assignVal(output, *next + pCol->offset, pCol->col.bytes, pCol->col.type); - } else { - setNull(output, pCol->col.type, pCol->col.bytes); - } + SGroupKeys *pKey = taosArrayGet(pFillInfo->next, i); + doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else { - assignVal(output, (char*)&pCol->val, pCol->col.bytes, pCol->col.type); + SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; + colDataAppend(pDst, pFillInfo->numOfCurrent, (char*)&pVar->i, false); } } } // set the tag value for final result - setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); +// setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); + SInterval *pInterval = &pFillInfo->interval; + pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ - if (pFillInfo->index >= pFillInfo->numOfRows) { - taosMemoryFreeClear(*next); - } - +// if (pFillInfo->index >= pFillInfo->numOfRows) { +// taosMemoryFreeClear(*next); +// } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; return pFillInfo->numOfCurrent; } @@ -271,14 +319,24 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputR return pFillInfo->numOfCurrent; } -static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { +static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull) { + SGroupKeys *pKey = taosArrayGet(rowBuf, columnIndex); + if (isNull) { + pKey->isNull = true; + } else { + memcpy(pKey->pData, src, pKey->bytes); + pKey->isNull = false; + } +} + +static int64_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. */ pFillInfo->numOfCurrent = 0; while (pFillInfo->numOfCurrent < resultCapacity) { - doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true); + doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, pFillInfo->start, true); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; @@ -295,15 +353,15 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t int32_t k = 0; for (int32_t i = 0; i < numOfCols; ++i) { SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = NULL; + SResSchema* pSchema = &pColInfo->pExpr->base.resSchema; - if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) { + if (TSDB_COL_IS_TAG(pColInfo->flag) || pSchema->type == TSDB_DATA_TYPE_BINARY) { numOfTags += 1; bool exists = false; int32_t index = -1; for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.slotId) { + if (pFillInfo->pTags[j].col.colId == pSchema->slotId) { exists = true; index = j; break; @@ -311,12 +369,12 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.slotId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; + SSchema* pSchema1 = &pFillInfo->pTags[k].col; + pSchema1->colId = pSchema->slotId; + pSchema1->type = pSchema->type; + pSchema1->bytes = pSchema->bytes; - pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pColInfo->col.bytes); + pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pSchema->bytes); pColInfo->tagIndex = k; k += 1; @@ -325,7 +383,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t } } - rowsize += pColInfo->col.bytes; + rowsize += pSchema->bytes; } pFillInfo->numOfTags = numOfTags; @@ -355,7 +413,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } taosResetFillInfo(pFillInfo, skey); - pFillInfo->order = order; switch(fillType) { @@ -364,6 +421,7 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag case FILL_MODE_NULL: pFillInfo->type = TSDB_FILL_NULL; break; case FILL_MODE_LINEAR: pFillInfo->type = TSDB_FILL_LINEAR;break; case FILL_MODE_NEXT: pFillInfo->type = TSDB_FILL_NEXT; break; + case FILL_MODE_VALUE: pFillInfo->type = TSDB_FILL_SET_VALUE; break; default: terrno = TSDB_CODE_INVALID_PARA; return NULL; @@ -376,7 +434,6 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag pFillInfo->alloc = capacity; pFillInfo->id = id; pFillInfo->interval = *pInterval; - pFillInfo->pData = taosMemoryMalloc(POINTER_BYTES * numOfCols); // if (numOfTags > 0) { pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); @@ -385,6 +442,11 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } // } + pFillInfo->next = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + pFillInfo->prev = taosArrayInit(numOfCols, sizeof(SGroupKeys)); + + initBeforeAfterDataBuf(pFillInfo); + pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); assert(pFillInfo->rowSize > 0); return pFillInfo; @@ -405,18 +467,15 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - taosMemoryFreeClear(pFillInfo->prevValues); - taosMemoryFreeClear(pFillInfo->nextValues); + taosArrayDestroy(pFillInfo->prev); + taosArrayDestroy(pFillInfo->next); for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } taosMemoryFreeClear(pFillInfo->pTags); - - taosMemoryFreeClear(pFillInfo->pData); taosMemoryFreeClear(pFillInfo->pFillCol); - taosMemoryFreeClear(pFillInfo); return NULL; } @@ -436,18 +495,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) } void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); - pFillInfo->pData[i] = pColData->pData; - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.slotId); - memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy?? - } - } + pFillInfo->pSrcBlock = (SSDataBlock*) pInput; } bool taosFillHasMoreResults(SFillInfo* pFillInfo) { @@ -465,8 +513,9 @@ bool taosFillHasMoreResults(SFillInfo* pFillInfo) { } int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; + SColumnInfoData* pCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); + int64_t* tsList = (int64_t*) pCol->pData; int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; @@ -513,7 +562,7 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* return TSDB_CODE_SUCCESS; } -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); @@ -521,9 +570,9 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t cap // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - appendFilledResult(pFillInfo, output, numOfRes); + appendFilledResult(pFillInfo, p, numOfRes); } else { - fillResultImpl(pFillInfo, output, (int32_t) numOfRes); + fillResultImpl(pFillInfo, p, (int32_t) numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); } @@ -538,28 +587,30 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { return pFillInfo->start; } -struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SValueNode* val) { - int32_t offset = 0; - - struct SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); +SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* pValNode) { + SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } + size_t len = (pValNode != NULL)? LIST_LENGTH(pValNode->pNodeList):0; for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; + SExprInfo* pExprInfo = &pExpr[i]; + pFillCol[i].pExpr = pExprInfo; + pFillCol[i].tagIndex = -2; - pFillCol[i].col = pExprInfo->base.resSchema; - pFillCol[i].offset = offset; - pFillCol[i].tagIndex = -2; + // todo refactor + if (len > 0) { + // if the user specified value is less than the column, alway use the last one as the fill value + int32_t index = (i >= len)? (len - 1):i; + + SValueNode* pv = (SValueNode*)nodesListGetNode(pValNode->pNodeList, index); + valueNodeToVariant(pv, &pFillCol[i].fillVal); + } if (pExprInfo->base.numOfParams > 0) { pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query } -// pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; -// pFillCol[i].val.d = *val; - - offset += pExprInfo->base.resSchema.bytes; } return pFillCol; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e1f9a9f6a4..6c85d9dff5 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1521,7 +1521,7 @@ static int32_t physiFillNodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToPhysiFillNode(const SJson* pJson, void* pObj) { SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; - int32_t code = jsonToPhysiWindowNode(pJson, pObj); + int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { code = tjsonGetNumberValue(pJson, jkFillPhysiPlanMode, pNode->mode); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e84f387dbe..bd7ee6321a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -945,7 +945,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); + code = TSDB_CODE_INVALID_MSG; + QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } From b2e7a161ab7b8fbe32aa001d533248a7bfe41ff1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 May 2022 23:59:59 +0800 Subject: [PATCH 48/90] refactor: remove some comments. --- source/libs/executor/src/tfill.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index d6f69c424d..018f0016a2 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -162,8 +162,6 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); colDataAppend(pDst, index, (char*)&v, false); } - -// colDataAppend(pDst, index, (char*)&pVar->i, false); } } From 80bb3d6e544c002ab21ebc9a7a3b6d8dcabb6a57 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 3 May 2022 12:57:55 +0800 Subject: [PATCH 49/90] test: add unitest for transaction --- source/dnode/mnode/impl/inc/mndUser.h | 5 +- source/dnode/mnode/impl/src/mndUser.c | 5 +- .../mnode/impl/test/trans/CMakeLists.txt | 40 +++++-- .../impl/test/trans/{trans.cpp => trans1.cpp} | 18 +-- source/dnode/mnode/impl/test/trans/trans2.cpp | 110 ++++++++++++++++++ 5 files changed, 158 insertions(+), 20 deletions(-) rename source/dnode/mnode/impl/test/trans/{trans.cpp => trans1.cpp} (93%) create mode 100644 source/dnode/mnode/impl/test/trans/trans2.cpp diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index e4ecdf3283..b3eb7f2f95 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,9 +24,12 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); +// for trans test +SSdbRaw *mndUserActionEncode(SUserObj *pUser); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2ced813003..1800d24cf2 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -25,7 +25,6 @@ #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); -static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); @@ -90,7 +89,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return 0; } -static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { +SSdbRaw *mndUserActionEncode(SUserObj *pUser) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); @@ -238,7 +237,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { diff --git a/source/dnode/mnode/impl/test/trans/CMakeLists.txt b/source/dnode/mnode/impl/test/trans/CMakeLists.txt index fa0ef9f263..3931433c19 100644 --- a/source/dnode/mnode/impl/test/trans/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/trans/CMakeLists.txt @@ -1,11 +1,37 @@ -aux_source_directory(. MNODE_TRANS_TEST_SRC) -add_executable(transTest ${MNODE_TRANS_TEST_SRC}) +add_executable(transTest1 "") +target_sources(transTest1 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans1.cpp" +) target_link_libraries( - transTest + transTest1 PUBLIC sut ) - -add_test( - NAME transTest - COMMAND transTest +target_include_directories( + transTest1 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest1 + COMMAND transTest1 +) + +add_executable(transTest2 "") +target_sources(transTest2 + PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/trans2.cpp" +) +target_link_libraries( + transTest2 + PUBLIC dnode mnode gtest_main +) +target_include_directories( + transTest2 + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) +add_test( + NAME transTest2 + COMMAND transTest2 ) diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp similarity index 93% rename from source/dnode/mnode/impl/test/trans/trans.cpp rename to source/dnode/mnode/impl/test/trans/trans1.cpp index bcf6fe8536..07bf9a2bcf 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -11,10 +11,10 @@ #include "sut.h" -class MndTestTrans : public ::testing::Test { +class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans", 9013); + test.Init("/tmp/mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); @@ -26,7 +26,7 @@ class MndTestTrans : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; + char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); @@ -60,10 +60,10 @@ class MndTestTrans : public ::testing::Test { void TearDown() override {} }; -Testbase MndTestTrans::test; -TestServer MndTestTrans::server2; +Testbase MndTestTrans1::test; +TestServer MndTestTrans1::server2; -TEST_F(MndTestTrans, 00_Create_User_Crash) { +TEST_F(MndTestTrans1, 00_Create_User_Crash) { { test.SendShowReq(TSDB_MGMT_TABLE_TRANS, "trans", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -83,7 +83,7 @@ TEST_F(MndTestTrans, 00_Create_User_Crash) { } } -TEST_F(MndTestTrans, 01_Create_User_Crash) { +TEST_F(MndTestTrans1, 01_Create_User_Crash) { { SCreateUserReq createReq = {0}; strcpy(createReq.user, "u1"); @@ -107,7 +107,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { EXPECT_EQ(test.GetShowRows(), 2); } -TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { +TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; @@ -142,7 +142,7 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { } } -TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { +TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { { SCreateDnodeReq createReq = {0}; strcpy(createReq.fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp new file mode 100644 index 0000000000..ef80aa14eb --- /dev/null +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -0,0 +1,110 @@ +/** + * @file trans.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module trans tests + * @version 1.0 + * @date 2022-05-02 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "mndTrans.h" +#include "mndUser.h" +#include "tcache.h" + +void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {} + +class MndTestTrans2 : public ::testing::Test { + protected: + static void SetUpTestSuite() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 135; + uDebugFlag = 135; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + const char *logpath = "/tmp/td"; + taosRemoveDir(logpath); + taosMkDir(logpath); + tstrncpy(tsLogDir, logpath, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file\n"); + } + + walInit(); + + static SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefaultMsgCb(&msgCb); + + SMnodeOpt opt = {0}; + opt.deploy = 1; + opt.replica = 1; + opt.replicas[0].id = 1; + opt.replicas[0].port = 9040; + strcpy(opt.replicas[0].fqdn, "localhost"); + opt.msgCb = msgCb; + const char *mnodepath = "/tmp/mnode_test_trans"; + pMnode = mndOpen(mnodepath, &opt); + } + + static void TearDownTestSuite() { + mndClose(pMnode); + walCleanUp(); + taosCloseLog(); + taosStopCacheRefreshWorker(); + } + + static SMnode *pMnode; + + public: + void SetUp() override {} + void TearDown() override {} + + void CreateUser(const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, "root", TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + char *param = strdup("====> test param <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + } +}; + +SMnode *MndTestTrans2::pMnode; + +TEST_F(MndTestTrans2, 01_CbFunc) { + ASSERT_NE(pMnode, nullptr); + + const char *user1 = "test1"; + CreateUser(user1); + + SUserObj *pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); +} From 0799a5eaf7e4e0a7924ce14e60426c64b202e5a3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 14:43:53 +0800 Subject: [PATCH 50/90] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +- source/libs/executor/inc/executorimpl.h | 14 + source/libs/executor/src/executorimpl.c | 1672 +---------------- source/libs/executor/src/timewindowoperator.c | 1488 +++++++++++++++ 4 files changed, 1514 insertions(+), 1663 deletions(-) create mode 100644 source/libs/executor/src/timewindowoperator.c diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2b8558c5ac..dc27087b5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -155,7 +155,6 @@ static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); @@ -1308,6 +1307,8 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* return code; } +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); + static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pTsdbReadHandle->cur; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a9420cf886..a49c3984aa 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -606,6 +606,9 @@ typedef struct SJoinOperatorInfo { SNode *pOnCondition; } SJoinOperatorInfo; +#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) +#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) + SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, __optr_decode_fn_t decode, __optr_get_explain_fn_t explain); @@ -635,6 +638,16 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); +void cleanupAggSup(SAggSupporter* pAggSup); +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); + +void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowCellInfoOffset); + +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, + char* pData, int16_t bytes, bool masterscan, uint64_t groupId, + SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -656,6 +669,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0b3343e074..4860e10c19 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -52,11 +52,6 @@ enum { TS_JOIN_TAG_NOT_EQUALS = 2, }; -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -95,48 +90,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -#define TSKEY_MAX_ADD(a, b) \ - do { \ - if (a < 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) > 0 && ((b) >= INT32_MAX - (a))) { \ - a = INT32_MAX; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) > 0 && ((b) >= INT64_MAX - (a))) { \ - a = INT64_MAX; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - -#define TSKEY_MIN_SUB(a, b) \ - do { \ - if (a >= 0) { \ - a = a + b; \ - break; \ - } \ - if (sizeof(a) == sizeof(int32_t)) { \ - if ((b) < 0 && ((b) <= INT32_MIN - (a))) { \ - a = INT32_MIN; \ - } else { \ - a = a + b; \ - } \ - } else { \ - if ((b) < 0 && ((b) <= INT64_MIN - (a))) { \ - a = INT64_MIN; \ - } else { \ - a = a + b; \ - } \ - } \ - } while (0) - int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo* pExprInfo) { @@ -144,39 +97,6 @@ static int32_t getExprFunctionId(SExprInfo* pExprInfo) { return 0; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); static bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -188,19 +108,13 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo* pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); -static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -// static SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); -static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); -static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput); @@ -216,9 +130,6 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { } } -#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) -#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) - int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; @@ -438,9 +349,9 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, * | 8 bytes | actual length | * +----------+---------------+ */ -static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = @@ -497,61 +408,6 @@ static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowI return pResult; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, - bool ascQuery) { - if (ascQuery) { - getAlignQueryTimeWindow(pInterval, precision, ts, w); - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pInterval, precision, ts, w); - - int64_t key = w->skey; - while (key < ts) { // moving towards end - key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); - if (key >= ts) { - break; - } - - w->skey = key; - } - } -} - -// get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win) { - STimeWindow w = {0}; - - if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pInterval, precision, ts, &w, true); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; - } - - if (w.skey > ts || w.ekey < ts) { - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - int64_t st = w.skey; - - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } - } - return w; -} - // get the correct time window according to the handled timestamp static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo* pResultRowInfo, int64_t ts, STaskAttr* pQueryAttr) { STimeWindow w = {0}; @@ -636,75 +492,6 @@ static bool chkWindowOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset); - -static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan, - SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); - SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, id, (char*)&win->skey, - TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); - - if (pResultRow == NULL) { - *pResult = NULL; - return TSDB_CODE_SUCCESS; - } - - // set time window for current result - pResultRow->win = (*win); - *pResult = pResultRow; - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); - return TSDB_CODE_SUCCESS; -} - -static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - pResult->startInterp = true; - } else { - pResult->endInterp = true; - } -} - -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); - if (type == RESULT_ROW_START_INTERP) { - return pResult->startInterp == true; - } else { - return pResult->endInterp == true; - } -} - -static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, - int16_t pos, int16_t order, int64_t* pData) { - int32_t forwardStep = 0; - - if (order == TSDB_ORDER_ASC) { - int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); - if (end >= 0) { - forwardStep = end; - - if (pData[end + pos] == ekey) { - forwardStep += 1; - } - } - } else { - int32_t end = searchFn((char*)pData, pos + 1, ekey, order); - if (end >= 0) { - forwardStep = pos - end; - - if (pData[end] == ekey) { - forwardStep += 1; - } - } - } - - assert(forwardStep >= 0); - return forwardStep; -} - static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; @@ -774,46 +561,8 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey // } //} -static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, - TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - - int32_t num = -1; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (order == TSDB_ORDER_ASC) { - if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; - } - } else { - num = pDataBlockInfo->rows - startPos; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.ekey + step; - } - } - } else { // desc - if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { - num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (item != NULL) { - item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; - } - } else { - num = startPos + 1; - if (item != NULL) { - item->lastKey = pDataBlockInfo->window.skey + step; - } - } - } - - assert(num >= 0); - return num; -} - // query_range_start, query_range_end, window_duration, window_start, window_end -static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); @@ -827,16 +576,6 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { - int64_t* ts = (int64_t*)pColData->pData; - int32_t delta = includeEndpoint ? 1 : 0; - - int64_t duration = pWin->ekey - pWin->skey + delta; - ts[2] = duration; // set the duration - ts[3] = pWin->skey; // window start key - ts[4] = pWin->ekey + delta; // window end key -} - void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { @@ -888,77 +627,6 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* } } -static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { - int32_t order = pInfo->order; - bool ascQuery = (order == TSDB_ORDER_ASC); - - int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); - - // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || - (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { - return -1; - } - - TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; - int32_t startPos = 0; - - // tumbling time window query, a special case of sliding time window query - if (pInterval->sliding == pInterval->interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - startPos = prevPosition + factor; - } else { - if (startKey <= pDataBlockInfo->window.skey && ascQuery) { - startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { - startPos = pDataBlockInfo->rows - 1; - } else { - startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); - } - } - - /* interp query with fill should not skip time window */ - // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - // return startPos; - // } - - /* - * This time window does not cover any data, try next time window, - * this case may happen when the time window is too small - */ - if (primaryKeys == NULL) { - if (ascQuery) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); - } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); - } - } else { - if (ascQuery && primaryKeys[startPos] > pNext->ekey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->skey = pNext->ekey - pInterval->interval + 1; - } - } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { - TSKEY next = primaryKeys[startPos]; - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); - pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - pNext->ekey = pNext->skey + pInterval->interval - 1; - } - } - } - - return startPos; -} - static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* pWindow) { TSKEY ekey = -1; int32_t order = TSDB_ORDER_ASC; @@ -977,41 +645,6 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* p return ekey; } -static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { - if (type == RESULT_ROW_START_INTERP) { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].start.key = INT64_MIN; - } - } else { - for (int32_t k = 0; k < numOfOutput; ++k) { - pCtx[k].end.key = INT64_MIN; - } - } -} - -static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { - if (pDataBlock == NULL) { - return; - } - - for (int32_t k = 0; k < numOfCols; ++k) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); - } -} - -static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { - TSKEY ts = TSKEY_INITIAL_VAL; - if (tsCols == NULL) { - ts = ascQuery ? win->skey : win->ekey; - } else { - int32_t offset = ascQuery ? 0 : rows - 1; - ts = tsCols[offset]; - } - - return ts; -} - static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); @@ -1280,412 +913,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc return TSDB_CODE_SUCCESS; } -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, - int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; - - SqlFunctionCtx* pCtx = pInfo->pCtx; - - for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { - pCtx[k].start.key = INT64_MIN; - continue; - } - - SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); - - // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - if (functionId == FUNCTION_INTERP) { - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = prevTs; - pCtx[k].start.val = v1; - - pCtx[k].end.key = curTs; - pCtx[k].end.val = v2; - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - if (prevRowIndex == -1) { - // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; - } else { - pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; - } - - pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; - } - } - } else if (functionId == FUNCTION_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; - - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } - } - } -} - -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - bool ascQuery = true; - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; - - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = ascQuery ? win->skey : win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); - return true; -} - -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, - STimeWindow* win) { - int32_t order = TSDB_ORDER_ASC; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - TSKEY actualEndKey = tsCols[endRowIndex]; - TSKEY key = order ? win->ekey : win->skey; - - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || - (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return false; - } - - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); - - TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); - return true; -} - -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, - int32_t order, bool timeWindowInterpo) { - if (!timeWindowInterpo) { - return; - } - - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - - if (pBlock->pDataBlock == NULL) { - // tscError("pBlock->pDataBlock == NULL"); - return; - } - - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - - TSKEY* tsCols = (TSKEY*)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - } - - // point interpolation does not require the end key time window interpolation. - // if (pointInterpQuery) { - // return; - // } - - // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; - bool interp = - setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); - } -} - -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; - - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - - SArray* pUpdated = NULL; - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, POINTER_BYTES); - } - - int32_t step = 1; - bool ascScan = true; - - // int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } - - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, &pInfo->win); - bool masterScan = true; - - SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - int32_t forwardStep = 0; - TSKEY ekey = win.ekey; - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // prev time window not interpolation yet. - // int32_t curIndex = pResultRowInfo->curPos; - -#if 0 - if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - // restore current time window - ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } -#endif - - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); - if (startPos < 0) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; - - taosArrayPush(pUpdated, &pos); - } - - ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - - // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } - - if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); - } - - return pUpdated; - // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); -} - -static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { - pRowSup->win.ekey = ts; - pRowSup->prevTs = ts; - pRowSup->numOfRows += 1; -} - -static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { - pRowSup->startRowIndex = rowIndex; - pRowSup->numOfRows = 0; - pRowSup->win.skey = tsList[rowIndex]; -} - -// todo handle multiple tables cases. -static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - // todo find the correct time stamp column slot - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - int64_t gid = pBlock->info.groupId; - - int64_t gap = pInfo->gap; - - if (!pInfo->reptScan) { - pInfo->reptScan = true; - pInfo->winSup.prevTs = INT64_MIN; - } - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (pInfo->winSup.prevTs == INT64_MIN) { - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { - // The gap is less than the threshold, so it belongs to current session window that has been opened already. - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // start a new session window - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - // pInfo->numOfRows data belong to the current session window - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { // todo disable this @@ -1710,8 +937,8 @@ int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = binfo->pCtx; - SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId, - pTaskInfo, false, pAggSup); + SResultRow* pResultRow = + doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); assert(pResultRow != NULL); setResultRowKey(pResultRow, pData, type); @@ -2496,69 +1723,6 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc return TSDB_CODE_SUCCESS; } -int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { - int32_t midPos = -1; - int32_t numOfRows; - - if (num <= 0) { - return -1; - } - - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY*)pValue; - int32_t firstPos = 0; - int32_t lastPos = num - 1; - - if (order == TSDB_ORDER_DESC) { - // find the first position which is smaller than the key - while (1) { - if (key >= keyList[lastPos]) return lastPos; - if (key == keyList[firstPos]) return firstPos; - if (key < keyList[firstPos]) return firstPos - 1; - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - - } else { - // find the first position which is bigger than the key - while (1) { - if (key <= keyList[firstPos]) return firstPos; - if (key == keyList[lastPos]) return lastPos; - - if (key > keyList[lastPos]) { - lastPos = lastPos + 1; - if (lastPos >= num) - return -1; - else - return lastPos; - } - - numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1u) + firstPos; - - if (key < keyList[midPos]) { - lastPos = midPos - 1; - } else if (key > keyList[midPos]) { - firstPos = midPos + 1; - } else { - break; - } - } - } - - return midPos; -} - /* * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer @@ -2726,8 +1890,8 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t int64_t tid = 0; int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, tid, (char*)&tid, sizeof(tid), true, - groupId, pTaskInfo, false, pSup); + SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, + pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); @@ -2868,39 +2032,6 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } -// todo merged with the build group result. -void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, - int32_t* rowCellInfoOffset) { - size_t num = taosArrayGetSize(pUpdateList); - - for (int32_t i = 0; i < num; ++i) { - SResKeyPos* pPos = taosArrayGetP(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); - // - for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - // - struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - // if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - // continue; - // } - // - // if (pCtx[j].fpSet.process) { // TODO set the dummy function. - //// pCtx[j].fpSet.finalize(&pCtx[j]); - // pResInfo->initialized = true; - // } - // - if (pRow->numOfRows < pResInfo->numOfRes) { - pRow->numOfRows = pResInfo->numOfRes; - } - } - - releaseBufPage(pBuf, bufPage); - } -} - STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; @@ -3013,7 +2144,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx; int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, + SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); assert(pResultRow != NULL); @@ -3043,47 +2174,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -/* - * There are two cases to handle: - * - * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including - * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. - * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be - * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there - * is a previous result generated or not. - */ -void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { - // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - // if (pResultRowInfo->curPos != -1) { - // return; - // } - - // pTableQueryInfo->win.skey = key; - // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - // STimeWindow w = TSWINDOW_INITIALIZER; - // - // TSKEY sk = TMIN(win.skey, win.ekey); - // TSKEY ek = TMAX(win.skey, win.ekey); - // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - - // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { - // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - // assert(win.ekey == pQueryAttr->window.ekey); - // } - // - // pResultRowInfo->prevSKey = w.skey; - // } - - // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; -} - /** * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, @@ -3624,30 +2714,6 @@ static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQuer #endif } -// SQueryTableDataCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { -// SQueryTableDataCond cond = { -// .colList = pQueryAttr->tableCols, -// .order = pQueryAttr->order.order, -// .numOfCols = pQueryAttr->numOfCols, -// .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, -// .loadExternalRows = false, -// }; -// -// TIME_WINDOW_COPY(cond.twindow, *win); -// return cond; -// } - -static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { - STableIdInfo tidInfo; - // STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - // - // tidInfo.uid = id->uid; - // tidInfo.tid = id->tid; - // tidInfo.key = pTableQueryInfo->lastKey; - - return tidInfo; -} - // static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t // order) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); @@ -4274,7 +3340,6 @@ _error: static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); -static void cleanupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*)param; @@ -4289,21 +3354,6 @@ static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); } -static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - dst->pExpr = exprdup(src->pExpr); - dst->base.pParam = taosMemoryCalloc(src->base.numOfParams, sizeof(SColumn)); - memcpy(dst->base.pParam, src->base.pParam, sizeof(SColumn) * src->base.numOfParams); - - // memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); - // for (int32_t j = 0; j < src->base.numOfParams; ++j) { - // taosVariantAssign(&dst->base.param[j], &src->base.param[j]); - // } -} - // TODO merge aggregate super table static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -5124,430 +4174,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) return (pInfo->pRes->info.rows > 0) ? pInfo->pRes : NULL; } -static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableIntervalOperatorInfo* pInfo = pOperator->info; - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = {0}; - bool newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); - -#if 0 // test for encode/decode result info - if(pOperator->encodeResultRow){ - char *result = NULL; - int32_t length = 0; - SAggSupporter *pSup = &pInfo->aggSup; - pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); - taosHashClear(pSup->pResultRowHashTable); - pInfo->binfo.resultRowInfo.size = 0; - pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); - if(result){ - taosMemoryFree(result); - } - } -#endif - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - OPTR_SET_OPENED(pOperator); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSDataBlock* pBlock = pInfo->binfo.pRes; - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->fpSet.getStreamResFn(pOperator, newgroup); - } else { - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBlock->info.rows == 0 ? NULL : pBlock; - } -} - -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; - int32_t order = TSDB_ORDER_ASC; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - // STimeWindow win = {0}; - *newgroup = false; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - SArray* pUpdated = NULL; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the - // caller. Note that all the time window are not close till now. - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); - pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); - } - - finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); - - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - ASSERT(pInfo->binfo.pRes->info.rows > 0); - pOperator->status = OP_RES_TO_RETURN; - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STimeSliceOperatorInfo* pSliceInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pSliceInfo->binfo.pRes; - } - - int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = pQueryAttr->window; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); - // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); - } - - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - - // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); - // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); - - if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - } - - return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; -} - -static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - STableIntervalOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - int64_t st = taosGetTimestampUs(); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC, true); - STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; - - setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - // hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pBlock->info.groupId); - } - - closeAllResultRows(&pInfo->binfo.resultRowInfo); - finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, - &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - - // initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); - OPTR_SET_OPENED(pOperator); - - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); - - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - -static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - int64_t gid = pBlock->info.groupId; - - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfOutput; - - int16_t bytes = pStateColInfoData->info.bytes; - int16_t type = pStateColInfoData->info.type; - - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - TSKEY* tsList = (TSKEY*)pColInfoData->pData; - - SWindowRowsSup* pRowSup = &pInfo->winSup; - pRowSup->numOfRows = 0; - - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { - continue; - } - - char* val = colDataGetData(pStateColInfoData, j); - - if (!pInfo->hasKey) { - memcpy(pInfo->stateKey.pData, val, bytes); - pInfo->hasKey = true; - - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { - doKeepTuple(pRowSup, tsList[j]); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } - } else { // a new state window started - SResultRow* pResult = NULL; - - // keep the time window for the closed time window. - STimeWindow window = pRowSup->win; - - pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - - // here we start a new session window - doKeepNewWindowStartInfo(pRowSup, tsList, j); - doKeepTuple(pRowSup, tsList[j]); - } - } - - SResultRow* pResult = NULL; - pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, - pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); - } - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, - pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); -} - -static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - return pBInfo->pRes; - } - - int32_t order = TSDB_ORDER_ASC; - STimeWindow win = pTaskInfo->window; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - - if (pBlock == NULL) { - break; - } - - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doStateWindowAggImpl(pOperator, pInfo, pBlock); - } - - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} - -static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - - return pBInfo->pRes; - } - - int32_t order = TSDB_ORDER_ASC; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); - publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); - if (pBlock == NULL) { - break; - } - - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); - doSessionWindowAggImpl(pOperator, pInfo, pBlock); - } - - // restore the value - pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, - pBInfo->rowCellInfoOffset); - - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); - if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); - } - - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} - static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, bool* newgroup, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; @@ -5713,11 +4339,9 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n return TSDB_CODE_SUCCESS; } -static void cleanupAggSup(SAggSupporter* pAggSup) { +void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); - // taosHashCleanup(pAggSup->pResultRowListSet); - // taosArrayDestroy(pAggSup->pResultRowArrayList); destroyDiskbasedBuf(pAggSup->pResultBuf); } @@ -5847,28 +4471,11 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(pInfo, numOfOutput); } -void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { - SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - taosMemoryFreeClear(pInfo->stateKey.pData); -} - void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); - cleanupAggSup(&pInfo->aggSup); -} - -void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { - SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; - doDestroyBasicInfo(&pInfo->binfo, numOfOutput); -} - void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); @@ -5960,250 +4567,6 @@ _error: return NULL; } -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, - destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, - SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; - pInfo->primaryTsIndex = primaryTsSlotId; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - - // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); - if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - - pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, - destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - destroyIntervalOperatorInfo(pInfo, numOfCols); - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { - STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pOperator == NULL || pInfo == NULL) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pOperator->name = "TimeSliceOperator"; - // pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, - NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, - SExecTaskInfo* pTaskInfo) { - SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - pInfo->colIndex = -1; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, 4096); - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - - pInfo->twAggSup = *pTwAggSup; - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, - destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - pTaskInfo->code = TSDB_CODE_SUCCESS; - return NULL; -} - -SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, - SExecTaskInfo* pTaskInfo) { - SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - initResultSizeInfo(pOperator, numOfRows); - int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->twAggSup = *pTwAggSupp; - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->winSup.prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; - pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, - destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - pOperator->pTaskInfo = pTaskInfo; - - code = appendDownstream(pOperator, &downstream, 1); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySWindowOperatorInfo(pInfo, numOfCols); - } - - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); @@ -6222,21 +4585,6 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } -static SArray* getFillValue(SNodeListNode* pNodeList) { - SArray* pList = taosArrayInit(4, sizeof(SVariant)); - - size_t len = LIST_LENGTH(pNodeList->pNodeList); - for(int32_t i = 0; i < len; ++i) { - SValueNode* pvalue = (SValueNode*)nodesListGetNode(pNodeList->pNodeList, i); - - SVariant v = {0}; - valueNodeToVariant(pvalue, &v); - taosArrayPush(pList, &v); - } - - return pList; -} - SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* pValueNode, bool multigroupResult, SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c new file mode 100644 index 0000000000..cc09c05e1c --- /dev/null +++ b/source/libs/executor/src/timewindowoperator.c @@ -0,0 +1,1488 @@ +#include "ttime.h" +#include "tdatablock.h" +#include "executorimpl.h" + +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including + * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { + // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + // if (pResultRowInfo->curPos != -1) { + // return; + // } + + // pTableQueryInfo->win.skey = key; + // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + // STimeWindow w = TSWINDOW_INITIALIZER; + // + // TSKEY sk = TMIN(win.skey, win.ekey); + // TSKEY ek = TMAX(win.skey, win.ekey); + // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); + + // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + // assert(win.ekey == pQueryAttr->window.ekey); + // } + // + // pResultRowInfo->prevSKey = w.skey; + // } + + // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +} + +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { + TSKEY ts = TSKEY_INITIAL_VAL; + if (tsCols == NULL) { + ts = ascQuery ? win->skey : win->ekey; + } else { + int32_t offset = ascQuery ? 0 : rows - 1; + ts = tsCols[offset]; + } + + return ts; +} + +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, + bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, precision, ts, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pInterval, precision, ts, w); + + int64_t key = w->skey; + while (key < ts) { // moving towards end + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, + SInterval* pInterval, int32_t precision, STimeWindow* win) { + STimeWindow w = {0}; + + if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pInterval, precision, ts, &w, true); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + } + + if (w.skey > ts || w.ekey < ts) { + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + } + return w; +} + +static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan, + SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + assert(win->skey <= win->ekey); + SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, + masterscan, tableGroupId, pTaskInfo, true, pAggSup); + + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + return TSDB_CODE_SUCCESS; +} + +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { + int64_t* ts = (int64_t*)pColData->pData; + int32_t delta = includeEndpoint ? 1 : 0; + + int64_t duration = pWin->ekey - pWin->skey + delta; + ts[2] = duration; // set the duration + ts[3] = pWin->skey; // window start key + ts[4] = pWin->ekey + delta; // window end key +} + +static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { + pRowSup->win.ekey = ts; + pRowSup->prevTs = ts; + pRowSup->numOfRows += 1; +} + +static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { + pRowSup->startRowIndex = rowIndex; + pRowSup->numOfRows = 0; + pRowSup->win.skey = tsList[rowIndex]; +} + +static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, + int16_t pos, int16_t order, int64_t* pData) { + int32_t forwardStep = 0; + + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; + + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } + } + } + + assert(forwardStep >= 0); + return forwardStep; +} + +static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + if (num <= 0) { + return -1; + } + + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY*)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, + TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, + int32_t order) { + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + + int32_t num = -1; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (order == TSDB_ORDER_ASC) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } else { + num = pDataBlockInfo->rows - startPos; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (item != NULL) { + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } else { + num = startPos + 1; + if (item != NULL) { + item->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = tw->skey + pInterval->interval - 1; + return; + } + + int64_t key = tw->skey, interval = pInterval->interval; + // convert key to second + key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + if (pInterval->intervalUnit == 'y') { + interval *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + mon = (int)(mon + interval); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + + tw->ekey -= 1; +} + +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SExprInfo* pExpr = pOperator->pExpr; + + SqlFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; + int16_t index = pColIndex->colIndex; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + + // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == FUNCTION_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (prevRowIndex == -1) { + // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; + } else { + pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; + } + + pCtx[k].end.ptr = (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes; + } + } + } else if (functionId == FUNCTION_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } + } + } +} + +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, + STimeWindow* win) { + bool ascQuery = true; + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery ? win->skey : win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, + RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, + SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, + STimeWindow* win) { + int32_t order = TSDB_ORDER_ASC; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + TSKEY actualEndKey = tsCols[endRowIndex]; + TSKEY key = order ? win->ekey : win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || + (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { + int32_t order = pInfo->order; + bool ascQuery = (order == TSDB_ORDER_ASC); + + int32_t precision = pInterval->precision; + getNextTimeWindow(pInterval, precision, order, pNext); + + // next time window is not in current block + if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || + (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { + return -1; + } + + TSKEY startKey = ascQuery ? pNext->skey : pNext->ekey; + int32_t startPos = 0; + + // tumbling time window query, a special case of sliding time window query + if (pInterval->sliding == pInterval->interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + startPos = prevPosition + factor; + } else { + if (startKey <= pDataBlockInfo->window.skey && ascQuery) { + startPos = 0; + } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { + startPos = pDataBlockInfo->rows - 1; + } else { + startPos = binarySearchForKey((char*)primaryKeys, pDataBlockInfo->rows, startKey, order); + } + } + + /* interp query with fill should not skip time window */ + // if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { + // return startPos; + // } + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (primaryKeys == NULL) { + if (ascQuery) { + assert(pDataBlockInfo->window.skey <= pNext->ekey); + } else { + assert(pDataBlockInfo->window.ekey >= pNext->skey); + } + } else { + if (ascQuery && primaryKeys[startPos] > pNext->ekey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->skey = pNext->ekey - pInterval->interval + 1; + } + } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { + TSKEY next = primaryKeys[startPos]; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->ekey = pNext->skey + pInterval->interval - 1; + } + } + } + + return startPos; +} + +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, + int32_t order, bool timeWindowInterpo) { + if (!timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (pBlock->pDataBlock == NULL) { + // tscError("pBlock->pDataBlock == NULL"); + return; + } + + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* tsCols = (TSKEY*)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, + pBlock->pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + // if (pointInterpQuery) { + // return; + // } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + bool interp = + setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); + } +} + +static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { + if (pDataBlock == NULL) { + return; + } + + for (int32_t k = 0; k < numOfCols; ++k) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} + +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + SArray* pUpdated = NULL; + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + pUpdated = taosArrayInit(4, POINTER_BYTES); + } + + int32_t step = 1; + bool ascScan = true; + + // int32_t prevIndex = pResultRowInfo->curPos; + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } + + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, &pInfo->win); + bool masterScan = true; + SResultRow* pResult = NULL; + + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + int32_t forwardStep = 0; + TSKEY ekey = win.ekey; + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // prev time window not interpolation yet. + // int32_t curIndex = pResultRowInfo->curPos; + +#if 0 + if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = getResultRow(pResultRowInfo, j); + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + // restore current time window + ret = setTimeWindowOutputBuf(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, + pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } +#endif + + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + + taosArrayPush(pUpdated, &pos); + } + + ekey = nextWin.ekey; // reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + pInfo->order, false); + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + } + + if (pInfo->timeWindowInterpo) { + int32_t rowIndex = ascScan ? (pSDataBlock->info.rows - 1) : 0; + saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); + } + + return pUpdated; + // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); +} + +static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableIntervalOperatorInfo* pInfo = pOperator->info; + + int32_t order = TSDB_ORDER_ASC; + // STimeWindow win = {0}; + bool newgroup = false; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; + + setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); + +#if 0 // test for encode/decode result info + if(pOperator->encodeResultRow){ + char *result = NULL; + int32_t length = 0; + SAggSupporter *pSup = &pInfo->aggSup; + pOperator->encodeResultRow(pOperator, pSup, &pInfo->binfo, &result, &length); + taosHashClear(pSup->pResultRowHashTable); + pInfo->binfo.resultRowInfo.size = 0; + pOperator->decodeResultRow(pOperator, pSup, &pInfo->binfo, result, length); + if(result){ + taosMemoryFree(result); + } + } +#endif + } + + closeAllResultRows(&pInfo->binfo.resultRowInfo); + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, + &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + OPTR_SET_OPENED(pOperator); + return TSDB_CODE_SUCCESS; +} + +static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + int64_t gid = pBlock->info.groupId; + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfOutput; + + int16_t bytes = pStateColInfoData->info.bytes; + int16_t type = pStateColInfoData->info.type; + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg[pInfo->colIndex])) { + continue; + } + + char* val = colDataGetData(pStateColInfoData, j); + + if (!pInfo->hasKey) { + memcpy(pInfo->stateKey.pData, val, bytes); + pInfo->hasKey = true; + + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (memcmp(pInfo->stateKey.pData, val, bytes) == 0) { + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // a new state window started + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStateWindowOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + STimeWindow win = pTaskInfo->window; + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doStateWindowAggImpl(pOperator, pInfo, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { + STableIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSDataBlock* pBlock = pInfo->binfo.pRes; + + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + return pOperator->fpSet.getStreamResFn(pOperator, newgroup); + } else { + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + + if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBlock->info.rows == 0 ? NULL : pBlock; + } +} + +// todo merged with the build group result. +static void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, + int32_t* rowCellInfoOffset) { + size_t num = taosArrayGetSize(pUpdateList); + + for (int32_t i = 0; i < num; ++i) { + SResKeyPos* pPos = taosArrayGetP(pUpdateList, i); + + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); + // + for (int32_t j = 0; j < numOfOutput; ++j) { + pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); + // + struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; + // if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { + // continue; + // } + // + // if (pCtx[j].fpSet.process) { // TODO set the dummy function. + //// pCtx[j].fpSet.finalize(&pCtx[j]); + // pResInfo->initialized = true; + // } + // + if (pRow->numOfRows < pResInfo->numOfRes) { + pRow->numOfRows = pResInfo->numOfRes; + } + } + + releaseBufPage(pBuf, bufPage); + } +} + +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { + STableIntervalOperatorInfo* pInfo = pOperator->info; + int32_t order = TSDB_ORDER_ASC; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + // STimeWindow win = {0}; + *newgroup = false; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + SArray* pUpdated = NULL; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, true); + pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + } + + finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, + pInfo->binfo.rowCellInfoOffset); + + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + + ASSERT(pInfo->binfo.pRes->info.rows > 0); + pOperator->status = OP_RES_TO_RETURN; + + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} + +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + taosMemoryFreeClear(pInfo->stateKey.pData); +} + +void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); +} + +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamTimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, + destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +// todo handle multiple tables cases. +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + // todo find the correct time stamp column slot + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfOutput; + int64_t gid = pBlock->info.groupId; + + int64_t gap = pInfo->gap; + + if (!pInfo->reptScan) { + pInfo->reptScan = true; + pInfo->winSup.prevTs = INT64_MIN; + } + + SWindowRowsSup* pRowSup = &pInfo->winSup; + pRowSup->numOfRows = 0; + + // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (pInfo->winSup.prevTs == INT64_MIN) { + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } else if (tsList[j] - pRowSup->prevTs <= gap && (tsList[j] - pRowSup->prevTs) >= 0) { + // The gap is less than the threshold, so it belongs to current session window that has been opened already. + doKeepTuple(pRowSup, tsList[j]); + if (j == 0 && pRowSup->startRowIndex != 0) { + pRowSup->startRowIndex = 0; + } + } else { // start a new session window + SResultRow* pResult = NULL; + + // keep the time window for the closed time window. + STimeWindow window = pRowSup->win; + + pRowSup->win.ekey = pRowSup->win.skey; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + // pInfo->numOfRows data belong to the current session window + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + + // here we start a new session window + doKeepNewWindowStartInfo(pRowSup, tsList, j); + doKeepTuple(pRowSup, tsList[j]); + } + } + + SResultRow* pResult = NULL; + pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; + int32_t ret = + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); +} + +static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSessionAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + return pBInfo->pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, order, true); + doSessionWindowAggImpl(pOperator, pInfo, pBlock); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); + + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STimeSliceOperatorInfo* pSliceInfo = pOperator->info; + if (pOperator->status == OP_RES_TO_RETURN) { + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pSliceInfo->binfo.pRes; + } + + int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); + // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); + } + + // restore the value + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); + // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); + + // initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); + // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); + + if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pSliceInfo->binfo.pRes->info.rows == 0 ? NULL : pSliceInfo->binfo.pRes; +} + +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { + STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pOperator == NULL || pInfo == NULL) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pOperator->name = "TimeSliceOperator"; + // pOperator->operatorType = OP_AllTimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo) { + SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->colIndex = -1; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, 4096); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + + pInfo->twAggSup = *pTwAggSup; + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pOperator->name = "StateWindowOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, + destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_SUCCESS; + return NULL; +} + +void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSessionAggOperatorInfo* pInfo = (SSessionAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { + SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->twAggSup = *pTwAggSupp; + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->winSup.prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, + destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->pTaskInfo = pTaskInfo; + + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySWindowOperatorInfo(pInfo, numOfCols); + } + + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} \ No newline at end of file From 9adc1ed8ac1a8b422bf175d909cf64035b8e44bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 15:04:34 +0800 Subject: [PATCH 51/90] refactor: do some internal refactor. --- source/libs/executor/inc/executil.h | 2 +- source/libs/executor/inc/executorimpl.h | 12 +- source/libs/executor/src/executorimpl.c | 23 ++-- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 120 ++++++++---------- 5 files changed, 74 insertions(+), 87 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f0cb1c1107..9f834fd659 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -73,7 +73,7 @@ typedef struct SResKeyPos { } SResKeyPos; typedef struct SResultRowInfo { - SResultRowPosition *pPosition; + SResultRowPosition *pPosition; // todo remove this int32_t size; // number of result set int32_t capacity; // max capacity SResultRowPosition cur; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a49c3984aa..e94564a1b1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -425,7 +425,7 @@ typedef struct STimeWindowSupp { SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; -typedef struct STableIntervalOperatorInfo { +typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info @@ -440,7 +440,7 @@ typedef struct STableIntervalOperatorInfo { SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; struct SFillInfo* pFillInfo; // fill info -} STableIntervalOperatorInfo; +} SIntervalAggOperatorInfo; typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; @@ -533,6 +533,7 @@ typedef struct SSessionAggOperatorInfo { SWindowRowsSup winSup; bool reptScan; // next round scan int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; } SSessionAggOperatorInfo; @@ -619,8 +620,8 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf); + void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, @@ -673,7 +674,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4860e10c19..f9173a50ba 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2246,10 +2246,13 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, - SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { +void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + int32_t* rowCellOffset = pbInfo->rowCellInfoOffset; + SSDataBlock* pBlock = pbInfo->pRes; + SqlFunctionCtx* pCtx = pbInfo->pCtx; + blockDataCleanup(pBlock); if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { return; @@ -3881,8 +3884,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, - pInfo->rowCellInfoOffset, pInfo->pCtx); + doBuildResultDatablock(pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4976,9 +4978,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType}; - int32_t primaryTsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, - pTableGroupInfo, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTableGroupInfo, + pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; @@ -4992,9 +4995,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; + int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, &as, pTaskInfo); + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -5021,7 +5026,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pFillNode->pTargets, NULL, &num); - SInterval* pInterval = &((STableIntervalOperatorInfo*)ops[0]->info)->interval; + SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval; pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode, (SNodeListNode*)pFillNode->pValues, false, pTaskInfo); } else { ASSERT(0); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index dd9cd52870..00ac925c2d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -311,7 +311,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { - doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cc09c05e1c..31ec6883d5 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -456,7 +456,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { + TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { int32_t order = pInfo->order; bool ascQuery = (order == TSDB_ORDER_ASC); @@ -609,7 +609,7 @@ static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowInd static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; int32_t numOfOutput = pOperatorInfo->numOfOutput; @@ -759,7 +759,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableIntervalOperatorInfo* pInfo = pOperator->info; + SIntervalAggOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; // STimeWindow win = {0}; @@ -893,8 +893,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -927,8 +926,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -937,7 +935,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { } static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; + SIntervalAggOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->status == OP_EXEC_DONE) { @@ -955,8 +953,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro } blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -967,7 +964,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro } // todo merged with the build group result. -static void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, +static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, int32_t* rowCellInfoOffset) { size_t num = taosArrayGetSize(pUpdateList); @@ -976,22 +973,11 @@ static void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDi SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); - // + for (int32_t j = 0; j < numOfOutput; ++j) { - pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - // - struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - // if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - // continue; - // } - // - // if (pCtx[j].fpSet.process) { // TODO set the dummy function. - //// pCtx[j].fpSet.finalize(&pCtx[j]); - // pResInfo->initialized = true; - // } - // - if (pRow->numOfRows < pResInfo->numOfRes) { - pRow->numOfRows = pResInfo->numOfRes; + SResultRowEntryInfo* pEntry = getResultCell(pRow, j, rowCellInfoOffset); + if (pRow->numOfRows < pEntry->numOfRes) { + pRow->numOfRows = pEntry->numOfRes; } } @@ -1000,7 +986,7 @@ static void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDi } static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { - STableIntervalOperatorInfo* pInfo = pOperator->info; + SIntervalAggOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; if (pOperator->status == OP_EXEC_DONE) { @@ -1008,8 +994,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -1040,13 +1025,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } - finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); + finalizeUpdatedResult(pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); ASSERT(pInfo->binfo.pRes->info.rows > 0); pOperator->status = OP_RES_TO_RETURN; @@ -1061,7 +1044,7 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { } void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { - STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)param; + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); cleanupAggSup(&pInfo->aggSup); } @@ -1070,26 +1053,26 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; // pInfo->execModel = OPTR_EXEC_MODEL_STREAM; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; pInfo->primaryTsIndex = primaryTsSlotId; - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); - initResultSizeInfo(pOperator, numOfRows); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); @@ -1099,14 +1082,14 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - pOperator->name = "TimeIntervalAggOperator"; + pOperator->name = "TimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); @@ -1130,7 +1113,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { - STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; @@ -1189,8 +1172,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - // todo find the correct time stamp column slot - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); bool masterScan = true; int32_t numOfOutput = pOperator->numOfOutput; @@ -1266,8 +1248,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -1300,8 +1281,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, - pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -1435,7 +1415,7 @@ void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1443,10 +1423,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo goto _error; } - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); - initResultSizeInfo(pOperator, numOfRows); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -1457,17 +1436,18 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->winSup.prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; + pInfo->tsSlotId = tsSlotId; + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->winSup.prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); From 3438ed13f363a6c34efb62974a12de2246ee186b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 15:27:13 +0800 Subject: [PATCH 52/90] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 7 ++- source/libs/executor/src/executorMain.c | 8 +-- source/libs/executor/src/executorimpl.c | 61 +++++++------------ source/libs/executor/src/groupoperator.c | 8 +-- source/libs/executor/src/scanoperator.c | 21 +++---- source/libs/executor/src/timewindowoperator.c | 26 ++++---- source/libs/executor/test/executorTests.cpp | 4 +- 7 files changed, 55 insertions(+), 80 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e94564a1b1..2f8aefcac0 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -197,7 +197,7 @@ typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggS struct SOptrBasicInfo* pInfo, char* result, int32_t length); typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); -typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param, int32_t num); typedef int32_t (*__optr_get_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain); @@ -643,7 +643,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWin void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, @@ -693,6 +693,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); + SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); @@ -713,7 +714,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 516afe5553..7705744694 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -154,14 +154,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); - int64_t st = 0; - - st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &newgroup); + int64_t st = taosGetTimestampUs(); + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); uint64_t el = (taosGetTimestampUs() - st); + pTaskInfo->cost.elapsedTime += el; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f9173a50ba..fc23722687 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -942,7 +942,7 @@ int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* assert(pResultRow != NULL); setResultRowKey(pResultRow, pData, type); - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -2032,20 +2032,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } -STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { +STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win) { STableQueryInfo* pTableQueryInfo = buf; pTableQueryInfo->lastKey = win.skey; - - // set more initial size of interval/groupby query - // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { - int32_t initialSize = 128; - // int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); - // if (code != TSDB_CODE_SUCCESS) { - // return NULL; - // } - // } else { // in other aggregate query, do not initialize the windowResInfo - // } - return pTableQueryInfo; } @@ -2058,8 +2047,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { // cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t* rowCellInfoOffset) { +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -2160,7 +2148,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u } } - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo, SAggOperatorInfo* pAggInfo) { @@ -3202,7 +3190,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3221,8 +3209,6 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { @@ -3397,8 +3383,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - bool newgroup = false; - return pOperator->fpSet.getNextFn(pOperator, &newgroup); + return pOperator->fpSet.getNextFn(pOperator); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { @@ -3569,7 +3554,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL; } -static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -3716,7 +3701,7 @@ _error: return NULL; } -static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSort(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -3819,7 +3804,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -3869,7 +3854,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; @@ -4086,7 +4071,7 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) } } -static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { SProjectOperatorInfo* pProjectInfo = pOperator->info; SOptrBasicInfo* pInfo = &pProjectInfo->binfo; @@ -4124,21 +4109,18 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - bool prevVal = *newgroup; - // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { - *newgroup = prevVal; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } // Return result of the previous group in the firstly. - if (*newgroup) { + if (false) { if (pRes->info.rows > 0) { pProjectInfo->existDataBlock = pBlock; break; @@ -4208,7 +4190,7 @@ static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo* pInfo, SResultInf } } -static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doFill(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -4220,6 +4202,9 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } + // todo handle different group data interpolation + bool n = false; + bool *newgroup = &n; doHandleRemainBlockFromNewGroup(pInfo, pResultInfo, newgroup, pTaskInfo); if (pResBlock->info.rows > pResultInfo->threshold || (!pInfo->multigroupResult && pResBlock->info.rows > 0)) { return pResBlock; @@ -4228,7 +4213,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream, newgroup); + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -4394,7 +4379,7 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf } STimeWindow win = {0, INT64_MAX}; - createTableQueryInfo(pTableQueryInfo, false, win); + createTableQueryInfo(pTableQueryInfo, win); return pTableQueryInfo; } @@ -5529,7 +5514,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { SJoinOperatorInfo* pJoinInfo = pOperator->info; SSDataBlock* pRes = pJoinInfo->pRes; @@ -5539,12 +5524,10 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) int32_t nrows = 0; while (1) { - bool prevVal = *newgroup; - if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { SOperatorInfo* ds1 = pOperator->pDownstream[0]; publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1, newgroup); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1); publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->leftPos = 0; @@ -5557,7 +5540,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { SOperatorInfo* ds2 = pOperator->pDownstream[1]; publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2, newgroup); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2); publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->rightPos = 0; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 00ac925c2d..3ac7706545 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -256,7 +256,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } -static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -277,7 +277,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -541,7 +541,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } -static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -558,7 +558,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c2a101e012..a66d2a263c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -257,11 +257,9 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction pTableScanInfo->cond.order = TSDB_ORDER_DESC; } -static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { if (isTaskKilled(pOperator->pTaskInfo)) { @@ -289,7 +287,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } -static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -298,10 +296,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return NULL; } - *newgroup = false; - while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } @@ -334,7 +330,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); while (pTableScanInfo->current < total) { - SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } @@ -421,13 +417,12 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { return pOperator; } -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } STableScanInfo* pTableScanInfo = pOperator->info; - *newgroup = false; STableBlockDistInfo tableBlockDist = {0}; tableBlockDist.numOfTables = 1; // TODO set the correct number of tables @@ -514,7 +509,7 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); } -static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; @@ -859,7 +854,7 @@ static SSDataBlock* buildSysTableMetaBlock() { return pBlock; } -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -1191,7 +1186,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return pOperator; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { #if 0 SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 31ec6883d5..04ff1515b0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -131,7 +131,7 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo // set time window for current result pResultRow->win = (*win); *pResult = pResultRow; - setResultRowOutputBufInitCtx_rv(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -763,12 +763,11 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; // STimeWindow win = {0}; - bool newgroup = false; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -883,7 +882,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } -static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -908,7 +907,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -934,7 +933,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -945,7 +944,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro SSDataBlock* pBlock = pInfo->binfo.pRes; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->fpSet.getStreamResFn(pOperator, newgroup); + return pOperator->fpSet.getStreamResFn(pOperator); } else { pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { @@ -985,7 +984,7 @@ static void finalizeUpdatedResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SArr } } -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; @@ -1002,14 +1001,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup } // STimeWindow win = {0}; - *newgroup = false; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = NULL; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -1239,7 +1237,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } -static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -1262,7 +1260,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -1289,7 +1287,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { +static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -1309,7 +1307,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index de8df7b916..f534cf0917 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -55,7 +55,7 @@ typedef struct SDummyInputInfo { SSDataBlock* pBlock; } SDummyInputInfo; -SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; @@ -121,7 +121,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { return pBlock; } -SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { +SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { SDummyInputInfo* pInfo = static_cast(pOperator->info); if (pInfo->current >= pInfo->totalPages) { return NULL; From 231b0c6cb56b4cfd98511fc60fcaabfa5f2c1d06 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 15:33:24 +0800 Subject: [PATCH 53/90] fix(query): set the correct primary timestamp column for state window operator. --- source/libs/executor/inc/executorimpl.h | 3 ++- source/libs/executor/src/executorimpl.c | 11 +++++---- source/libs/executor/src/timewindowoperator.c | 23 +++++++++---------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2f8aefcac0..6305022b27 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -551,6 +551,7 @@ typedef struct SStateWindowOperatorInfo { int32_t colIndex; // start row index bool hasKey; SStateKeys stateKey; + int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; // bool reptScan; } SStateWindowOperatorInfo; @@ -688,7 +689,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SInterval* pInterval, STimeWindow* pWindow, SSDataBlock* pResBlock, int32_t fillType, SNodeListNode* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SExecTaskInfo* pTaskInfo); SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fc23722687..98804d4f1a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4977,13 +4977,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType}; - - int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; @@ -4999,7 +4998,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, pTaskInfo); + int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; + + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 04ff1515b0..2bd1cb8902 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -809,7 +809,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); int64_t gid = pBlock->info.groupId; @@ -818,9 +817,9 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t numOfOutput = pOperator->numOfOutput; int16_t bytes = pStateColInfoData->info.bytes; - int16_t type = pStateColInfoData->info.type; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + // todo set the correct primary timestamp column + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); TSKEY* tsList = (TSKEY*)pColInfoData->pData; SWindowRowsSup* pRowSup = &pInfo->winSup; @@ -1368,7 +1367,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* } SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, int32_t tsSlotId, SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1383,18 +1382,18 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - pInfo->twAggSup = *pTwAggSup; + pInfo->twAggSup = *pTwAggSup; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pOperator->name = "StateWindowOperator"; + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); From 90202c72217923cb71bbebc311757fd1cc41ba98 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 15:39:46 +0800 Subject: [PATCH 54/90] fix(query): set the timewindow value for the result of partition by operator. --- source/libs/executor/src/groupoperator.c | 1 + source/libs/executor/src/timewindowoperator.c | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 3ac7706545..acef8e4bc4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -537,6 +537,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pageIndex += 1; + blockDataUpdateTsWindow(pInfo->binfo.pRes); pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId; return pInfo->binfo.pRes; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2bd1cb8902..33112b5982 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -762,7 +762,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; - // STimeWindow win = {0}; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -815,10 +814,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI bool masterScan = true; int32_t numOfOutput = pOperator->numOfOutput; - int16_t bytes = pStateColInfoData->info.bytes; - // todo set the correct primary timestamp column SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); TSKEY* tsList = (TSKEY*)pColInfoData->pData; From f5c58a578288730547f049c9f9eb72ae3afb1b8b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 08:09:58 +0000 Subject: [PATCH 55/90] more refact --- include/util/tskiplist2.h | 98 +------------------------------- source/libs/tdb/test/tdbTest.cpp | 65 ++++++++++++++++++++- 2 files changed, 65 insertions(+), 98 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index ce44ff96eb..95fbee5a23 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -60,100 +60,4 @@ struct SSLCfg { } #endif -#endif /*_TD_UTIL_SKIPLIST2_H_*/ - -#if 0 -#ifndef _TD_UTIL_SKILIST_H -#define _TD_UTIL_SKILIST_H - -#include "os.h" -#include "taos.h" -#include "tarray.h" -#include "tfunctional.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_SKIP_LIST_LEVEL 15 -#define SKIP_LIST_RECORD_PERFORMANCE 0 - -// For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) -#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) - -// For thread safety setting -#define SL_THREAD_SAFE (uint8_t)0x4 - -typedef char *SSkipListKey; -typedef char *(*__sl_key_fn_t)(const void *); - -typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); -typedef void *(*iter_next_fn_t)(void *iter); - -typedef struct SSkipListNode { - uint8_t level; - void *pData; - struct SSkipListNode *forwards[]; -} SSkipListNode; - -#define SL_GET_NODE_DATA(n) (n)->pData -#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] -#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] - -typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; - -typedef struct SSkipList { - uint32_t seed; - __compar_fn_t comparFn; - __sl_key_fn_t keyFn; - TdThreadRwlock *lock; - uint16_t len; - uint8_t maxLevel; - uint8_t flags; - uint8_t type; // static info above - uint8_t level; - uint32_t size; - SSkipListNode *pHead; // point to the first element - SSkipListNode *pTail; // point to the last element - tGenericSavedFunc *insertHandleFn; -} SSkipList; - -typedef struct SSkipListIterator { - SSkipList *pSkipList; - SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator - SSkipListNode *next; // next points to the true qualified node in skiplist -} SSkipListIterator; - -#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) -#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) -#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) -#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) -#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) -#define SL_SIZE(s) (s)->size - -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, - __sl_key_fn_t fn); -void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); -void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); -SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); -bool tSkipListIterNext(SSkipListIterator *iter); -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); -void *tSkipListDestroyIter(SSkipListIterator *iter); -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_SKILIST_H*/ - -#endif \ No newline at end of file +#endif /*_TD_UTIL_SKIPLIST2_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4f20efd904..78e5bc7832 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -318,4 +318,67 @@ TEST(tdb_test, simple_test2) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_delete1) { taosRemoveDir("tdb"); } \ No newline at end of file +TEST(tdb_test, simple_delete1) { + int ret; + TDB *pDb; + char key[128]; + char data[128]; + TXN txn; + TENV *pEnv; + SPoolMem *pPool; + void *pKey = NULL; + void *pData = NULL; + int nKey; + int nData; + int nKV = 254; + + taosRemoveDir("tdb"); + + pPool = openPool(); + + // open env + ret = tdbEnvOpen("tdb", 4096, 256, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + // loop to insert batch data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); + } + + // loop to delete some data + for (int iData = nKV - 1; iData >= 0; iData--) { + sprintf(key, "key%d", iData); + + ret = tdbDbDelete(pDb, key, strlen(key), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + + tdbCommit(pEnv, &txn); + + closePool(pPool); + + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} \ No newline at end of file From bf355e705a3ab4af0ff64cd821c03cf28ee686ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 09:59:58 +0000 Subject: [PATCH 56/90] fix a TDB balance bug --- source/libs/tdb/src/db/tdbBtree.c | 8 ++++---- source/libs/tdb/test/tdbTest.cpp | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 887ab10ec0..176a76cf28 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -582,14 +582,14 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx SCell *pCell; int szLCell, szRCell; + // balance page (iNew) and (iNew-1) for (;;) { pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); - if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); + if (!childNotLeaf) { + szRCell = szLCell; } else { - szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; SPage *pPage; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 78e5bc7832..46c13be3c2 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -202,6 +202,8 @@ TEST(tdb_test, simple_test) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; @@ -283,6 +285,8 @@ TEST(tdb_test, simple_test2) { ret = tdbDbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); + tdbDbcMoveToFirst(pDBC); + for (;;) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; @@ -341,7 +345,7 @@ TEST(tdb_test, simple_delete1) { GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbDbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); From 6a7630e040df676e2b4850dcffaccb5abd66a637 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 18:18:11 +0800 Subject: [PATCH 57/90] enh(index): add prefix query --- include/libs/index/index.h | 4 ++-- source/libs/executor/src/indexoperator.c | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c33f95cb08..7b4414d329 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -46,8 +46,8 @@ typedef enum { DEFAULT // query } SIndexOperOnColumn; -typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType; +typedef enum { MUST = 1, SHOULD, NOT } EIndexOperatorType; +typedef enum { QUERY_TERM = 1, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 438c4254f7..87abe6a9bf 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -62,6 +62,20 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN]; } SIFParam; +static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { + if (src == OP_TYPE_GREATER_THAN || src == OP_TYPE_GREATER_EQUAL || src == OP_TYPE_LOWER_THAN || + src == OP_TYPE_LOWER_EQUAL) { + *dst = QUERY_RANGE; + } else if (src == OP_TYPE_EQUAL) { + *dst = QUERY_TERM; + } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { + *dst = QUERY_REGEX; + } else { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; +} + typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); // construct tag filter operator later static void destroyTagFilterOperatorInfo(void *param) { @@ -238,9 +252,12 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); - indexMultiTermQueryAdd(mtm, tm, QUERY_TERM); + + EIndexQueryType qtype = 0; + SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + + indexMultiTermQueryAdd(mtm, tm, qtype); return indexSearch(NULL, mtm, output->result); } From d3487c88c25852d63c6f99a8cf65c34692fdb407 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 11:24:52 +0000 Subject: [PATCH 58/90] TDB impl delete --- source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++-- source/libs/tdb/test/tdbTest.cpp | 44 ++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 176a76cf28..85cbae316d 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -766,6 +766,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } + if (TDB_BTREE_PAGE_IS_ROOT(pParent) && TDB_PAGE_TOTAL_CELLS(pParent) == 0) { + i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]); + // copy content to the parent page + tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); + tdbPageCopy(pNews[0], pParent); + } + for (int i = 0; i < 3; i++) { if (pDivCell[i]) { tdbOsFree(pDivCell[i]); @@ -1454,9 +1461,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } } else { - // delete the leaf page and do balance (TODO) + // delete the leaf page and do balance ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); - ASSERT(0); + + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } } } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 46c13be3c2..994e53cd9d 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -120,7 +120,7 @@ TEST(tdb_test, simple_test) { TENV *pEnv; TDB *pDb; tdb_cmpr_fn_t compFunc; - int nData = 10000000; + int nData = 1000000; TXN txn; taosRemoveDir("tdb"); @@ -291,9 +291,9 @@ TEST(tdb_test, simple_test2) { ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; - std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; - std::cout.write((char *)pVal, vLen) /* << " " << vLen */; - std::cout << std::endl; + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; count++; } @@ -333,15 +333,16 @@ TEST(tdb_test, simple_delete1) { void *pKey = NULL; void *pData = NULL; int nKey; + TDBC *pDbc; int nData; - int nKV = 254; + int nKV = 69; taosRemoveDir("tdb"); pPool = openPool(); // open env - ret = tdbEnvOpen("tdb", 4096, 256, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database @@ -370,7 +371,7 @@ TEST(tdb_test, simple_delete1) { } // loop to delete some data - for (int iData = nKV - 1; iData >= 0; iData--) { + for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); ret = tdbDbDelete(pDb, key, strlen(key), &txn); @@ -378,6 +379,35 @@ TEST(tdb_test, simple_delete1) { } // query the data + for (int iData = 0; iData < nKV; iData++) { + sprintf(key, "key%d", iData); + + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + if (iData <= 30) { + GTEST_ASSERT_EQ(ret, 0); + } else { + GTEST_ASSERT_EQ(ret, -1); + } + } + + // loop to iterate the data + tdbDbcOpen(pDb, &pDbc, NULL); + + ret = tdbDbcMoveToFirst(pDbc); + GTEST_ASSERT_EQ(ret, 0); + + pKey = NULL; + pData = NULL; + for (;;) { + ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + if (ret < 0) break; + + std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; + std::cout.write((char *)pData, nData) /* << " " << vLen */; + std::cout << std::endl; + } + + tdbDbcClose(pDbc); tdbCommit(pEnv, &txn); From 8e7cf4a661d70ea1e6e1265298d6f38cd6258251 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 20:14:53 +0800 Subject: [PATCH 59/90] enh(index): add prefix query --- include/libs/index/index.h | 4 ++-- source/libs/executor/src/indexoperator.c | 5 ++++- source/libs/index/src/indexCache.c | 28 +----------------------- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 7b4414d329..5239452aa3 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -46,8 +46,8 @@ typedef enum { DEFAULT // query } SIndexOperOnColumn; -typedef enum { MUST = 1, SHOULD, NOT } EIndexOperatorType; -typedef enum { QUERY_TERM = 1, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; +typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 87abe6a9bf..15a3e5901a 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -258,7 +258,9 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); indexMultiTermQueryAdd(mtm, tm, qtype); - return indexSearch(NULL, mtm, output->result); + int ret = indexSearch(NULL, mtm, output->result); + indexMultiTermQueryDestroy(mtm); + return ret; } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { @@ -524,6 +526,7 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) { if (pFilterNode == NULL) { return SFLT_NOT_INDEX; } + // impl later return SFLT_ACCURATE_INDEX; } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 2035766f26..78368981b3 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -318,33 +318,7 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SI if (mem == NULL) { return 0; } - char* key = indexCacheTermGet(ct); - - SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - if (node != NULL) { - CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (qtype == QUERY_TERM) { - if (0 == strcmp(c->colVal, ct->colVal)) { - if (c->operaType == ADD_VALUE) { - INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) - // taosArrayPush(result, &c->uid); - *s = kTypeValue; - } else if (c->operaType == DEL_VALUE) { - INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) - } - } else { - break; - } - } else if (qtype == QUERY_PREFIX) { - } else if (qtype == QUERY_SUFFIX) { - } else if (qtype == QUERY_RANGE) { - } - } - } - tSkipListDestroyIter(iter); - return 0; + return cacheSearch[qtype](mem, ct, tr, s); } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); From dc6bdab4e4f313e0117d57b82b5a9b822ee1ea53 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 May 2022 21:56:05 +0800 Subject: [PATCH 60/90] enh(index): add prefix query --- include/libs/index/index.h | 4 +- source/libs/executor/src/indexoperator.c | 7 +++- source/libs/index/inc/indexInt.h | 5 ++- source/libs/index/src/index.c | 5 ++- source/libs/index/test/indexTests.cc | 50 ++++++++++++------------ source/libs/index/test/jsonUT.cc | 16 ++++---- 6 files changed, 46 insertions(+), 41 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 5239452aa3..bf260d4899 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -168,8 +168,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 15a3e5901a..0f075c5de2 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -247,8 +247,11 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), - right->condValue, strlen(right->condValue)); + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, + strlen(left->colName), right->condValue, strlen(right->condValue)); + if (operType == OP_TYPE_LOWER_EQUAL || operType == OP_TYPE_GREATER_EQUAL || operType == OP_TYPE_GREATER_THAN || + operType == OP_TYPE_LOWER_THAN) { + } if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 37318767c7..62ddf85985 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -57,7 +57,7 @@ struct SIndex { char* path; - SIndexStat stat; + SIndexStat stat; TdThreadMutex mtx; }; @@ -86,6 +86,7 @@ typedef struct SIndexTerm { int32_t nColName; char* colVal; int32_t nColVal; + int8_t qType; // just use for range } SIndexTerm; typedef struct SIndexTermQuery { @@ -165,7 +166,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index e0c24ac3bd..ee77aa92e9 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, - int32_t nColName, const char* colVal, int32_t nColVal) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, + const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { return NULL; @@ -262,6 +262,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(tm->colVal, colVal, nColVal); tm->nColVal = nColVal; + tm->qType = queryType; return tm; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 0162b754ee..d8ea6a8233 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,9 +483,9 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM}; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,9 +623,9 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -638,9 +638,9 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = { term, QUERY_TERM }; + SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index f789d23136..e1e5004701 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); From cfada1cc8ae2ee8e03baf6cf0b99ffa384336578 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 May 2022 14:08:30 +0000 Subject: [PATCH 61/90] more sl refact --- include/util/tskiplist2.h | 6 ++++ source/util/src/tskiplist2.c | 57 +++++++++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index 95fbee5a23..f3366dd574 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -32,6 +32,7 @@ typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2 // SSkipList2 int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl); int32_t slClose(SSkipList2 *pSl); +int32_t slClear(SSkipList2 *pSl); // SSLCursor int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc); @@ -56,6 +57,11 @@ struct SSLCfg { void (*xFree)(void *, void *); }; +struct SSLCursor { + SSkipList2 *pSl; + SSLNode **forwards[SL_MAX_LEVEL]; +}; + #ifdef __cplusplus } #endif diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c index 6657896131..e7391da285 100644 --- a/source/util/src/tskiplist2.c +++ b/source/util/src/tskiplist2.c @@ -27,11 +27,7 @@ struct SSkipList2 { uint32_t seed; int32_t size; const SSLCfg *pCfg; -}; - -struct SSLCursor { - SSkipList2 *pSl; - SSLNode *forwards[SL_MAX_LEVEL]; + SSLNode *pHead[]; }; static void *slMalloc(void *pPool, int32_t size); @@ -48,25 +44,66 @@ const SSLCfg slDefaultCfg = {.maxLevel = SL_MAX_LEVEL, int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl) { SSkipList2 *pSl = NULL; + int32_t size; *ppSl = NULL; if (pCfg == NULL) pCfg = &slDefaultCfg; - // check cfg (TODO) + // check config (TODO) + + // malloc handle + size = sizeof(*pSl) + sizeof(SSLNode *) * pCfg->maxLevel * 2; + pSl = pCfg->xMalloc(pCfg->pPool, size); + if (pSl == NULL) { + return -1; + } + + pSl->level = 0; + pSl->seed = taosRand(); + pSl->size = 0; + pSl->pCfg = pCfg; + + // init an empty skiplist + for (int32_t i = 0; i < pCfg->maxLevel * 2; i++) { + pSl->pHead[i] = NULL; + } - // create handle - pSl = pCfg->xMalloc(pCfg->pPool, sizeof(*pSl)); - // (TODO) *ppSl = pSl; return 0; } int32_t slClose(SSkipList2 *pSl) { - // TODO + if (pSl) { + slClear(pSl); + if (pSl->pCfg->xFree) { + pSl->pCfg->xFree(pSl->pCfg->pPool, pSl); + } + } + + return 0; +} + +int32_t slClear(SSkipList2 *pSl) { + // loop to clear sl + for (;;) { + // (TODO) + } + + // init sl (TODO) + return 0; } int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc) { + pSlc->pSl = pSl; + + for (int i = 0; i < SL_MAX_LEVEL; i++) { + if (i < pSl->pCfg->maxLevel) { + } else { + pSlc->forwards[i] = NULL; + } + } + // TODO return 0; } From ad3075db1eceb730f7d549c1fca3755f4046a0e5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 23:23:49 +0800 Subject: [PATCH 62/90] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/executorimpl.c | 21 +- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 257 ++++++++---------- source/libs/executor/src/timewindowoperator.c | 10 +- 5 files changed, 136 insertions(+), 165 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6305022b27..2795109603 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -268,7 +268,7 @@ typedef struct SOperatorFpSet { typedef struct SOperatorInfo { uint8_t operatorType; - bool blockingOptr; // block operator or not + bool blocking; // block operator or not uint8_t status; // denote if current operator is completed int32_t numOfOutput; // number of columns of the current operator results char* name; // name, used to show the query execution plan @@ -341,9 +341,9 @@ typedef struct STableScanInfo { int64_t elapsedTime; int32_t prevGroupId; // previous table group id SScanInfo scanInfo; - int32_t current; - SNode* pFilterNode; // filter operator info - SqlFunctionCtx* pCtx; // next operator query context + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer + SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context SResultRowInfo* pResultRowInfo; int32_t* rowCellInfoOffset; SExprInfo* pExpr; @@ -397,7 +397,6 @@ typedef struct SSysTableScanInfo { SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; - int32_t capacity; int64_t numOfBlocks; // extract basic running information. SLoadRemoteDataInfo loadInfo; } SSysTableScanInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 98804d4f1a..a6cdc6ab1a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,6 +132,7 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } @@ -3282,7 +3283,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pBlock->info.numOfCols; @@ -3673,7 +3674,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->name = "SortedMerge"; // pOperator->operatorType = OP_SortedMerge; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = num; @@ -3756,7 +3757,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; @@ -4419,7 +4420,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; @@ -4532,7 +4533,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExprInfo; @@ -4615,7 +4616,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp } pOperator->name = "FillOperator"; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; pOperator->pExpr = pExpr; @@ -4861,7 +4862,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + + int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4877,8 +4879,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); @@ -5628,7 +5629,7 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf pOperator->name = "JoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index acef8e4bc4..f5dd67217e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -353,7 +353,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pOperator->name = "GroupbyAggOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; pOperator->pExpr = pExprInfo; @@ -612,7 +612,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* } pOperator->name = "PartitionOperator"; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; pInfo->binfo.pRes = pResultBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a66d2a263c..30865b42f5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "common/ttime.h" +#include "ttime.h" #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -36,10 +36,10 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo); +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -158,7 +158,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } -int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; @@ -296,15 +296,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + // do the ascending order traverse in the first place. + while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; @@ -319,7 +320,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; - if (pTableScanInfo->current < total) { + if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -329,21 +330,20 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { qDebug("%s start to descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); - while (pTableScanInfo->current < total) { + while (pTableScanInfo->scanTimes < total) { SSDataBlock* p = doTableScanImpl(pOperator); if (p != NULL) { return p; } - pTableScanInfo->current += 1; + pTableScanInfo->scanTimes += 1; - if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) { + if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 - "-%" PRId64, - GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); + "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); // do prepare for the next round table scan operation tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -372,48 +372,50 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon pInfo->cond = *pCond; pInfo->scanInfo = (SScanInfo){.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]}; - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; pInfo->dataBlockLoadFlag = dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pDataReader; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pDataReader; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->fpSet.getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->name = "TableScanOperator"; // for dubug purpose + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, NULL, NULL, NULL, NULL); static int32_t cost = 0; - pOperator->cost.openCost = ++cost; + + // for non-blocking operator, the open cost is always 0 + pOperator->cost.openCost = 0; pOperator->cost.totalCost = ++cost; pOperator->resultInfo.totalRows = ++cost; return pOperator; } -SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { +SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); - - pInfo->dataReader = pTsdbReadHandle; - pInfo->current = 0; - pInfo->prevGroupId = -1; - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet.getNextFn = doTableScanImpl; + pInfo->dataReader = pReadHandle; + pInfo->prevGroupId = -1; + + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL); return pOperator; } @@ -482,7 +484,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pOperator->name = "DataBlockInfoScanOperator"; // pOperator->operatorType = OP_TableBlockInfoScan; - pOperator->blockingOptr = false; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doBlockInfoScan; @@ -636,20 +638,22 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* } pInfo->readerHandle = streamReadHandle; - pInfo->pRes = pResBlock; - pInfo->pCondition = pCondition; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; - pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->name = "StreamBlockScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->fpSet._openFn = operatorDummyOpenFn; pOperator->fpSet.getNextFn = doStreamBlockScan; - pOperator->fpSet.closeFn = operatorDummyCloseFn; + pOperator->fpSet.closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamBlockScan, NULL, NULL, operatorDummyCloseFn, NULL, NULL, NULL); + return pOperator; } @@ -671,14 +675,12 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { switch (nType) { case QUERY_NODE_OPERATOR: { SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { *(int32_t*)pContext = 1; return DEAL_RES_CONTINUE; } *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; } case QUERY_NODE_COLUMN: { @@ -709,19 +711,17 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { default: break; } - return DEAL_RES_CONTINUE; } -void getDBNameFromCondition(SNode* pCondition, char* dbName) { +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { if (NULL == pCondition) { return; } - - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*) dbName); } -static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t code) { +static int32_t loadSysTableCallback(void* param, const SDataBuf* pMsg, int32_t code) { SOperatorInfo* operator=(SOperatorInfo*) param; SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; if (TSDB_CODE_SUCCESS == code) { @@ -746,13 +746,14 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } SFilterInfo* filter = NULL; - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); + + int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); filterFreeInfo(filter); SSDataBlock* px = createOneDataBlock(pInfo->pRes, false); @@ -795,61 +796,31 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { static SSDataBlock* buildSysTableMetaBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = 10; - pBlock->info.hasVarCol = true; + + size_t size = 0; + const SSysTableMeta *pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for(int32_t i = 0; i < size; ++i) { + if(strcmp(pMeta[i].name, TSDB_INS_TABLE_USER_TABLES) == 0) { + index = i; + break; + } + } pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = 1; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + for(int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = {0}; + colInfoData.info.colId = i + 1; + colInfoData.info.type = pMeta[index].schema[i].type; + colInfoData.info.bytes = pMeta[index].schema[i].bytes; + taosArrayPush(pBlock->pDataBlock, &colInfoData); + } - colInfoData.info.colId = 2; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 3; - colInfoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfoData.info.bytes = 8; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 4; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 5; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 6; - colInfoData.info.type = TSDB_DATA_TYPE_BIGINT; - colInfoData.info.bytes = 8; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 7; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 8; - colInfoData.info.type = TSDB_DATA_TYPE_INT; - colInfoData.info.bytes = 4; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 9; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = 512 + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); - - colInfoData.info.colId = 10; - colInfoData.info.type = TSDB_DATA_TYPE_VARCHAR; - colInfoData.info.bytes = 20 + VARSTR_HEADER_SIZE; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + pBlock->info.numOfCols = pMeta[index].colNum; + pBlock->info.hasVarCol = true; return pBlock; } @@ -868,7 +839,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { return NULL; } - buildSysDbTableInfo(pInfo); + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); doFilterResult(pInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -896,7 +867,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { varDataSetLen(dbname, strlen(varDataVal(dbname))); SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pInfo->capacity); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; while (metaTbCursorNext(pInfo->pCur) == 0) { @@ -977,7 +948,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { pColInfoData = taosArrayGet(p->pDataBlock, 9); colDataAppend(pColInfoData, numOfRows, str, false); - if (++numOfRows >= pInfo->capacity) { + if (++numOfRows >= pOperator->resultInfo.capacity) { break; } } @@ -1022,7 +993,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { pMsgSendInfo->msgInfo.pData = buf1; pMsgSendInfo->msgInfo.len = contLen; pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; - pMsgSendInfo->fp = loadSysTableContentCb; + pMsgSendInfo->fp = loadSysTableCallback; int64_t transporterId = 0; int32_t code = asyncSendMsgToServer(pInfo->pTransporter, &pInfo->epSet, &transporterId, pMsgSendInfo); @@ -1060,9 +1031,9 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } } -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo) { +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pInfo->capacity); + blockDataEnsureCapacity(p, capacity); size_t size = 0; const SSysTableMeta* pSysDbTableMeta = NULL; @@ -1133,18 +1104,19 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe return NULL; } - pInfo->accountId = accountId; + pInfo->accountId = accountId; pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->capacity = 4096; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pCondition; + pInfo->scanCols = colList; + + initResultSizeInfo(pOperator, 4096); tNameAssign(&pInfo->name, pName); const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*) readHandle; - blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { tsem_init(&pInfo->ready, 0, 0); pInfo->epSet = epset; @@ -1173,12 +1145,12 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe #endif } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pResBlock->info.numOfCols; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pResBlock->info.numOfCols; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; @@ -1188,7 +1160,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { #if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -1322,29 +1293,29 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = pReaderHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; + pInfo->pReader = readHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; return pOperator; + _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 33112b5982..0384f52dda 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1078,7 +1078,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; @@ -1137,7 +1137,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr pOperator->name = "StreamTimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; @@ -1343,7 +1343,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TimeSliceOperator"; // pOperator->operatorType = OP_AllTimeWindow; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; @@ -1385,7 +1385,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->tsSlotId = tsSlotId; pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfCols; @@ -1437,7 +1437,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo pInfo->reptScan = false; pOperator->name = "SessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; - pOperator->blockingOptr = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; From 05bd6a52c2705c5078c3575a4dd40253353d1bc5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 May 2022 23:52:17 +0800 Subject: [PATCH 63/90] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 28 ++++++++++++++----------- source/libs/executor/src/executorimpl.c | 13 +++++++----- source/libs/executor/src/scanoperator.c | 11 +++++----- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2795109603..0404b4d447 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -119,6 +119,17 @@ typedef struct SLimit { int64_t offset; } SLimit; +typedef struct SFileBlockLoadRecorder { + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t skipBlocks; + uint32_t filterOutBlocks; + uint64_t elapsedTime; +} SFileBlockLoadRecorder; + typedef struct STaskCostInfo { int64_t created; int64_t start; @@ -132,14 +143,10 @@ typedef struct STaskCostInfo { uint64_t loadDataInCacheSize; uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t skipBlocks; - uint32_t filterOutBlocks; + + SFileBlockLoadRecorder* pRecoder; uint64_t elapsedTime; + uint64_t firstStageMergeTime; uint64_t winInfoSize; uint64_t tableInfoSize; @@ -333,13 +340,10 @@ typedef struct SScanInfo { typedef struct STableScanInfo { void* dataReader; - - int32_t numOfBlocks; // extract basic running information. - int32_t numOfSkipped; - int32_t numOfBlockStatis; + SFileBlockLoadRecorder readRecorder; int64_t numOfRows; int64_t elapsedTime; - int32_t prevGroupId; // previous table group id +// int32_t prevGroupId; // previous table group id SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a6cdc6ab1a..a79abc9949 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1593,8 +1593,8 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc STaskCostInfo* pCost = &pTaskInfo->cost; - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; +// pCost->totalBlocks += 1; +// pCost->totalRows += pBlock->info.rows; #if 0 // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. @@ -2415,8 +2415,8 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, - pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->pRecoder->totalBlocks, + pSummary->pRecoder->loadBlockStatis, pSummary->pRecoder->loadBlocks, pSummary->pRecoder->totalRows, pSummary->pRecoder->totalCheckedRows); // // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, @@ -4869,9 +4869,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SInterval interval = extractIntervalInfo(pTableScanNode); - return createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, + SOperatorInfo* pOperator = createTableScanOperatorInfo(pDataReader, &cond, numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq, pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + STableScanInfo* pScanInfo = pOperator->info; + pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; + return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 30865b42f5..53b436c70d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -162,7 +162,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableScanInfo* pInfo = pOperator->info; - STaskCostInfo* pCost = &pTaskInfo->cost; + SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; @@ -188,11 +188,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; - bool allHave = true; + bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allHave); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); - if (allHave == true) { + if (allColumnsHaveAgg == true) { int32_t numOfCols = pBlock->info.numOfCols; // todo create this buffer during creating operator @@ -266,7 +266,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); uint32_t status = 0; @@ -406,7 +405,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); pInfo->dataReader = pReadHandle; - pInfo->prevGroupId = -1; +// pInfo->prevGroupId = -1; pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; From ac0fb0237ec3ff4f0fc882322c617599e9655187 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 03:57:16 +0000 Subject: [PATCH 64/90] drop table code --- include/util/tskiplist2.h | 1 + source/dnode/vnode/src/inc/meta.h | 2 - source/dnode/vnode/src/inc/vnodeInt.h | 18 +++--- source/dnode/vnode/src/meta/metaTable.c | 77 ++++++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 39 +++++++++++-- source/libs/tdb/inc/tdb.h | 2 +- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbDb.c | 6 +- source/util/src/tskiplist2.c | 1 - 9 files changed, 117 insertions(+), 31 deletions(-) diff --git a/include/util/tskiplist2.h b/include/util/tskiplist2.h index f3366dd574..83e6ad0868 100644 --- a/include/util/tskiplist2.h +++ b/include/util/tskiplist2.h @@ -26,6 +26,7 @@ extern "C" { typedef struct SSkipList2 SSkipList2; typedef struct SSLCursor SSLCursor; typedef struct SSLCfg SSLCfg; +typedef struct SSLNode SSLNode; typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index ca7e823d6b..5970fdbe92 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -103,8 +103,6 @@ typedef struct { #if 1 -// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaDropTable(SMeta* pMeta, tb_uid_t uid); SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseSmaCursor(SMSmaCursor* pSmaCur); int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1c7f6034ee..5538a63b4b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -46,13 +46,13 @@ extern "C" { #endif -typedef struct SVnodeInfo SVnodeInfo; -typedef struct SMeta SMeta; -typedef struct STsdb STsdb; -typedef struct STQ STQ; -typedef struct SVState SVState; -typedef struct SVBufPool SVBufPool; -typedef struct SQWorker SQHandle; +typedef struct SVnodeInfo SVnodeInfo; +typedef struct SMeta SMeta; +typedef struct STsdb STsdb; +typedef struct STQ STQ; +typedef struct SVState SVState; +typedef struct SVBufPool SVBufPool; +typedef struct SQWorker SQHandle; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -77,6 +77,7 @@ int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -100,7 +101,7 @@ int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, - uint64_t taskId); + uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pMemRef); int32_t tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); @@ -189,7 +190,6 @@ struct STbUidStore { #define TD_VID(PVNODE) (PVNODE)->config.vgId - static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) { SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]); return (pRetention->freq > 0 && pRetention->keep > 0); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 41ef231dcf..c09bafd859 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -98,7 +98,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); - // tdbDbcDrop(pDbc1); + // tdbDbcDelete(pDbc1); // // drop from pTagIdx // // drop from pTtlIdx // } @@ -166,18 +166,79 @@ _err: return -1; } -int metaDropTable(SMeta *pMeta, tb_uid_t uid) { -#if 0 - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO: handle error +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + TDBC *pTbDbc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pNameIdxc = NULL; + const void *pData; + int nData; + tb_uid_t uid; + int64_t tver; + SMetaEntry me = {0}; + SCoder coder = {0}; + int c, ret; + + // search & delete the name idx + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c) { + tdbDbcClose(pNameIdxc); + terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - if (metaRemoveTableFromIdx(pMeta, uid) < 0) { - // TODO + ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); return -1; } -#endif + + uid = *(tb_uid_t *)pData; + + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pNameIdxc); + + // search & delete uid idx + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + tver = *(int64_t *)pData; + tdbDbcDelete(pUidIdxc); + tdbDbcClose(pUidIdxc); + + // search and get meta entry + tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER); + // ret = metaDecodeEntry(&coder, &me); + // if (ret < 0) { + // ASSERT(0); + // return -1; + // } + + tCoderClear(&coder); + tdbDbcClose(pTbDbc); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 91a470b118..878dd5fca4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -445,14 +445,45 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM } static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SVDropTbReq req = {0}; - SVDropTbReq rsp = {0}; + SVDropTbBatchReq req = {0}; + SVDropTbBatchRsp rsp = {0}; + SCoder coder = {0}; + int ret; + + pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->pCont = NULL; + pRsp->contLen = 0; + pRsp->code = TSDB_CODE_SUCCESS; // decode req + tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER); + ret = tDecodeSVDropTbBatchReq(&coder, &req); + if (ret < 0) { + terrno = TSDB_CODE_INVALID_MSG; + pRsp->code = terrno; + goto _exit; + } // process req + rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs); + for (int iReq = 0; iReq < req.nReqs; iReq++) { + SVDropTbReq *pDropTbReq = req.pReqs + iReq; + SVDropTbRsp dropTbRsp = {0}; - // return rsp + /* code */ + ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); + if (ret < 0) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } + + taosArrayPush(rsp.pArray, &dropTbRsp); + } + +_exit: + tCoderClear(&coder); + // encode rsp (TODO) return 0; } @@ -482,7 +513,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if(!pReq) { + if (!pReq) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index dae085c672..cab242d5b3 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -58,7 +58,7 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); -int tdbDbcDrop(TDBC *pDbc); +int tdbDbcDelete(TDBC *pDbc); int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); // TXN diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 85cbae316d..ed7aed552a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1394,7 +1394,7 @@ int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int if (ppVal) { *ppVal = (void *)pBtc->coder.pVal; - *kLen = pBtc->coder.vLen; + *vLen = pBtc->coder.vLen; } return 0; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index f9ef2e7901..3358aefec0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -136,11 +136,7 @@ int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int v return 0; } -int tdbDbcDrop(TDBC *pDbc) { - // TODO - ASSERT(0); - return 0; -} +int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); diff --git a/source/util/src/tskiplist2.c b/source/util/src/tskiplist2.c index e7391da285..77f5ed5051 100644 --- a/source/util/src/tskiplist2.c +++ b/source/util/src/tskiplist2.c @@ -16,7 +16,6 @@ #include "tskiplist2.h" -typedef struct SSLNode SSLNode; struct SSLNode { int8_t level; SSLNode *forwards[]; From e674de5bf9c905d03f067aacc2d2d13c01f4387f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 12:25:39 +0800 Subject: [PATCH 65/90] refactor: adjust logs --- source/dnode/mnode/impl/src/mndTrans.c | 148 +++++++++--------- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 + 2 files changed, 76 insertions(+), 74 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bbd952b6d0..5a2f37429a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -22,9 +22,9 @@ #include "mndSync.h" #include "mndUser.h" -#define MND_TRANS_VER_NUMBER 1 -#define MND_TRANS_ARRAY_SIZE 8 -#define MND_TRANS_RESERVE_SIZE 64 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 +#define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -84,7 +84,7 @@ void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t rawDataLen = sizeof(STrans) + MND_TRANS_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -116,7 +116,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, MND_TRANS_VER_NUMBER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -182,7 +182,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) } - SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) terrno = 0; @@ -214,38 +214,38 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; STransAction action = {0}; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TRANS_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != MND_TRANS_VER_NUMBER) { + if (sver != TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto TRANS_DECODE_OVER; + goto _OVER; } pRow = sdbAllocRow(sizeof(STrans)); - if (pRow == NULL) goto TRANS_DECODE_OVER; + if (pRow == NULL) goto _OVER; pTrans = sdbGetRowObj(pRow); - if (pTrans == NULL) goto TRANS_DECODE_OVER; + if (pTrans == NULL) goto _OVER; - SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) int16_t type = 0; int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &stage, TRANS_DECODE_OVER) - SDB_GET_INT16(pRaw, dataPos, &type, TRANS_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->policy = policy; pTrans->stage = stage; pTrans->transType = type; - SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, TRANS_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, TRANS_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &redoActionNum, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &undoActionNum, TRANS_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &commitLogNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) + SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) pTrans->redoLogs = taosArrayInit(redoLogNum, sizeof(void *)); pTrans->undoLogs = taosArrayInit(undoLogNum, sizeof(void *)); @@ -253,79 +253,79 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->redoActions = taosArrayInit(redoActionNum, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(undoActionNum, sizeof(STransAction)); - if (pTrans->redoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->commitLogs == NULL) goto TRANS_DECODE_OVER; - if (pTrans->redoActions == NULL) goto TRANS_DECODE_OVER; - if (pTrans->undoActions == NULL) goto TRANS_DECODE_OVER; + if (pTrans->redoLogs == NULL) goto _OVER; + if (pTrans->undoLogs == NULL) goto _OVER; + if (pTrans->commitLogs == NULL) goto _OVER; + if (pTrans->redoActions == NULL) goto _OVER; + if (pTrans->undoActions == NULL) goto _OVER; for (int32_t i = 0; i < redoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->redoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < undoLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->undoLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < commitLogNum; ++i) { - SDB_GET_INT32(pRaw, dataPos, &dataLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) pData = taosMemoryMalloc(dataLen); - if (pData == NULL) goto TRANS_DECODE_OVER; + if (pData == NULL) goto _OVER; mTrace("raw:%p, is created", pData); - SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto TRANS_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, pData, dataLen, _OVER); + if (taosArrayPush(pTrans->commitLogs, &pData) == NULL) goto _OVER; pData = NULL; } for (int32_t i = 0; i < redoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } for (int32_t i = 0; i < undoActionNum; ++i) { - SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); - SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); + SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) + SDB_GET_INT32(pRaw, dataPos, &action.contLen, _OVER) action.pCont = taosMemoryMalloc(action.contLen); - if (action.pCont == NULL) goto TRANS_DECODE_OVER; - SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, TRANS_DECODE_OVER); - if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto TRANS_DECODE_OVER; + if (action.pCont == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); + if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; } - SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, _OVER) if (pTrans->paramLen != 0) { pTrans->param = taosMemoryMalloc(pTrans->paramLen); - SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER); } - SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; -TRANS_DECODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); mndTransDropData(pTrans); @@ -545,11 +545,11 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; pTrans->rpcRefId = pReq->refId; - pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); - pTrans->undoActions = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->redoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -855,7 +855,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans == NULL) { mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } SArray *pArray = NULL; @@ -865,18 +865,18 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pArray = pTrans->undoActions; } else { mError("trans:%d, invalid trans stage:%d while recv action rsp", pTrans->id, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } if (pArray == NULL) { mError("trans:%d, invalid trans stage:%d", transId, pTrans->stage); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } int32_t actionNum = taosArrayGetSize(pTrans->redoActions); if (action < 0 || action >= actionNum) { mError("trans:%d, invalid action:%d", transId, action); - goto HANDLE_ACTION_RSP_OVER; + goto _OVER; } STransAction *pAction = taosArrayGet(pArray, action); @@ -892,7 +892,7 @@ void mndTransProcessRsp(SNodeMsg *pRsp) { pAction->acceptableCode); mndTransExecute(pMnode, pTrans); -HANDLE_ACTION_RSP_OVER: +_OVER: mndReleaseTrans(pMnode, pTrans); } @@ -1281,19 +1281,19 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto KILL_OVER; + goto _OVER; } mInfo("trans:%d, start to kill", killReq.transId); pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto KILL_OVER; + goto _OVER; } if (!pUser->superUser) { terrno = TSDB_CODE_MND_NO_RIGHTS; - goto KILL_OVER; + goto _OVER; } pTrans = mndAcquireTrans(pMnode, killReq.transId); @@ -1305,7 +1305,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { code = mndKillTrans(pMnode, pTrans); -KILL_OVER: +_OVER: if (code != 0) { mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index ef80aa14eb..abab24723f 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -57,7 +57,9 @@ class MndTestTrans2 : public ::testing::Test { opt.replicas[0].port = 9040; strcpy(opt.replicas[0].fqdn, "localhost"); opt.msgCb = msgCb; + const char *mnodepath = "/tmp/mnode_test_trans"; + taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); } From 5ed7b44b165ded8d3b391fd748e64bd2f4b2c55a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 4 May 2022 12:35:05 +0800 Subject: [PATCH 66/90] enh(index): support index filter --- include/libs/index/index.h | 12 +++++++++++- source/libs/executor/src/indexoperator.c | 14 ++++++++------ source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexInt.h | 1 - source/libs/index/src/index.c | 1 - source/libs/index/src/indexCache.c | 23 ++++++++++++++++++++++- source/libs/index/src/indexTfile.c | 23 ++++++++++++++++++++++- 7 files changed, 64 insertions(+), 11 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index bf260d4899..c94d75338a 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -47,7 +47,17 @@ typedef enum { } SIndexOperOnColumn; typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX, QUERY_SUFFIX, QUERY_REGEX, QUERY_RANGE } EIndexQueryType; +typedef enum { + QUERY_TERM = 0, + QUERY_PREFIX, + QUERY_SUFFIX, + QUERY_REGEX, + QUERY_LESS_THAN, + QUERY_LESS_EQUAL, + QUERY_GREATER_THAN, + QUERY_GREATER_EQUAL, + QUERY_RANGE +} EIndexQueryType; /* * create multi query diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 0f075c5de2..3b62bdd664 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -63,9 +63,14 @@ typedef struct SIFParam { } SIFParam; static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { - if (src == OP_TYPE_GREATER_THAN || src == OP_TYPE_GREATER_EQUAL || src == OP_TYPE_LOWER_THAN || - src == OP_TYPE_LOWER_EQUAL) { - *dst = QUERY_RANGE; + if (src == OP_TYPE_GREATER_THAN) { + *dst = QUERY_GREATER_THAN; + } else if (src == OP_TYPE_GREATER_EQUAL) { + *dst = QUERY_GREATER_EQUAL; + } else if (src == OP_TYPE_LOWER_THAN) { + *dst = QUERY_LESS_THAN; + } else if (src == OP_TYPE_LOWER_EQUAL) { + *dst = QUERY_LESS_EQUAL; } else if (src == OP_TYPE_EQUAL) { *dst = QUERY_TERM; } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { @@ -249,9 +254,6 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); - if (operType == OP_TYPE_LOWER_EQUAL || operType == OP_TYPE_GREATER_EQUAL || operType == OP_TYPE_GREATER_THAN || - operType == OP_TYPE_LOWER_THAN) { - } if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 3ea986ad48..44355db7a7 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -56,6 +56,7 @@ typedef struct CacheTerm { int8_t colType; SIndexOperOnColumn operaType; + int8_t qType; // query type } CacheTerm; // diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 62ddf85985..04817c4df3 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -86,7 +86,6 @@ typedef struct SIndexTerm { int32_t nColName; char* colVal; int32_t nColVal; - int8_t qType; // just use for range } SIndexTerm; typedef struct SIndexTermQuery { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ee77aa92e9..83b5025ad0 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -262,7 +262,6 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryT tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1); memcpy(tm->colVal, colVal, nColVal); tm->nColVal = nColVal; - tm->qType = queryType; return tm; } diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 78368981b3..ea3cbf5658 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -38,10 +38,15 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { - cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchRange}; + cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, + cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); @@ -88,6 +93,22 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, // impl later return 0; } +static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} +static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + // impl later + return 0; +} static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 7072baf574..327a06084c 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -64,10 +64,15 @@ static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { - tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchRange}; + tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, + tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache)); @@ -299,6 +304,22 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } +static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} +static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { + // impl later + return 0; +} static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); int ret = 0; From d5e578cf163b2b6477642744542314cb3bc71c9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 06:16:29 +0000 Subject: [PATCH 67/90] implement drop table --- source/dnode/vnode/src/meta/metaTable.c | 55 ++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c09bafd859..7ddac9337d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -176,6 +176,9 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int64_t tver; SMetaEntry me = {0}; SCoder coder = {0}; + int8_t type; + int64_t ctime; + tb_uid_t suid; int c, ret; // search & delete the name idx @@ -230,16 +233,56 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { return -1; } - // tCoderInit(&coder, TD_LITTLE_ENDIAN, pData, nData, TD_DECODER); - // ret = metaDecodeEntry(&coder, &me); - // if (ret < 0) { - // ASSERT(0); - // return -1; - // } + // decode entry + void *pDataCopy = taosMemoryMalloc(nData); // remove the copy (todo) + memcpy(pDataCopy, pData, nData); + tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER); + ret = metaDecodeEntry(&coder, &me); + if (ret < 0) { + ASSERT(0); + return -1; + } + type = me.type; + if (type == TSDB_CHILD_TABLE) { + ctime = me.ctbEntry.ctime; + suid = me.ctbEntry.suid; + } else if (type == TSDB_NORMAL_TABLE) { + ctime = me.ntbEntry.ctime; + suid = 0; + } else { + ASSERT(0); + } + + taosMemoryFree(pDataCopy); tCoderClear(&coder); tdbDbcClose(pTbDbc); + if (type == TSDB_CHILD_TABLE) { + // remove the pCtbIdx + TDBC *pCtbIdxc = NULL; + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || c != 0) { + ASSERT(0); + return -1; + } + + tdbDbcDelete(pCtbIdxc); + tdbDbcClose(pCtbIdxc); + + // remove tags from pTagIdx (todo) + } else if (type == TSDB_NORMAL_TABLE) { + // remove from pSkmDb + } else { + ASSERT(0); + } + + // remove from ttl (todo) + if (ctime > 0) { + } + return 0; } From db00d25b9b7f5d849122e56a62056b9f9fb36d90 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 May 2022 14:34:38 +0800 Subject: [PATCH 68/90] fix(query): add one more check for null pointer. --- source/libs/executor/src/executorimpl.c | 13 +++++++------ source/libs/executor/src/scanoperator.c | 1 + source/libs/executor/src/timewindowoperator.c | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a79abc9949..b5d972321b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2412,12 +2412,13 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { // // calculateOperatorProfResults(pQInfo); - qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 - " us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->pRecoder->totalBlocks, - pSummary->pRecoder->loadBlockStatis, pSummary->pRecoder->loadBlocks, pSummary->pRecoder->totalRows, pSummary->pRecoder->totalCheckedRows); - // + SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; + if (pSummary->pRecoder != NULL) { + qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 " us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks, + pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + } // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 53b436c70d..7e721455c3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -236,6 +236,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols); + // todo record the filter time cost doFilter(pTableScanInfo->pFilterNode, pBlock); if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0384f52dda..580614dc02 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1078,7 +1078,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->name = "TimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blocking = true; + pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; From 9d2df3531670cb37266aa73fb71ae9a7cf5ff105 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 07:07:34 +0000 Subject: [PATCH 69/90] drop super table --- source/dnode/vnode/src/meta/metaTable.c | 75 +++++++++++++++---------- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +-- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7ddac9337d..397e074061 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -72,44 +72,61 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - SMetaReader mr = {0}; + TDBC *pNameIdxc = NULL; + TDBC *pUidIdxc = NULL; + TDBC *pCtbIdxc = NULL; + SCtbIdxKey *pCtbIdxKey; + const void *pKey = NULL; + int nKey; + const void *pData = NULL; + int nData; + int c, ret; - // validate req - metaReaderInit(&mr, pMeta, 0); - if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + // prepare uid idx cursor + tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c != 0) { + terrno = TSDB_CODE_VND_TB_NOT_EXIST; + tdbDbcClose(pUidIdxc); goto _err; } - // do drop - // drop from pTbDb - // drop from pSkmDb - // drop from pUidIdx - // drop from pNameIdx - // { - // TDBC *pDbc1 = NULL; - // void *pKey = NULL; - // void *pVal = NULL; - // int kLen = 0; - // int vLen = 0; - // int ret = 0; + // prepare name idx cursor + tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || c != 0) { + ASSERT(0); + } - // // drop from pCtbIdx - // ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1); - // tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/); - // tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen); - // tdbDbcDelete(pDbc1); - // // drop from pTagIdx - // // drop from pTtlIdx - // } + tdbDbcDelete(pUidIdxc); + tdbDbcDelete(pNameIdxc); + tdbDbcClose(pUidIdxc); + tdbDbcClose(pNameIdxc); - // clear and return - metaReaderClear(&mr); - metaError("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + // loop to drop each child table + tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { + tdbDbcClose(pCtbIdxc); + goto _exit; + } + + for (;;) { + tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + pCtbIdxKey = (SCtbIdxKey *)pKey; + + if (pCtbIdxKey->suid > pReq->suid) break; + + // drop the child table (TODO) + + if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + } + +_exit: + metaDebug("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaReaderClear(&mr); metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..7d02840600 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i } // process request - // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { - // rcode = terrno; - // goto _exit; - // } + if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + rcode = terrno; + goto _exit; + } // return rsp _exit: From 8fda7b64dcb531ad8240f1a13f7f9c3615459143 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 15:11:40 +0800 Subject: [PATCH 70/90] refactor: adjust transaction unitest --- include/util/taoserror.h | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 5 +++- source/dnode/mnode/impl/src/mndTrans.c | 38 +++++++++++++------------- source/util/src/terror.c | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 27af7eb27e..b8baf99552 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -264,7 +264,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CANT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 263fd0bad5..e48a9b83c0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -91,10 +91,12 @@ typedef enum { TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_CONSUMER_RECOVER = 1023, TRN_TYPE_BASIC_SCOPE_END, + TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_DROP_DNODE = 2002, TRN_TYPE_GLOBAL_SCOPE_END, + TRN_TYPE_DB_SCOPE = 3000, TRN_TYPE_CREATE_DB = 3001, TRN_TYPE_ALTER_DB = 3002, @@ -102,6 +104,7 @@ typedef enum { TRN_TYPE_SPLIT_VGROUP = 3004, TRN_TYPE_MERGE_VGROUP = 3015, TRN_TYPE_DB_SCOPE_END, + TRN_TYPE_STB_SCOPE = 4000, TRN_TYPE_CREATE_STB = 4001, TRN_TYPE_ALTER_STB = 4002, @@ -131,7 +134,7 @@ typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; - ETrnType transType; + ETrnType type; int32_t code; int32_t failedTimes; void* rpcHandle; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5a2f37429a..eb946e9f53 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -126,7 +126,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->transType, TRANS_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, TRANS_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) @@ -237,7 +237,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &type, _OVER) pTrans->policy = policy; pTrans->stage = stage; - pTrans->transType = type; + pTrans->type = type; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) @@ -540,7 +540,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->transType = type; + pTrans->type = type; pTrans->createdTime = taosGetTimestampMs(); pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; @@ -592,7 +592,7 @@ void mndTransDrop(STrans *pTrans) { static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_INVALID_PARA; return -1; } @@ -689,12 +689,12 @@ static bool mndIsStbTrans(STrans *pTrans) { return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; } -static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewTrans) { - if (mndIsBasicTrans(pNewTrans)) return 0; - +static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - int32_t code = 0; + bool canParallel = true; + + if (mndIsBasicTrans(pNewTrans)) return canParallel; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -703,7 +703,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -712,13 +712,13 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsBasicTrans(pTrans)) continue; if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; + canParallel = false; break; } if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -728,13 +728,13 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsBasicTrans(pTrans)) continue; if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - code = -1; + canParallel = false; break; } if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - code = -1; + canParallel = false; break; } } @@ -746,12 +746,12 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return code; + return canParallel; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (mndCheckTransCanBeStartedInParallel(pMnode, pTrans) != 0) { - terrno = TSDB_CODE_MND_TRANS_CANT_PARALLEL; + if (!mndCheckTransCanParallel(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -1360,10 +1360,10 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char transType[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->transType), pShow->pMeta->pSchemas[cols].bytes); + char type[TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndTransType(pTrans->type), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)transType, false); + colDataAppend(pColInfo, numOfRows, (const char *)type, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->lastExecTime, false); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 709efa3125..bd787d50b0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -271,7 +271,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL, "Conflicting transaction not completed") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") From c734e2d9a19ed6033ff02fd0a93b9d830e6a261e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 May 2022 07:15:50 +0000 Subject: [PATCH 71/90] make case pass --- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 7d02840600..878dd5fca4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -426,10 +426,10 @@ static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, i } // process request - if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { - rcode = terrno; - goto _exit; - } + // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + // rcode = terrno; + // goto _exit; + // } // return rsp _exit: From 1dd08f30e6250aee6871c17504c9d9cd5515d99a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 15:47:01 +0800 Subject: [PATCH 72/90] fix: only write three sages prepare, rollback, and commit of trans to sdb --- source/dnode/mnode/impl/inc/mndDef.h | 10 +-- source/dnode/mnode/impl/src/mndTrans.c | 105 ++++++++++++++----------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e48a9b83c0..a3c143df41 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -57,11 +57,11 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT = 3, - TRN_STAGE_COMMIT_LOG = 4, - TRN_STAGE_UNDO_ACTION = 5, - TRN_STAGE_UNDO_LOG = 6, - TRN_STAGE_ROLLBACK = 7, + TRN_STAGE_ROLLBACK = 3, + TRN_STAGE_UNDO_ACTION = 4, + TRN_STAGE_UNDO_LOG = 5, + TRN_STAGE_COMMIT = 6, + TRN_STAGE_COMMIT_LOG = 7, TRN_STAGE_FINISHED = 8 } ETrnStage; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index eb946e9f53..f3d7bf8c66 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -123,71 +123,82 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { } int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pTrans->type, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, TRANS_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, redoActionNum, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, undoActionNum, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->id, _OVER) + + ETrnStage stage = pTrans->stage; + if (stage == TRN_STAGE_REDO_LOG || stage == TRN_STAGE_REDO_ACTION) { + stage = TRN_STAGE_PREPARE; + } else if (stage == TRN_STAGE_UNDO_ACTION || stage == TRN_STAGE_UNDO_LOG) { + stage = TRN_STAGE_ROLLBACK; + } else if (stage == TRN_STAGE_COMMIT_LOG || stage == TRN_STAGE_FINISHED) { + stage = TRN_STAGE_COMMIT; + } else { + } + + SDB_SET_INT16(pRaw, dataPos, stage, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->policy, _OVER) + SDB_SET_INT16(pRaw, dataPos, pTrans->type, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pTrans->dbUid, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, commitLogNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, redoActionNum, _OVER) + SDB_SET_INT32(pRaw, dataPos, undoActionNum, _OVER) for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); - SDB_SET_INT32(pRaw, dataPos, len, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, len, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len, _OVER) } for (int32_t i = 0; i < redoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) } for (int32_t i = 0; i < undoActionNum; ++i) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); - SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) - SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) + SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, _OVER) } - SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, _OVER) if (pTrans->param != NULL) { - SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -TRANS_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr()); sdbFreeRaw(pRaw); @@ -229,14 +240,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->id, _OVER) - int16_t type = 0; - int16_t policy = 0; int16_t stage = 0; - SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) + int16_t policy = 0; + int16_t type = 0; SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) + SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) SDB_GET_INT16(pRaw, dataPos, &type, _OVER) - pTrans->policy = policy; pTrans->stage = stage; + pTrans->policy = policy; pTrans->type = type; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->dbUid, _OVER) @@ -453,7 +464,6 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); if (pTrans->startFunc > 0) { @@ -516,8 +526,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { } static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); + STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; } @@ -558,7 +567,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -585,7 +594,7 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local object is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } @@ -674,19 +683,19 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { } static bool mndIsBasicTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_BASIC_SCOPE && pTrans->stage < TRN_TYPE_BASIC_SCOPE_END; + return pTrans->type > TRN_TYPE_BASIC_SCOPE && pTrans->type < TRN_TYPE_BASIC_SCOPE_END; } static bool mndIsGlobalTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_GLOBAL_SCOPE && pTrans->stage < TRN_TYPE_GLOBAL_SCOPE_END; + return pTrans->type > TRN_TYPE_GLOBAL_SCOPE && pTrans->type < TRN_TYPE_GLOBAL_SCOPE_END; } static bool mndIsDbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_DB_SCOPE && pTrans->stage < TRN_TYPE_DB_SCOPE_END; + return pTrans->type > TRN_TYPE_DB_SCOPE && pTrans->type < TRN_TYPE_DB_SCOPE_END; } static bool mndIsStbTrans(STrans *pTrans) { - return pTrans->stage > TRN_TYPE_STB_SCOPE && pTrans->stage < TRN_TYPE_STB_SCOPE_END; + return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { From 7eede87869b466e804645d565ea1f42fa27891d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 16:16:35 +0800 Subject: [PATCH 73/90] fix: reset confict trans types --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndTrans.c | 61 +++++++++++-------- source/dnode/mnode/impl/test/trans/trans2.cpp | 2 +- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a3c143df41..d516b0bf26 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,6 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3d7bf8c66..ce2a0efd2f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -63,13 +63,15 @@ static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + SSdbTable table = { + .sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); @@ -411,6 +413,16 @@ static const char *mndTransType(ETrnType type) { return "subscribe"; case TRN_TYPE_REBALANCE: return "rebalance"; + case TRN_TYPE_COMMIT_OFFSET: + return "commit-offset"; + case TRN_TYPE_CREATE_STREAM: + return "create-stream"; + case TRN_TYPE_DROP_STREAM: + return "drop-stream"; + case TRN_TYPE_CONSUMER_LOST: + return "consumer-lost"; + case TRN_TYPE_CONSUMER_RECOVER: + return "consumer-recover"; case TRN_TYPE_CREATE_DNODE: return "create-qnode"; case TRN_TYPE_DROP_DNODE: @@ -713,41 +725,34 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; + } else { } } - if (mndIsDbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); canParallel = false; - break; - } - if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { + } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; } + } else { } } - if (mndIsStbTrans(pNewTrans)) { - if (mndIsBasicTrans(pTrans)) continue; + else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); canParallel = false; - break; - } - if (mndIsDbTrans(pTrans)) { + } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); canParallel = false; - break; } + } else { } - if (mndIsStbTrans(pTrans)) continue; } sdbRelease(pMnode->pSdb, pTrans); @@ -842,12 +847,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, - .code = pTrans->code, - .ahandle = pTrans->rpcAHandle, - .refId = pTrans->rpcRefId, - .pCont = rpcCont, - .contLen = pTrans->rpcRspLen}; + SRpcMsg rspMsg = { + .handle = pTrans->rpcHandle, + .ahandle = pTrans->rpcAHandle, + .refId = pTrans->rpcRefId, + .code = pTrans->code, + .pCont = rpcCont, + .contLen = pTrans->rpcRspLen, + }; tmsgSendRsp(&rspMsg); pTrans->rpcHandle = NULL; pTrans->rpcRsp = NULL; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index abab24723f..252cd105a0 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -22,7 +22,7 @@ class MndTestTrans2 : public ::testing::Test { static void SetUpTestSuite() { dDebugFlag = 143; vDebugFlag = 0; - mDebugFlag = 143; + mDebugFlag = 207; cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 135; From 5b2b4da11bf0c9cd5604a456b35ec1d83b7275a8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 16:49:24 +0800 Subject: [PATCH 74/90] refactor: fix: reset confict trans types --- include/util/taoserror.h | 3 ++- source/dnode/mnode/impl/src/mndTrans.c | 31 ++++++++++++++------------ source/util/src/terror.c | 3 ++- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b8baf99552..abc752955d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -264,7 +264,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) #define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) #define TSDB_CODE_MND_TRANS_INVALID_STAGE TAOS_DEF_ERROR_CODE(0, 0x03D2) -#define TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL TAOS_DEF_ERROR_CODE(0, 0x03D4) +#define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) +#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03D4) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ce2a0efd2f..1c238d4082 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -710,12 +710,12 @@ static bool mndIsStbTrans(STrans *pTrans) { return pTrans->type > TRN_TYPE_STB_SCOPE && pTrans->type < TRN_TYPE_STB_SCOPE_END; } -static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { +static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNewTrans) { STrans *pTrans = NULL; void *pIter = NULL; - bool canParallel = true; + bool conflict = false; - if (mndIsBasicTrans(pNewTrans)) return canParallel; + if (mndIsBasicTrans(pNewTrans)) return conflict; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -724,7 +724,7 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { if (mndIsGlobalTrans(pNewTrans)) { if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } else { } } @@ -732,11 +732,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { else if (mndIsDbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - canParallel = false; + conflict = true; } else if (mndIsDbTrans(pTrans) || mndIsStbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } } else { } @@ -745,11 +745,11 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { else if (mndIsStbTrans(pNewTrans)) { if (mndIsGlobalTrans(pTrans)) { mError("trans:%d, can't execute since trans:%d in progress", pNewTrans->id, pTrans->id); - canParallel = false; + conflict = true; } else if (mndIsDbTrans(pTrans)) { if (pNewTrans->dbUid == pTrans->dbUid) { mError("trans:%d, can't execute since trans:%d in progress db:%s", pNewTrans->id, pTrans->id, pTrans->dbname); - canParallel = false; + conflict = true; } } else { } @@ -760,12 +760,12 @@ static bool mndCheckTransCanParallel(SMnode *pMnode, STrans *pNewTrans) { sdbCancelFetch(pMnode->pSdb, pIter); sdbRelease(pMnode->pSdb, pTrans); - return canParallel; + return conflict; } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { - if (!mndCheckTransCanParallel(pMnode, pTrans)) { - terrno = TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL; + if (mndCheckTransConflict(pMnode, pTrans)) { + terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); return -1; } @@ -819,7 +819,8 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { - bool sendRsp = false; + bool sendRsp = false; + int32_t code = pTrans->code; if (pTrans->stage == TRN_STAGE_FINISHED) { sendRsp = true; @@ -829,12 +830,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { sendRsp = true; + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; } } if (pTrans->policy == TRN_POLICY_RETRY) { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { sendRsp = true; + if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; } } @@ -845,13 +848,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = { .handle = pTrans->rpcHandle, .ahandle = pTrans->rpcAHandle, .refId = pTrans->rpcRefId, - .code = pTrans->code, + .code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, }; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bd787d50b0..00fe8bd0e9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -271,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CAN_NOT_PARALLEL, "Conflicting transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") From 71021334fb6a928f38a0e926010e6ce51c7388b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 20:37:37 +0800 Subject: [PATCH 75/90] refactor: adjust transaction code --- source/dnode/mnode/impl/src/mndTrans.c | 12 +++--- source/dnode/mnode/impl/src/mndUser.c | 3 -- source/dnode/mnode/impl/test/trans/trans2.cpp | 39 ++++++++++++++----- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1c238d4082..e80a16b16e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -829,15 +829,13 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->policy == TRN_POLICY_ROLLBACK) { if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { - sendRsp = true; if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; + sendRsp = true; } - } - - if (pTrans->policy == TRN_POLICY_RETRY) { + } else { if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { - sendRsp = true; if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; + sendRsp = true; } } @@ -1102,8 +1100,8 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_REDO_ACTION; - mError("trans:%d, stage from commit to redoAction since %s, failedTimes:%d", pTrans->id, terrstr(), + pTrans->stage = TRN_STAGE_UNDO_ACTION; + mError("trans:%d, stage from commit to undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = true; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1800d24cf2..5e15bdeb43 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -275,9 +275,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test code to be deleted later <====="); - mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 252cd105a0..aaf2c649e4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -19,7 +19,7 @@ void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) { class MndTestTrans2 : public ::testing::Test { protected: - static void SetUpTestSuite() { + static void InitLog() { dDebugFlag = 143; vDebugFlag = 0; mDebugFlag = 207; @@ -42,9 +42,9 @@ class MndTestTrans2 : public ::testing::Test { if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } + } - walInit(); - + static void InitMnode() { static SMsgCb msgCb = {0}; msgCb.reportStartupFp = reportStartup; msgCb.pWrapper = (SMgmtWrapper *)(&msgCb); // hack @@ -63,6 +63,12 @@ class MndTestTrans2 : public ::testing::Test { pMnode = mndOpen(mnodepath, &opt); } + static void SetUpTestSuite() { + InitLog(); + walInit(); + InitMnode(); + } + static void TearDownTestSuite() { mndClose(pMnode); walCleanUp(); @@ -76,11 +82,11 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void CreateUser(const char *user) { + int32_t CreateUser(const char *acct, const char *user) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); - tstrncpy(userObj.acct, "root", TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = 1; @@ -94,19 +100,32 @@ class MndTestTrans2 : public ::testing::Test { char *param = strdup("====> test param <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); - mndTransPrepare(pMnode, pTrans); + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); + + return code; } }; SMnode *MndTestTrans2::pMnode; TEST_F(MndTestTrans2, 01_CbFunc) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "test1"; + const char *user2 = "test2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + ASSERT_NE(pMnode, nullptr); - const char *user1 = "test1"; - CreateUser(user1); - - SUserObj *pUser1 = mndAcquireUser(pMnode, user1); + // create user success + EXPECT_EQ(CreateUser(acct, user1), 0); + pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUser(acct_invalid, user2), 0); + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); } From fb7da7e566bedc0af30758785c29c30a7a4b2999 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 4 May 2022 21:02:27 +0800 Subject: [PATCH 76/90] enh(index): support index filter --- source/libs/index/inc/indexCache.h | 1 - source/libs/index/inc/indexFst.h | 7 ++-- source/libs/index/inc/indexInt.h | 1 + source/libs/index/src/indexCache.c | 60 ++++++++++++++++++++++++++---- source/libs/index/src/indexTfile.c | 43 +++++++++++++++++---- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 44355db7a7..3ea986ad48 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -56,7 +56,6 @@ typedef struct CacheTerm { int8_t colType; SIndexOperOnColumn operaType; - int8_t qType; // query type } CacheTerm; // diff --git a/source/libs/index/inc/indexFst.h b/source/libs/index/inc/indexFst.h index 39ad5ffa8c..0a360c1c72 100644 --- a/source/libs/index/inc/indexFst.h +++ b/source/libs/index/inc/indexFst.h @@ -52,7 +52,6 @@ typedef struct FstRange { uint64_t end; } FstRange; -typedef enum { GE, GT, LE, LT } RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; @@ -174,9 +173,9 @@ Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) -#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) -#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) -#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) +#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) +#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) +#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) typedef struct FstLastTransition { uint8_t inp; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 04817c4df3..5c7b8b9afe 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -34,6 +34,7 @@ extern "C" { #endif +typedef enum { LT, LE, GT, GE } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ea3cbf5658..13768ce682 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -43,6 +43,20 @@ static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +/*comm func of compare, used in (LE/LT/GE/GT compare)*/ +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type); + +typedef enum { MATCH, CONTINUE, BREAK } TExeCond; +typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); + +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } + +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, + tCompareGreaterThan, tCompareGreaterEqual}; static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, @@ -93,21 +107,51 @@ static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, // impl later return 0; } +static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + char* key = indexCacheTermGet(ct); + + SSkipListIterator* iter = tSkipListCreateIter(mem->mem); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + } else if (cond == BREAK) { + break; + } + } + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, LT); } static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, LE); } static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, GT); } static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - // impl later - return 0; + return cacheSearchCompareFunc(cache, ct, tr, s, GE); } static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { // impl later diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 327a06084c..5aed2bd6b0 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -70,6 +70,8 @@ static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr); static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr); +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype); + static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = { tfSearchTerm, tfSearchPrefix, tfSearchSuffix, tfSearchRegex, tfSearchLessThan, tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange}; @@ -304,21 +306,46 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) fstSliceDestroy(&key); return 0; } + +static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + uint64_t sz = tem->nColVal; + if (hasJson) { + p = indexPackJsonData(tem); + sz = strlen(p); + } + SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); + + AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); + FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); + + FstSlice h = fstSliceCreate((uint8_t*)p, sz); + fstStreamBuilderSetRange(sb, &h, type); + fstSliceDestroy(&h); + + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + taosArrayPush(offsets, &(rt->out.out)); + swsResultDestroy(rt); + } + streamWithStateDestroy(st); + fstStreamBuilderDestroy(sb); + return TSDB_CODE_SUCCESS; +} static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, LT); } static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, LE); } static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, GT); } static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { - // impl later - return 0; + return tfSearchCompareFunc(reader, tem, tr, GE); } static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); From 2f42c2e7933f2231c15f2236fe17b86d70c49fca Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 22:00:04 +0800 Subject: [PATCH 77/90] refactor: adjust transaction code --- include/common/tglobal.h | 4 ++ source/common/src/tglobal.c | 4 ++ source/common/src/tmsgcb.c | 62 ++++++++++++++++--- source/dnode/mnode/impl/src/mndTrans.c | 6 +- source/dnode/mnode/impl/src/mnode.c | 11 ++-- source/dnode/mnode/impl/test/trans/trans2.cpp | 6 ++ 6 files changed, 75 insertions(+), 18 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fc3d575317..f253d31963 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -121,6 +121,10 @@ extern char tsCompressor[]; extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; +// internal +extern int32_t tsTransPullupMs; +extern int32_t tsMaRebalanceMs; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bbf14b2fdc..4470f5b0a3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR +// internal +int32_t tsTransPullupMs = 6000; +int32_t tsMaRebalanceMs = 2000; + void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); tsDiskCfg[index].level = level; diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index c9cbb73884..42612cecb9 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -15,37 +15,83 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" +#include "taoserror.h" static SMsgCb tsDefaultMsgCb; void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { - return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); + PutToQueueFp fp = pMsgCb->queueFps[qtype]; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype); + GetQueueSizeFp fp = pMsgCb->qsizeFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, vgId, qtype); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) { - return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); + SendReqFp fp = pMsgCb->sendReqFp; + if (fp != NULL) { + return (*fp)(pMsgCb->pWrapper, epSet, pReq); + } else { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } } -void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); } +void tmsgSendRsp(const SRpcMsg* pRsp) { + SendRspFp fp = tsDefaultMsgCb.sendRspFp; + if (fp != NULL) { + return (*fp)(tsDefaultMsgCb.pWrapper, pRsp); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } +} void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) { - return (*tsDefaultMsgCb.sendRedirectRspFp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) { - (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg); + RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp; + if (fp != NULL) { + (*fp)(pMsgCb->pWrapper, pMsg); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReleaseHandle(void* handle, int8_t type) { - (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type); + ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + if (fp != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, handle, type); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } void tmsgReportStartup(const char* name, const char* desc) { - (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc); + ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) { + (*fp)(tsDefaultMsgCb.pWrapper, name, desc); + } else { + terrno = TSDB_CODE_INVALID_PTR; + } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e80a16b16e..6110d0be44 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1126,7 +1126,7 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; pTrans->failedTimes++; - mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on commitLog since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1162,7 +1162,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { continueExec = false; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on undoAction since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on undoAction since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } @@ -1179,7 +1179,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { continueExec = true; } else { pTrans->failedTimes++; - mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on rollback since %s, failedTimes:%d", pTrans->id, terrstr(), pTrans->failedTimes); continueExec = false; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index b38fd6178c..e2814e95f0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,9 +43,6 @@ #include "mndUser.h" #include "mndVgroup.h" -#define MQ_TIMER_MS 2000 -#define TRNAS_TIMER_MS 6000 - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } - taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer); } static void mndPullupTelem(void *param, void *tmrId) { @@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) { return -1; } - if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index aaf2c649e4..1fabf677c4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -58,9 +58,12 @@ class MndTestTrans2 : public ::testing::Test { strcpy(opt.replicas[0].fqdn, "localhost"); opt.msgCb = msgCb; + tsTransPullupMs = 1000; + const char *mnodepath = "/tmp/mnode_test_trans"; taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); + mndStart(pMnode); } static void SetUpTestSuite() { @@ -70,6 +73,7 @@ class MndTestTrans2 : public ::testing::Test { } static void TearDownTestSuite() { + mndStop(pMnode); mndClose(pMnode); walCleanUp(); taosCloseLog(); @@ -128,4 +132,6 @@ TEST_F(MndTestTrans2, 01_CbFunc) { EXPECT_EQ(CreateUser(acct_invalid, user2), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); + + mndTransPullup(pMnode); } From 0218def49a41c7391d762ce6ee544a02292b4747 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 May 2022 23:58:42 +0800 Subject: [PATCH 78/90] test: add action test for transaction --- source/dnode/mnode/impl/inc/mndTrans.h | 2 + source/dnode/mnode/impl/src/mndTrans.c | 45 ++++-- source/dnode/mnode/impl/test/trans/trans2.cpp | 128 ++++++++++++++++-- 3 files changed, 156 insertions(+), 19 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 7644ec3c4c..2a011f414d 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -44,6 +44,8 @@ typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId); +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans); STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6110d0be44..c5a31a6d85 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -537,7 +537,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { return 0; } -static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; @@ -545,7 +545,7 @@ static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { return pTrans; } -static void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTrans); } @@ -919,27 +919,41 @@ static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { if (arraySize == 0) return 0; + int32_t code = 0; for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWriteWithoutFree(pSdb, pRaw); - if (code != 0) { - return code; + if (sdbWriteWithoutFree(pSdb, pRaw) != 0) { + code = ((terrno != 0) ? terrno : -1); } } - return 0; + terrno = code; + return code; } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->redoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("failed to execute redoLogs since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->undoLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("failed to execute undoLogs since %s, return success", terrstr()); + } + + return 0; // return success in any case } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteLogs(pMnode, pTrans->commitLogs); + int32_t code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("failed to execute commitLogs since %s", terrstr()); + } + return code; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -983,6 +997,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } @@ -1029,11 +1044,19 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); + if (code != 0) { + mError("failed to execute redoActions since %s", terrstr()); + } + return code; } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); + if (code != 0) { + mError("failed to execute undoActions since %s", terrstr()); + } + return code; } static bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 1fabf677c4..e9d3b54c1d 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -86,7 +86,7 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - int32_t CreateUser(const char *acct, const char *user) { + int32_t CreateUserLog(const char *acct, const char *user) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -101,7 +101,11 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - char *param = strdup("====> test param <====="); + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); int32_t code = mndTransPrepare(pMnode, pTrans); @@ -109,29 +113,137 @@ class MndTestTrans2 : public ::testing::Test { return code; } + + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test action <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + + { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world redo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendRedoAction(pTrans, &action); + } + + if (hasUndoAction) { + STransAction action = {0}; + action.epSet.inUse = 0; + action.epSet.numOfEps = 1; + action.epSet.eps[0].port = 9040; + strcpy(action.epSet.eps[0].fqdn, "localhost"); + + int32_t contLen = 1024; + void *pReq = taosMemoryCalloc(1, contLen); + strcpy((char *)pReq, "hello world undo"); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_MNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + mndTransAppendUndoAction(pTrans, &action); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } }; SMnode *MndTestTrans2::pMnode; -TEST_F(MndTestTrans2, 01_CbFunc) { +TEST_F(MndTestTrans2, 01_Log) { const char *acct = "root"; const char *acct_invalid = "root1"; - const char *user1 = "test1"; - const char *user2 = "test2"; + const char *user1 = "log1"; + const char *user2 = "log2"; SUserObj *pUser1 = NULL; SUserObj *pUser2 = NULL; ASSERT_NE(pMnode, nullptr); - // create user success - EXPECT_EQ(CreateUser(acct, user1), 0); + EXPECT_EQ(CreateUserLog(acct, user1), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUser(acct_invalid, user2), 0); + EXPECT_EQ(CreateUserLog(acct_invalid, user2), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); mndTransPullup(pMnode); } + +TEST_F(MndTestTrans2, 02_Action) { + const char *acct = "root"; + const char *acct_invalid = "root1"; + const char *user1 = "action1"; + const char *user2 = "action2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 4; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + EXPECT_EQ(CreateUserAction(acct, user1, true), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); +} From 43822106bc35ee92e930739244fdcd2d6c9bca2b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 00:12:11 +0800 Subject: [PATCH 79/90] test: add unitest for transaction --- source/dnode/mnode/impl/test/trans/trans2.cpp | 109 ++++++++++++++---- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index e9d3b54c1d..599f1882d4 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -114,7 +114,7 @@ class MndTestTrans2 : public ::testing::Test { return code; } - int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction) { + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -124,7 +124,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, TRN_TYPE_CREATE_USER, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -209,41 +209,100 @@ TEST_F(MndTestTrans2, 02_Action) { SUserObj *pUser1 = NULL; SUserObj *pUser2 = NULL; STrans *pTrans = NULL; - int32_t transId = 4; + int32_t transId = 0; int32_t action = 0; ASSERT_NE(pMnode, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); // create user, and fake a response - EXPECT_EQ(CreateUserAction(acct, user1, true), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_NE(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); - pTrans = mndAcquireTrans(pMnode, transId); - EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); - EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); - EXPECT_EQ(pTrans->failedTimes, 1); + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); - STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); - pAction->msgSent = 1; + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; - SNodeMsg rspMsg = {0}; - rspMsg.pNode = pMnode; - int64_t signature = transId; - signature = (signature << 32); - signature += action; - rspMsg.rpcMsg.ahandle = (void *)signature; - mndTransProcessRsp(&rspMsg); - mndReleaseTrans(pMnode, pTrans); + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_EQ(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + + { + transId = 5; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } + } } From 00b29f06770d97476e25ab12f3ad08bacf14ad51 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 5 May 2022 09:36:11 +0800 Subject: [PATCH 80/90] feature/qnode --- tests/script/api/batchprepare.c | 828 ++++---------------------------- 1 file changed, 94 insertions(+), 734 deletions(-) diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7bea19ae90..828216d1d3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -462,6 +462,29 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { } } +void generateErrorSQL(BindData *data, int32_t tblIdx) { + int32_t len = 0; + data->sql = taosMemoryCalloc(1, 1024); + + switch(tblIdx) { + case 0: + len = sprintf(data->sql, "insert into %s%d values (?, 1)", bpTbPrefix, tblIdx); + break; + case 1: + len = sprintf(data->sql, "select * from ?"); + break; + case 2: + len = sprintf(data->sql, "select * from %s%d where ? = ?", bpTbPrefix, tblIdx); + break; + default: + len = sprintf(data->sql, "select count(*) from %s%d group by ?", bpTbPrefix, tblIdx); + break; + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} void generateDataType(BindData *data, int32_t bindIdx, int32_t colIdx, int32_t *dataType) { if (bindIdx < gCurCase->bindColNum) { @@ -712,6 +735,8 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } + + void destroyData(BindData *data) { taosMemoryFree(data->tsData); taosMemoryFree(data->boolData); @@ -1354,6 +1379,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* select * from table */ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -1399,6 +1425,74 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* value in query sql */ +int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + prepareQueryData(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + for (int32_t n = 0; n< (gCurCase->rowNum/gCurCase->bindRowNum); ++n) { + bpCheckIsInsert(stmt, 0); + + if (gCaseCtrl.checkParamNum) { + bpCheckParamNum(stmt); + } + + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckQueryResult(stmt, taos, data.sql, data.pBind + n * gCurCase->bindColNum); + } + + bpCheckIsInsert(stmt, 0); + + destroyData(&data); + } + + return 0; +} + + +int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + memset(&data, 0, sizeof(data)); + generateErrorSQL(&data, t); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("*taos_stmt_prepare error as expected, error:%s\n", taos_stmt_errstr(stmt)); + } else { + printf("!!!taos_stmt_prepare succeed, which should be error\n"); + exit(1); + } + + destroyData(&data); + } + + return 0; +} + #if 0 @@ -1423,153 +1517,6 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2383,153 +2330,6 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -2600,152 +2400,6 @@ int stmt_funcb_autoctb_e3(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - unsigned long long starttime = taosGetTimestampUs(); @@ -2815,153 +2469,6 @@ int stmt_funcb_autoctb_e4(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; @@ -3042,153 +2549,6 @@ int stmt_funcb_autoctb_e5(TAOS_STMT *stmt) { TAOS_BIND *tags = taosMemoryCalloc(1, sizeof(TAOS_BIND) * 9 * 1); TAOS_MULTI_BIND *params = taosMemoryCalloc(1, sizeof(TAOS_MULTI_BIND) * 1*10); -// int one_null = 1; - int one_not_null = 0; - - char* is_null = taosMemoryMalloc(sizeof(char) * 10); - char* no_null = taosMemoryMalloc(sizeof(char) * 10); - - for (int i = 0; i < 10; ++i) { - lb[i] = 40; - no_null[i] = 0; - is_null[i] = (i % 10 == 2) ? 1 : 0; - v.b[i] = (int8_t)(i % 2); - v.v1[i] = (int8_t)((i+1) % 2); - v.v2[i] = (int16_t)i; - v.v4[i] = (int32_t)(i+1); - v.v8[i] = (int64_t)(i+2); - v.f4[i] = (float)(i+3); - v.f8[i] = (double)(i+4); - memset(v.bin[i], '0'+i%10, 40); - } - - for (int i = 0; i < 10; i+=10) { - params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[i+0].buffer_length = sizeof(int64_t); - params[i+0].buffer = &v.ts[10*i/10]; - params[i+0].length = NULL; - params[i+0].is_null = no_null; - params[i+0].num = 10; - - params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[i+1].buffer_length = sizeof(int8_t); - params[i+1].buffer = v.b; - params[i+1].length = NULL; - params[i+1].is_null = is_null; - params[i+1].num = 10; - - params[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[i+2].buffer_length = sizeof(int8_t); - params[i+2].buffer = v.v1; - params[i+2].length = NULL; - params[i+2].is_null = is_null; - params[i+2].num = 10; - - params[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[i+3].buffer_length = sizeof(int16_t); - params[i+3].buffer = v.v2; - params[i+3].length = NULL; - params[i+3].is_null = is_null; - params[i+3].num = 10; - - params[i+4].buffer_type = TSDB_DATA_TYPE_INT; - params[i+4].buffer_length = sizeof(int32_t); - params[i+4].buffer = v.v4; - params[i+4].length = NULL; - params[i+4].is_null = is_null; - params[i+4].num = 10; - - params[i+5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[i+5].buffer_length = sizeof(int64_t); - params[i+5].buffer = v.v8; - params[i+5].length = NULL; - params[i+5].is_null = is_null; - params[i+5].num = 10; - - params[i+6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[i+6].buffer_length = sizeof(float); - params[i+6].buffer = v.f4; - params[i+6].length = NULL; - params[i+6].is_null = is_null; - params[i+6].num = 10; - - params[i+7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[i+7].buffer_length = sizeof(double); - params[i+7].buffer = v.f8; - params[i+7].length = NULL; - params[i+7].is_null = is_null; - params[i+7].num = 10; - - params[i+8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+8].buffer_length = 40; - params[i+8].buffer = v.bin; - params[i+8].length = lb; - params[i+8].is_null = is_null; - params[i+8].num = 10; - - params[i+9].buffer_type = TSDB_DATA_TYPE_BINARY; - params[i+9].buffer_length = 40; - params[i+9].buffer = v.bin; - params[i+9].length = lb; - params[i+9].is_null = is_null; - params[i+9].num = 10; - - } - - int64_t tts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = tts + i; - } - - - for (int i = 0; i < 1; ++i) { - tags[i+0].buffer_type = TSDB_DATA_TYPE_INT; - tags[i+0].buffer = v.v4; - tags[i+0].is_null = &one_not_null; - tags[i+0].length = NULL; - - tags[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; - tags[i+1].buffer = v.b; - tags[i+1].is_null = &one_not_null; - tags[i+1].length = NULL; - - tags[i+2].buffer_type = TSDB_DATA_TYPE_TINYINT; - tags[i+2].buffer = v.v1; - tags[i+2].is_null = &one_not_null; - tags[i+2].length = NULL; - - tags[i+3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - tags[i+3].buffer = v.v2; - tags[i+3].is_null = &one_not_null; - tags[i+3].length = NULL; - - tags[i+4].buffer_type = TSDB_DATA_TYPE_BIGINT; - tags[i+4].buffer = v.v8; - tags[i+4].is_null = &one_not_null; - tags[i+4].length = NULL; - - tags[i+5].buffer_type = TSDB_DATA_TYPE_FLOAT; - tags[i+5].buffer = v.f4; - tags[i+5].is_null = &one_not_null; - tags[i+5].length = NULL; - - tags[i+6].buffer_type = TSDB_DATA_TYPE_DOUBLE; - tags[i+6].buffer = v.f8; - tags[i+6].is_null = &one_not_null; - tags[i+6].length = NULL; - - tags[i+7].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[i+7].buffer = v.bin; - tags[i+7].is_null = &one_not_null; - tags[i+7].length = (uintptr_t *)lb; - - tags[i+8].buffer_type = TSDB_DATA_TYPE_NCHAR; - tags[i+8].buffer = v.bin; - tags[i+8].is_null = &one_not_null; - tags[i+8].length = (uintptr_t *)lb; - } - - unsigned long long starttime = taosGetTimestampUs(); char *sql = "insert into ? using stb1 tags(?,?,?,?,?,?,?,?,?) values(?,?,?,?,?,?,?,?,?,?)"; From 7c5c1c32ee48b3a49728f654522e06f2936d7687 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 May 2022 03:17:34 +0000 Subject: [PATCH 81/90] implement TDB upsert --- source/libs/tdb/inc/tdb.h | 2 +- source/libs/tdb/src/db/tdbBtree.c | 148 ++++++++++++++++++++---------- source/libs/tdb/src/db/tdbDb.c | 5 +- source/libs/tdb/src/inc/tdbInt.h | 3 +- source/libs/tdb/test/tdbTest.cpp | 69 +++++++++++++- 5 files changed, 170 insertions(+), 57 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index cab242d5b3..9e79ec1d89 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -42,7 +42,7 @@ int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ed7aed552a..0d13ec4d2b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -138,67 +138,25 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in } if (btc.idx == -1) { - idx = 0; + btc.idx = 0; } else { if (c > 0) { - idx = btc.idx + 1; - } else if (c < 0) { - idx = btc.idx; - } else { - // TDB does NOT allow same key - tdbBtcClose(&btc); + btc.idx++; + } else if (c == 0) { + // dup key not allowed ASSERT(0); return -1; } } - // make sure enough space to hold the cell - szBuf = kLen + vLen + 14; - pBuf = tdbRealloc(pBt->pBuf, pBt->pageSize > szBuf ? szBuf : pBt->pageSize); - if (pBuf == NULL) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - pBt->pBuf = pBuf; - pCell = (SCell *)pBt->pBuf; - - // encode cell - ret = tdbBtreeEncodeCell(btc.pPage, pKey, kLen, pVal, vLen, pCell, &szCell); + ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - tdbBtcClose(&btc); ASSERT(0); - return -1; - } - - // mark the page dirty - ret = tdbPagerWrite(pBt->pPager, btc.pPage); - if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); return -1; } - // insert the cell - ret = tdbPageInsertCell(btc.pPage, idx, pCell, szCell, 0); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - - // check if need balance - if (btc.pPage->nOverflow > 0) { - ret = tdbBtreeBalance(&btc); - if (ret < 0) { - tdbBtcClose(&btc); - ASSERT(0); - return -1; - } - } - tdbBtcClose(&btc); - return 0; } @@ -232,6 +190,41 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { return 0; } +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn) { + SBTC btc; + int c; + int ret; + + tdbBtcOpen(&btc, pBt, pTxn); + + // move the cursor + ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + if (btc.idx == -1) { + btc.idx = 0; + c = 1; + } else { + if (c > 0) { + btc.idx = btc.idx + 1; + } + } + + ret = tdbBtcUpsert(&btc, pKey, nKey, pData, nData, c); + if (ret < 0) { + ASSERT(0); + tdbBtcClose(&btc); + return -1; + } + + tdbBtcClose(&btc); + return 0; +} + int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreePGet(pBt, pKey, kLen, NULL, NULL, ppVal, vLen); } @@ -1475,9 +1468,64 @@ int tdbBtcDelete(SBTC *pBtc) { return 0; } -int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen) { - ASSERT(0); - // TODO +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert) { + SCell *pCell; + int szCell; + int nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + int szBuf; + void *pBuf; + int ret; + + ASSERT(pBtc->idx >= 0); + + // alloc space + szBuf = kLen + nData + 14; + pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); + if (pBuf == NULL) { + ASSERT(0); + return -1; + } + pBtc->pBt->pBuf = pBuf; + pCell = (SCell *)pBtc->pBt->pBuf; + + // encode cell + ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // mark dirty + ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + // insert or update + if (insert) { + ASSERT(pBtc->idx <= nCells); + + ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); + } else { + ASSERT(pBtc->idx < nCells); + + ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell); + } + if (ret < 0) { + ASSERT(0); + return -1; + } + + // check balance + if (pBtc->pPage->nOverflow > 0) { + ret = tdbBtreeBalance(pBtc); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 3358aefec0..c31441d849 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -81,9 +81,8 @@ int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int va int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } -int tdbUpsert(TDB *pTDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { - // TODO - return 0; +int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); } int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index e18952505c..76dacf7b84 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -129,6 +129,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBT int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); +int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); @@ -143,7 +144,7 @@ int tdbBtcMoveToPrev(SBTC *pBtc); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen); int tdbBtcDelete(SBTC *pBtc); -int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert); // tdbPager.c ==================================== diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 994e53cd9d..e575ac156f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -115,7 +115,7 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -TEST(tdb_test, simple_test) { +TEST(tdb_test, simple_insert1) { int ret; TENV *pEnv; TDB *pDb; @@ -235,7 +235,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, simple_test2) { +TEST(tdb_test, simple_insert2) { int ret; TENV *pEnv; TDB *pDb; @@ -413,6 +413,71 @@ TEST(tdb_test, simple_delete1) { closePool(pPool); + tdbDbClose(pDb); + tdbEnvClose(pEnv); +} + +TEST(tdb_test, simple_upsert1) { + int ret; + TENV *pEnv; + TDB *pDb; + int nData = 100000; + char key[64]; + char data[64]; + void *pData = NULL; + SPoolMem *pPool; + TXN txn; + + taosRemoveDir("tdb"); + + // open env + ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + GTEST_ASSERT_EQ(ret, 0); + + // open database + ret = tdbDbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + // insert some data + tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn); + + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + + // upsert some data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, &txn); + + // query the data + for (int iData = 0; iData < nData; iData++) { + sprintf(key, "key%d", iData); + sprintf(data, "data%d-u", iData); + ret = tdbDbGet(pDb, key, strlen(key), &pData, &nData); + GTEST_ASSERT_EQ(ret, 0); + GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); + } + tdbDbClose(pDb); tdbEnvClose(pEnv); } \ No newline at end of file From c87eb6cc479353f85a60558d418fa0e736176f76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 May 2022 03:24:00 +0000 Subject: [PATCH 82/90] refact --- source/libs/tdb/inc/tdb.h | 4 +--- source/libs/tdb/src/db/tdbDb.c | 16 ++++------------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 9e79ec1d89..18f8ddec5c 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -55,11 +55,9 @@ int tdbDbcMoveToLast(TDBC *pDbc); int tdbDbcMoveToNext(TDBC *pDbc); int tdbDbcMoveToPrev(TDBC *pDbc); int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); - -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen); int tdbDbcDelete(TDBC *pDbc); int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c31441d849..553bb2c646 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -123,24 +123,16 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); } -int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { - // TODO - ASSERT(0); - return 0; -} - -int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) { - // TODO - ASSERT(0); - return 0; -} - int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } +int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); +} + int tdbDbcClose(TDBC *pDbc) { if (pDbc) { tdbBtcClose(&pDbc->btc); From af7000e3dc2ae6cae7583f0f4ee07f47b8053323 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 May 2022 11:48:52 +0800 Subject: [PATCH 83/90] feat: update taos-tools for 3.0 (#12112) [TD-13052] --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index bf6c766986..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c From 5191a1ca91e74b0f357f73a0190e559c8aadcf38 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 13:13:23 +0800 Subject: [PATCH 84/90] test: add rollback test for transaction --- source/dnode/mnode/impl/src/mndTrans.c | 3 + source/dnode/mnode/impl/test/trans/trans2.cpp | 121 +++++++++++++----- 2 files changed, 94 insertions(+), 30 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c5a31a6d85..9144de82b3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -997,6 +997,9 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { + pAction->msgSent = 0; + pAction->msgReceived = 0; + pAction->errCode = terrno; if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 599f1882d4..e81827041b 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -170,6 +170,12 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendUndoAction(pTrans, &action); } + { + void *pRsp = taosMemoryCalloc(1, 256); + strcpy((char *)pRsp, "simple rsponse"); + mndTransSetRpcRsp(pTrans, pRsp, 256); + } + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); @@ -214,40 +220,42 @@ TEST_F(MndTestTrans2, 02_Action) { ASSERT_NE(pMnode, nullptr); - // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_EQ(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); - - // create user, and fake a response { - EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); - pUser1 = mndAcquireUser(pMnode, user1); - ASSERT_NE(pUser1, nullptr); - mndReleaseUser(pMnode, pUser1); - - transId = 4; - pTrans = mndAcquireTrans(pMnode, transId); - EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); - EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); - EXPECT_EQ(pTrans->failedTimes, 1); - - STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); - pAction->msgSent = 1; - - SNodeMsg rspMsg = {0}; - rspMsg.pNode = pMnode; - int64_t signature = transId; - signature = (signature << 32); - signature += action; - rspMsg.rpcMsg.ahandle = (void *)signature; - mndTransProcessRsp(&rspMsg); - mndReleaseTrans(pMnode, pTrans); - + // failed to create user and rollback + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); + + // create user, and fake a response + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 4; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } } { @@ -305,4 +313,57 @@ TEST_F(MndTestTrans2, 02_Action) { mndReleaseUser(pMnode, pUser1); } } + + { + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK), 0); + SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + rspMsg.rpcMsg.code = 0; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_NE(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + + { + transId = 6; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 2); + + STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action); + pAction->msgSent = 1; + + SNodeMsg rspMsg = {0}; + rspMsg.pNode = pMnode; + int64_t signature = transId; + signature = (signature << 32); + signature += action; + rspMsg.rpcMsg.ahandle = (void *)signature; + mndTransProcessRsp(&rspMsg); + mndReleaseTrans(pMnode, pTrans); + + pUser2 = mndAcquireUser(pMnode, user2); + ASSERT_EQ(pUser2, nullptr); + mndReleaseUser(pMnode, pUser2); + } + } } From 1361bf4eedb39601d5b74e7c220150e31e05121b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 5 May 2022 14:49:00 +0800 Subject: [PATCH 85/90] fix: plan problem of tag scanning --- include/libs/parser/parser.h | 68 +++++++------- source/libs/function/src/builtins.c | 2 +- source/libs/parser/test/mockCatalog.cpp | 22 +++++ source/libs/planner/test/planSTableTest.cpp | 2 + source/libs/planner/test/planTestMain.cpp | 4 +- source/libs/planner/test/planTestUtil.cpp | 99 ++++++++++++++++----- source/libs/planner/test/planTestUtil.h | 2 +- 7 files changed, 142 insertions(+), 57 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index de71192a9b..84fa166881 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "querynodes.h" #include "query.h" +#include "querynodes.h" typedef struct SStmtCallback { TAOS_STMT* pStmt; @@ -34,24 +34,26 @@ typedef struct SStmtCallback { typedef struct SParseContext { uint64_t requestId; int32_t acctId; - const char *db; + const char* db; bool topicQuery; - void *pTransporter; + void* pTransporter; SEpSet mgmtEpSet; - const char *pSql; // sql string - size_t sqlLen; // length of the sql string - char *pMsg; // extended error message if exists to help identifying the problem in sql statement. - int32_t msgLen; // max length of the msg - struct SCatalog *pCatalog; - SStmtCallback *pStmtCb; + const char* pSql; // sql string + size_t sqlLen; // length of the sql string + char* pMsg; // extended error message if exists to help identifying the problem in sql statement. + int32_t msgLen; // max length of the msg + struct SCatalog* pCatalog; + SStmtCallback* pStmtCb; + const char* pUser; + bool isSuperUser; } SParseContext; typedef struct SCmdMsgInfo { int16_t msgType; - SEpSet epSet; - void* pMsg; + SEpSet epSet; + void* pMsg; int32_t msgLen; - void* pExtension; // todo remove it soon + void* pExtension; // todo remove it soon } SCmdMsgInfo; typedef enum EQueryExecMode { @@ -63,21 +65,21 @@ typedef enum EQueryExecMode { typedef struct SQuery { EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pDbList; - SArray* pTableList; - bool showRewrite; - int32_t placeholderNum; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pDbList; + SArray* pTableList; + bool showRewrite; + int32_t placeholderNum; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); -bool isInsertSql(const char* pStr, size_t length); +bool isInsertSql(const char* pStr, size_t length); void qDestroyQuery(SQuery* pQueryNode); @@ -89,14 +91,16 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); void qFreeStmtDataBlock(void* pDataBlock); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); void qDestroyStmtDataBlock(void* pBlock); -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); -int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); -int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); - +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum); +int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen); +void destroyBoundColumnInfo(void* pBoundInfo); +int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, + int32_t msgBufLen); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bd2fb9dca1..38922833f9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -913,7 +913,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index ab02ff9558..2d9a907d9b 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -100,6 +100,17 @@ void generateInformationSchema(MockCatalogService* mcs) { } } +/* + * Table:t1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * c3 | column | BIGINT | 8 | + * c4 | column | DOUBLE | 8 | + * c5 | column | DOUBLE | 8 | + */ void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) @@ -113,6 +124,17 @@ void generateTestT1(MockCatalogService* mcs) { builder.done(); } +/* + * Super Table: st1 + * Field | Type | DataType | Bytes | + * ========================================================================== + * ts | column | TIMESTAMP | 8 | + * c1 | column | INT | 4 | + * c2 | column | VARCHAR | 20 | + * tag1 | tag | INT | 4 | + * tag2 | tag | VARCHAR | 20 | + * Child Table: st1s1, st1s2 + */ void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI) diff --git a/source/libs/planner/test/planSTableTest.cpp b/source/libs/planner/test/planSTableTest.cpp index 0b60eaea32..6e349f8f9c 100644 --- a/source/libs/planner/test/planSTableTest.cpp +++ b/source/libs/planner/test/planSTableTest.cpp @@ -23,4 +23,6 @@ TEST_F(PlanSuperTableTest, tbname) { useDb("root", "test"); run("select tbname from st1"); + + run("select tbname, tag1, tag2 from st1"); } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index a7726973f0..464c636b66 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -36,11 +36,11 @@ class PlannerEnv : public testing::Environment { static void parseArg(int argc, char* argv[]) { int opt = 0; const char* optstring = ""; - static struct option long_options[] = {{"dump", no_argument, NULL, 'd'}, {0, 0, 0, 0}}; + static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}}; while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { switch (opt) { case 'd': - g_isDump = true; + setDumpModule(optarg); break; default: break; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 9ddf2a4959..e434a6be96 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -34,7 +34,41 @@ using namespace testing; } \ } while (0); -bool g_isDump = false; +enum DumpModule { + DUMP_MODULE_NOTHING = 1, + DUMP_MODULE_PARSER, + DUMP_MODULE_LOGIC, + DUMP_MODULE_OPTIMIZED, + DUMP_MODULE_SPLIT, + DUMP_MODULE_SCALED, + DUMP_MODULE_PHYSICAL, + DUMP_MODULE_SUBPLAN, + DUMP_MODULE_ALL +}; + +DumpModule g_dumpModule = DUMP_MODULE_NOTHING; + +void setDumpModule(const char* pModule) { + if (NULL == pModule) { + g_dumpModule = DUMP_MODULE_ALL; + } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PARSER; + } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_LOGIC; + } else if (0 == strncasecmp(pModule, "optimized", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_OPTIMIZED; + } else if (0 == strncasecmp(pModule, "split", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SPLIT; + } else if (0 == strncasecmp(pModule, "scaled", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SCALED; + } else if (0 == strncasecmp(pModule, "physical", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } else if (0 == strncasecmp(pModule, "subplan", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_SUBPLAN; + } else if (0 == strncasecmp(pModule, "all", strlen(pModule))) { + g_dumpModule = DUMP_MODULE_PHYSICAL; + } +} class PlannerTestBaseImpl { public: @@ -66,11 +100,9 @@ class PlannerTestBaseImpl { SQueryPlan* pPlan = nullptr; doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); - if (g_isDump) { - dump(); - } + dump(g_dumpModule); } catch (...) { - dump(); + dump(DUMP_MODULE_ALL); throw; } } @@ -109,23 +141,48 @@ class PlannerTestBaseImpl { res_.physiSubplans_.clear(); } - void dump() { + void dump(DumpModule module) { + if (DUMP_MODULE_NOTHING == module) { + return; + } + cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; - cout << "syntax tree : " << endl; - cout << res_.ast_ << endl; - cout << "raw logic plan : " << endl; - cout << res_.rawLogicPlan_ << endl; - cout << "optimized logic plan : " << endl; - cout << res_.optimizedLogicPlan_ << endl; - cout << "split logic plan : " << endl; - cout << res_.splitLogicPlan_ << endl; - cout << "scaled logic plan : " << endl; - cout << res_.scaledLogicPlan_ << endl; - cout << "physical plan : " << endl; - cout << res_.physiPlan_ << endl; - cout << "physical subplan : " << endl; - for (const auto& subplan : res_.physiSubplans_) { - cout << subplan << endl; + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { + cout << "syntax tree : " << endl; + cout << res_.ast_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) { + cout << "raw logic plan : " << endl; + cout << res_.rawLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) { + cout << "optimized logic plan : " << endl; + cout << res_.optimizedLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) { + cout << "split logic plan : " << endl; + cout << res_.splitLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) { + cout << "scaled logic plan : " << endl; + cout << res_.scaledLogicPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) { + cout << "physical plan : " << endl; + cout << res_.physiPlan_ << endl; + } + + if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) { + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 7aabff811f..7913ef531f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,6 +32,6 @@ class PlannerTestBase : public testing::Test { std::unique_ptr impl_; }; -extern bool g_isDump; +extern void setDumpModule(const char* pModule); #endif // PLAN_TEST_UTIL_H From 96372a7322f9f69765109a1d1af4ac11782f7291 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 May 2022 14:57:48 +0800 Subject: [PATCH 86/90] fix(query): set the correct join on column slot id. --- source/libs/executor/src/executorimpl.c | 35 +++++++++++++++++-------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b5d972321b..a91d204efa 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4830,6 +4830,7 @@ static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); +static void setJoinColumnInfo(SColumnInfo* pInfo, const SColumnNode* pLeftNode); static SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SInterval interval = { @@ -5624,25 +5625,29 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf goto _error; } - pOperator->resultInfo.capacity = 4096; - pOperator->resultInfo.threshold = 4096 * 0.75; + initResultSizeInfo(pOperator, 4096); - // initResultRowInf - // o(&pInfo->binfo.resultRowInfo, 8); - pInfo->pRes = pResBlock; - - pOperator->name = "JoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; + SOperatorNode* pNode = (SOperatorNode*)pOnCondition; + setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); + setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; _error: @@ -5651,3 +5656,11 @@ _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + +void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { + pColumn->slotId = pColumnNode->slotId; + pColumn->type = pColumnNode->node.resType.type; + pColumn->bytes = pColumnNode->node.resType.bytes; + pColumn->precision = pColumnNode->node.resType.precision; + pColumn->scale = pColumnNode->node.resType.scale; +} From d457f80afc829cd0ec06a8bdb84588ae660b478a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 5 May 2022 15:55:11 +0800 Subject: [PATCH 87/90] fix: turn fma off by default (#12114) [TD-15262] --- cmake/cmake.define | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 851d8f9a73..1655154506 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -67,7 +67,13 @@ ELSE () IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ADD_DEFINITIONS("-D_TD_ARM_") ELSE () - ADD_DEFINITIONS("-msse4.2 -mfma") + ADD_DEFINITIONS("-msse4.2") + IF("${FMA_SUPPORT}" MATCHES "true") + MESSAGE(STATUS "turn fma function support on") + ADD_DEFINITIONS("-mfma") + ELSE () + MESSAGE(STATUS "turn fma function support off") + ENDIF() ENDIF () ENDIF () From d2f889a3a08e4ea64816ae9ed7020cbb0d126b2c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 16:13:51 +0800 Subject: [PATCH 88/90] test: add conflict test for transaction --- source/dnode/mnode/impl/inc/mndTrans.h | 1 + source/dnode/mnode/impl/src/mndTrans.c | 6 +- source/dnode/mnode/impl/test/trans/trans2.cpp | 164 ++++++++++++++++-- 3 files changed, 158 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 2a011f414d..7bd2bd70ee 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -61,6 +61,7 @@ void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransProcessRsp(SNodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9144de82b3..299667fb45 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1278,7 +1278,7 @@ static int32_t mndProcessTransReq(SNodeMsg *pReq) { return 0; } -static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { +int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { SArray *pArray = NULL; if (pTrans->stage == TRN_STAGE_REDO_ACTION) { pArray = pTrans->redoActions; @@ -1296,14 +1296,14 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { if (pAction == NULL) continue; if (pAction->msgReceived == 0) { - mInfo("trans:%d, action:%d set processed", pTrans->id, i); + mInfo("trans:%d, action:%d set processed for kill msg received", pTrans->id, i); pAction->msgSent = 1; pAction->msgReceived = 1; pAction->errCode = 0; } if (pAction->errCode != 0) { - mInfo("trans:%d, action:%d set processed, errCode from %s to success", pTrans->id, i, + mInfo("trans:%d, action:%d set processed for kill msg received, errCode from %s to success", pTrans->id, i, tstrerror(pAction->errCode)); pAction->msgSent = 1; pAction->msgReceived = 1; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index e81827041b..b3cbcb6898 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -86,7 +86,7 @@ class MndTestTrans2 : public ::testing::Test { void SetUp() override {} void TearDown() override {} - int32_t CreateUserLog(const char *acct, const char *user) { + int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -96,7 +96,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -108,13 +108,18 @@ class MndTestTrans2 : public ::testing::Test { char *param = strdup("====> test log <====="); mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); return code; } - int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy) { + int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type, + SDbObj *pDb) { SUserObj userObj = {0}; taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); @@ -124,7 +129,7 @@ class MndTestTrans2 : public ::testing::Test { userObj.superUser = 1; SRpcMsg rpcMsg = {0}; - STrans *pTrans = mndTransCreate(pMnode, policy, TRN_TYPE_CREATE_USER, &rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); mndTransAppendRedolog(pTrans, pRedoRaw); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); @@ -176,6 +181,38 @@ class MndTestTrans2 : public ::testing::Test { mndTransSetRpcRsp(pTrans, pRsp, 256); } + if (pDb != NULL) { + mndTransSetDbInfo(pTrans, pDb); + } + + int32_t code = mndTransPrepare(pMnode, pTrans); + mndTransDrop(pTrans); + + return code; + } + + int32_t CreateUserGlobal(const char *acct, const char *user) { + SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass); + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + userObj.superUser = 1; + + SRpcMsg rpcMsg = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg); + SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); + mndTransAppendRedolog(pTrans, pRedoRaw); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); + mndTransAppendUndolog(pTrans, pUndoRaw); + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + char *param = strdup("====> test log <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + int32_t code = mndTransPrepare(pMnode, pTrans); mndTransDrop(pTrans); @@ -195,12 +232,12 @@ TEST_F(MndTestTrans2, 01_Log) { ASSERT_NE(pMnode, nullptr); - EXPECT_EQ(CreateUserLog(acct, user1), 0); + EXPECT_EQ(CreateUserLog(acct, user1, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); // failed to create user and rollback - EXPECT_EQ(CreateUserLog(acct_invalid, user2), 0); + EXPECT_EQ(CreateUserLog(acct_invalid, user2, TRN_TYPE_CREATE_USER, NULL), 0); pUser2 = mndAcquireUser(pMnode, user2); ASSERT_EQ(pUser2, nullptr); @@ -222,14 +259,14 @@ TEST_F(MndTestTrans2, 02_Action) { { // failed to create user and rollback - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_EQ(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); // create user, and fake a response { - EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); @@ -259,7 +296,7 @@ TEST_F(MndTestTrans2, 02_Action) { } { - EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY), 0); + EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL), 0); pUser1 = mndAcquireUser(pMnode, user1); ASSERT_NE(pUser1, nullptr); mndReleaseUser(pMnode, pUser1); @@ -315,7 +352,7 @@ TEST_F(MndTestTrans2, 02_Action) { } { - EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK), 0); + EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2); ASSERT_NE(pUser2, nullptr); mndReleaseUser(pMnode, pUser2); @@ -367,3 +404,110 @@ TEST_F(MndTestTrans2, 02_Action) { } } } + +TEST_F(MndTestTrans2, 03_Kill) { + const char *acct = "root"; + const char *user1 = "kill1"; + const char *user2 = "kill2"; + SUserObj *pUser1 = NULL; + SUserObj *pUser2 = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t action = 0; + + ASSERT_NE(pMnode, nullptr); + + { + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0); + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + + transId = 7; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + EXPECT_EQ(pTrans->failedTimes, 1); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser1 = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser1, nullptr); + mndReleaseUser(pMnode, pUser1); + } +} + +TEST_F(MndTestTrans2, 04_Conflict) { + const char *acct = "root"; + const char *user1 = "conflict1"; + const char *user2 = "conflict2"; + const char *user3 = "conflict3"; + const char *user4 = "conflict4"; + const char *user5 = "conflict5"; + const char *user6 = "conflict6"; + const char *user7 = "conflict7"; + const char *user8 = "conflict8"; + SUserObj *pUser = NULL; + STrans *pTrans = NULL; + int32_t transId = 0; + int32_t code = 0; + + ASSERT_NE(pMnode, nullptr); + + { + SDbObj dbObj1 = {0}; + dbObj1.uid = 9521; + strcpy(dbObj1.name, "db"); + SDbObj dbObj2 = {0}; + dbObj2.uid = 9522; + strcpy(dbObj2.name, "conflict db"); + + EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &dbObj1), 0); + pUser = mndAcquireUser(pMnode, user1); + ASSERT_NE(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + + transId = 8; + pTrans = mndAcquireTrans(pMnode, transId); + EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR); + EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION); + + // stb scope + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DNODE, NULL), -1); + code = terrno; + EXPECT_EQ(code, TSDB_CODE_MND_TRANS_CONFLICT); + + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user3, TRN_TYPE_CREATE_STB, &dbObj1), 0); + + // db scope + pTrans->type = TRN_TYPE_CREATE_DB; + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DNODE, NULL), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj2), 0); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj2), 0); + + // global scope + pTrans->type = TRN_TYPE_CREATE_DNODE; + EXPECT_EQ(CreateUserLog(acct, user6, TRN_TYPE_CREATE_DNODE, NULL), 0); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj2), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj1), -1); + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj2), -1); + + // global scope + pTrans->type = TRN_TYPE_CREATE_USER; + EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), 0); + EXPECT_EQ(CreateUserLog(acct, user8, TRN_TYPE_CREATE_DB, &dbObj2), 0); + + mndKillTrans(pMnode, pTrans); + mndReleaseTrans(pMnode, pTrans); + + pUser = mndAcquireUser(pMnode, user1); + ASSERT_EQ(pUser, nullptr); + mndReleaseUser(pMnode, pUser); + } +} \ No newline at end of file From 516fca738c5e89ea7ac5aaba8d079b8835d31396 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 16:31:07 +0800 Subject: [PATCH 89/90] fix: forbid the startup of illegal fqdn --- source/common/src/tglobal.c | 47 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4470f5b0a3..87a9c521af 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -224,7 +224,8 @@ struct SConfig *taosGetCfg() { return tsCfg; } -static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { +static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, + char *apolloUrl) { char cfgDir[PATH_MAX] = {0}; char cfgFile[PATH_MAX + 100] = {0}; @@ -300,15 +301,10 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { static int32_t taosAddClientCfg(SConfig *pCfg) { char defaultFqdn[TSDB_FQDN_LEN] = {0}; int32_t defaultServerPort = 6030; - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) return -1; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; @@ -478,15 +474,18 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); SEp firstEp = {0}; - taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); SEp secondEp = {0}; - taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); @@ -583,8 +582,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, - char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { osDefaultInit(); SConfig *pCfg = cfgInit(); @@ -636,7 +635,24 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { +static int32_t taosCheckGlobalCfg() { + uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); + if (ipv4 == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr()); + return -1; + } + + if (tsServerPort <= 0) { + uError("invalid server port:%u, dnode can not be initialized", tsServerPort); + return -1; + } + + return 0; +} + +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, + bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -674,6 +690,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetSystemCfg(tsCfg); cfgDumpCfg(tsCfg, tsc, false); + + if (taosCheckGlobalCfg() != 0) { + return -1; + } + return 0; } From 9927b6b014c477ebb04578559dbcc818d644d6ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 17:01:50 +0800 Subject: [PATCH 90/90] fix: invalid write while pullup trans on start --- source/dnode/mnode/impl/src/mndTrans.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 299667fb45..f3438bd0dc 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1000,7 +1000,9 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgSent = 0; pAction->msgReceived = 0; pAction->errCode = terrno; - if (terrno == TSDB_CODE_INVALID_PTR) rpcFreeCont(rpcMsg.pCont); + if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { + rpcFreeCont(rpcMsg.pCont); + } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; }