diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2519f4247d..7ff52b83d5 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -805,19 +805,19 @@ typedef struct { int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateVnodeMsg, SAlterVnodeMsg; +} SCreateVnodeReq, SAlterVnodeReq; typedef struct { int32_t vgId; int32_t dnodeId; - char db[TSDB_DB_FNAME_LEN]; uint64_t dbUid; -} SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg; + char db[TSDB_DB_FNAME_LEN]; +} SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; typedef struct { int32_t vgId; int8_t accessState; -} SAuthVnodeMsg; +} SAuthVnodeReq; typedef struct { SMsgHead header; diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index a7e5d50ee0..2212a8c29a 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -32,6 +32,8 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; typedef struct SVnodeCfg { + int32_t vgId; + /** vnode buffer pool options */ struct { /** write buffer size */ diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c3c7d740f7..17c11b5d09 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,16 +20,16 @@ extern "C" { #endif -typedef void* qinfo_t; +typedef void* qTaskInfo_t; /** * create the qinfo object according to QueryTableMsg * @param tsdb * @param pQueryTableMsg - * @param qinfo + * @param pTaskInfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableInfo* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); +int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); /** * the main query execution function, including query on both table and multiple tables, @@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableInfo* pQueryTableM * @param qinfo * @return */ -bool qTableQuery(qinfo_t qinfo, uint64_t *qId); +bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -48,7 +48,7 @@ bool qTableQuery(qinfo_t qinfo, uint64_t *qId); * @param qinfo * @return */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); +int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext); /** * @@ -60,41 +60,41 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) * @param qinfo * @return */ -void* qGetResultRetrieveMsg(qinfo_t qinfo); +void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); /** * kill the ongoing query and free the query handle and corresponding resources automatically * @param qinfo qhandle * @return */ -int32_t qKillQuery(qinfo_t qinfo); +int32_t qKillTask(qTaskInfo_t qinfo); /** * return whether query is completed or not * @param qinfo * @return */ -int32_t qIsQueryCompleted(qinfo_t qinfo); +int32_t qIsQueryCompleted(qTaskInfo_t qinfo); /** * destroy query info structure * @param qHandle */ -void qDestroyQueryInfo(qinfo_t qHandle); +void qDestroyTask(qTaskInfo_t qHandle); /** * Get the queried table uid * @param qHandle * @return */ -int64_t qGetQueriedTableUid(qinfo_t qHandle); +int64_t qGetQueriedTableUid(qTaskInfo_t qHandle); /** * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. @@ -121,7 +121,7 @@ int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGro * @param type operation type: ADD|DROP * @return */ -int32_t qUpdateQueriedTableIdList(qinfo_t qinfo, int64_t uid, int32_t type); +int32_t qUpdateQueriedTableIdList(qTaskInfo_t qinfo, int64_t uid, int32_t type); //================================================================================================ // query handle management @@ -130,13 +130,13 @@ int32_t qUpdateQueriedTableIdList(qinfo_t qinfo, int64_t uid, int32_t type); * @param vgId * @return */ -void* qOpenQueryMgmt(int32_t vgId); +void* qOpenTaskMgmt(int32_t vgId); /** * broadcast the close information and wait for all query stop. * @param pExecutor */ -void qQueryMgmtNotifyClosed(void* pExecutor); +void qTaskMgmtNotifyClosing(void* pExecutor); /** * Re-open the query handle management module when opening the vnode again. @@ -148,7 +148,7 @@ void qQueryMgmtReOpen(void *pExecutor); * Close query mgmt and clean up resources. * @param pExecutor */ -void qCleanupQueryMgmt(void* pExecutor); +void qCleanupTaskMgmt(void* pExecutor); /** * Add the query into the query mgmt object @@ -157,7 +157,7 @@ void qCleanupQueryMgmt(void* pExecutor); * @param qInfo * @return */ -void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); +void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo); /** * acquire the query handle according to the key from query mgmt object. @@ -165,7 +165,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); * @param key * @return */ -void** qAcquireQInfo(void* pMgmt, uint64_t key); +void** qAcquireTask(void* pMgmt, uint64_t key); /** * release the query handle and decrease the reference count in cache @@ -174,7 +174,7 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key); * @param freeHandle * @return */ -void** qReleaseQInfo(void* pMgmt, void* pQInfo); +void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle); /** * De-register the query handle from the management module and free it immediately. diff --git a/include/libs/function/function.h b/include/libs/function/function.h index d7360a81bc..bf2937a220 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -89,7 +89,7 @@ enum { }; enum { - MASTER_SCAN = 0x0u, + MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan MERGE_STAGE = 0x20u, @@ -183,7 +183,6 @@ typedef struct tExprNode { struct {// function node char functionName[FUNCTIONS_NAME_MAX_LENGTH]; -// int32_t functionId; int32_t num; // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 9e3cfbb0b0..9a7ffa8c18 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -155,7 +155,7 @@ int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** // @subplan subplan to be schedule // @templateId templateId of a group of datasource subplans of this @subplan // @ep one execution location of this group of datasource subplans -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 56f05ec0bf..80241405a6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -277,9 +277,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_BNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0452) #define TSDB_CODE_DND_BNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0453) #define TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0454) -#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0460) -#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0461) -#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0462) +#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0460) +#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0461) +#define TSDB_CODE_DND_VNODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x0462) +#define TSDB_CODE_DND_VNODE_READ_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0463) +#define TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0464) +#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index bf5f0122c1..b5fae62959 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -29,12 +29,12 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); +int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 61b220158f..15be59a419 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -42,18 +42,13 @@ static SBnode *dndAcquireBnode(SDnode *pDnode) { } static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) { + if (pBnode == NULL) return; + SBnodeMgmt *pMgmt = &pDnode->bmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pBnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pBnode != NULL) { - dTrace("release bnode, refCount:%d", refCount); - } + dTrace("release bnode, refCount:%d", refCount); } static int32_t dndReadBnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index ae37967b3d..6c23af7f00 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -43,18 +43,13 @@ static SMnode *dndAcquireMnode(SDnode *pDnode) { } static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode) { + if (pMnode == NULL) return; + SMnodeMgmt *pMgmt = &pDnode->mmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pMnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pMnode != NULL) { - dTrace("release mnode, refCount:%d", refCount); - } + dTrace("release mnode, refCount:%d", refCount); } static int32_t dndReadMnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 3deee93e29..9d2f623c45 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -42,18 +42,13 @@ static SQnode *dndAcquireQnode(SDnode *pDnode) { } static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode) { + if (pQnode == NULL) return; + SQnodeMgmt *pMgmt = &pDnode->qmgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pQnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pQnode != NULL) { - dTrace("release qnode, refCount:%d", refCount); - } + dTrace("release qnode, refCount:%d", refCount); } static int32_t dndReadQnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index ab4e38bfb2..00435d4c3e 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -42,18 +42,13 @@ static SSnode *dndAcquireSnode(SDnode *pDnode) { } static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) { + if (pSnode == NULL) return; + SSnodeMgmt *pMgmt = &pDnode->smgmt; - int32_t refCount = 0; - taosRLockLatch(&pMgmt->latch); - if (pSnode != NULL) { - refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - - if (pSnode != NULL) { - dTrace("release snode, refCount:%d", refCount); - } + dTrace("release snode, refCount:%d", refCount); } static int32_t dndReadSnodeFile(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index bd1e08afc0..a68a3d05d1 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -53,22 +53,8 @@ typedef struct { SWrapperCfg *pCfgs; } SVnodeThread; -static int32_t dndInitVnodeReadWorker(SDnode *pDnode); -static int32_t dndInitVnodeWriteWorker(SDnode *pDnode); -static int32_t dndInitVnodeSyncWorker(SDnode *pDnode); -static void dndCleanupVnodeReadWorker(SDnode *pDnode); -static void dndCleanupVnodeWriteWorker(SDnode *pDnode); -static void dndCleanupVnodeSyncWorker(SDnode *pDnode); -static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); +static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); @@ -117,11 +103,9 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { if (pVnode == NULL) return; SVnodesMgmt *pMgmt = &pDnode->vmgmt; - taosRLockLatch(&pMgmt->latch); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } @@ -134,7 +118,7 @@ static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { } pVnode->vgId = pCfg->vgId; - pVnode->refCount = 1; + pVnode->refCount = 0; pVnode->dropped = 0; pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->pImpl = pImpl; @@ -148,23 +132,8 @@ static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } - if (dndAllocVnodeQueryQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeFetchQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeWriteQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeApplyQueue(pDnode, pVnode) != 0) { - return -1; - } - - if (dndAllocVnodeSyncQueue(pDnode, pVnode) != 0) { + if (dndAllocVnodeQueue(pDnode, pVnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -192,12 +161,7 @@ static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10); while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10); - dndFreeVnodeQueryQueue(pDnode, pVnode); - dndFreeVnodeFetchQueue(pDnode, pVnode); - dndFreeVnodeWriteQueue(pDnode, pVnode); - dndFreeVnodeApplyQueue(pDnode, pVnode); - dndFreeVnodeSyncQueue(pDnode, pVnode); - + dndFreeVnodeQueue(pDnode, pVnode); vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; @@ -508,7 +472,6 @@ static void dndCloseVnodes(SDnode *pDnode) { SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - dndReleaseVnode(pDnode, pVnodes[i]); dndCloseVnode(pDnode, pVnodes[i]); } @@ -524,8 +487,8 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = rpcMsg->pCont; +static SCreateVnodeReq *dndParseCreateVnodeReq(SRpcMsg *pReq) { + SCreateVnodeReq *pCreate = pReq->pCont; pCreate->vgId = htonl(pCreate->vgId); pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dbUid = htobe64(pCreate->dbUid); @@ -549,7 +512,8 @@ static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { return pCreate; } -static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { +static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize; @@ -572,7 +536,7 @@ static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.vgId = pCreate->vgId; } -static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) { +static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN); pCfg->dbUid = pCreate->dbUid; pCfg->dropped = 0; @@ -581,20 +545,20 @@ static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWra pCfg->vgVersion = pCreate->vgVersion; } -static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = rpcMsg->pCont; +static SDropVnodeReq *vnodeParseDropVnodeReq(SRpcMsg *pReq) { + SDropVnodeReq *pDrop = pReq->pCont; pDrop->vgId = htonl(pDrop->vgId); return pDrop; } -static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = rpcMsg->pCont; +static SAuthVnodeReq *vnodeParseAuthVnodeReq(SRpcMsg *pReq) { + SAuthVnodeReq *pAuth = pReq->pCont; pAuth->vgId = htonl(pAuth->vgId); return pAuth; } -int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg); +int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SCreateVnodeReq *pCreate = dndParseCreateVnodeReq(pReq); dDebug("vgId:%d, create vnode req is received", pCreate->vgId); SVnodeCfg vnodeCfg = {0}; @@ -607,16 +571,19 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { if (pVnode != NULL) { dDebug("vgId:%d, already exist, return success", pCreate->vgId); dndReleaseVnode(pDnode, pVnode); + terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; return 0; } SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/, pCreate->vgId); if (pImpl == NULL) { + dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); return -1; } int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); if (code != 0) { + dError("vgId:%d, failed to open vnode since %s", pCreate->vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; @@ -634,23 +601,20 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg); +int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SAlterVnodeReq *pAlter = (SAlterVnodeReq *)dndParseCreateVnodeReq(pReq); dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); SVnodeCfg vnodeCfg = {0}; dndGenerateVnodeCfg(pAlter, &vnodeCfg); - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); - return terrno; + return -1; } - if (wrapperCfg.vgVersion == pVnode->vgVersion) { + if (pAlter->vgVersion == pVnode->vgVersion) { dndReleaseVnode(pDnode, pVnode); dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); return 0; @@ -659,11 +623,11 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = wrapperCfg.vgVersion; + pVnode->vgVersion = pAlter->vgVersion; int32_t code = dndWriteVnodesToFile(pDnode); if (code != 0) { pVnode->vgVersion = oldVersion; @@ -673,8 +637,8 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return code; } -int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SDropVnodeMsg *pDrop = vnodeParseDropVnodeReq(rpcMsg); +int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDropVnodeReq *pDrop = vnodeParseDropVnodeReq(pReq); int32_t vgId = pDrop->vgId; dDebug("vgId:%d, drop vnode req is received", vgId); @@ -688,10 +652,10 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { pVnode->dropped = 1; if (dndWriteVnodesToFile(pDnode) != 0) { pVnode->dropped = 0; - return terrno; + dndReleaseVnode(pDnode, pVnode); + return -1; } - dndReleaseVnode(pDnode, pVnode); dndCloseVnode(pDnode, pVnode); vnodeClose(pVnode->pImpl); vnodeDestroy(pVnode->path); @@ -700,17 +664,16 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); +int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SAuthVnodeReq *pAuth = (SAuthVnodeReq *)vnodeParseAuthVnodeReq(pReq); - int32_t code = 0; int32_t vgId = pAuth->vgId; dDebug("vgId:%d, auth vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); - return terrno; + return -1; } pVnode->accessState = pAuth->accessState; @@ -718,30 +681,30 @@ int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { return 0; } -int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SAuthVnodeMsg *pAuth = (SAuthVnodeMsg *)vnodeParseAuthVnodeReq(rpcMsg); +int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SSyncVnodeReq *pSync = (SSyncVnodeReq *)vnodeParseDropVnodeReq(pReq); - int32_t vgId = pAuth->vgId; - dDebug("vgId:%d, auth vnode req is received", vgId); + int32_t vgId = pSync->vgId; + dDebug("vgId:%d, sync vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); - return terrno; + dDebug("vgId:%d, failed to sync since %s", vgId, terrstr()); + return -1; } if (vnodeSync(pVnode->pImpl) != 0) { - dError("vgId:%d, failed to auth vnode since %s", vgId, terrstr()); + dError("vgId:%d, failed to sync vnode since %s", vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } dndReleaseVnode(pDnode, pVnode); return 0; } -int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { - SCompactVnodeMsg *pCompact = (SCompactVnodeMsg *)vnodeParseDropVnodeReq(rpcMsg); +int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SCompactVnodeReq *pCompact = (SCompactVnodeReq *)vnodeParseDropVnodeReq(pReq); int32_t vgId = pCompact->vgId; dDebug("vgId:%d, compact vnode req is received", vgId); @@ -749,13 +712,13 @@ int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to compact since %s", vgId, terrstr()); - return terrno; + return -1; } if (vnodeCompact(pVnode->pImpl) != 0) { dError("vgId:%d, failed to compact vnode since %s", vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); - return terrno; + return -1; } dndReleaseVnode(pDnode, pVnode); @@ -814,6 +777,7 @@ static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_ for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); + // todo SRpcMsg *pRsp = NULL; (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); } @@ -825,6 +789,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); + // todo SRpcMsg *pRsp = NULL; (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); } @@ -848,21 +813,25 @@ static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) } if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); + if (pRpcMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } rpcFreeCont(pRpcMsg->pCont); } } static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { - SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; - rpcSendResponse(&rsp); + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + rpcSendResponse(&rsp); + } rpcFreeCont(pMsg->pCont); } @@ -903,193 +872,96 @@ void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - return -1; - } + if (pVnode == NULL) return -1; int32_t code = taosWriteQitem(pVnode->pApplyQ, pMsg); dndReleaseVnode(pDnode, pVnode); return code; } -static int32_t dndAllocVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); - if (pVnode->pQueryQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeQueryQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - pVnode->pQueryQ = NULL; -} - -static int32_t dndAllocVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); - if (pVnode->pFetchQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeFetchQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - pVnode->pFetchQ = NULL; -} - -static int32_t dndInitVnodeReadWorker(SDnode *pDnode) { +static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - float threadsForQuery = MAX(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores, 1); + int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->opt.numOfCores); + int32_t minQueryThreads = MAX((int32_t)(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + int32_t maxWriteThreads = MAX(pDnode->opt.numOfCores, 1); + int32_t maxSyncThreads = MAX(pDnode->opt.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; - pPool->min = (int32_t)threadsForQuery; - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + pPool->min = minQueryThreads; + pPool->max = maxQueryThreads; + if (tWorkerInit(pPool) != 0) return -1; pPool = &pMgmt->fetchPool; pPool->name = "vnode-fetch"; - pPool->min = MIN(maxFetchThreads, pDnode->opt.numOfCores); - pPool->max = pPool->min; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + pPool->min = minFetchThreads; + pPool->max = maxFetchThreads; + if (tWorkerInit(pPool) != 0) return -1; - dDebug("vnode read worker is initialized"); + SMWorkerPool *pMPool = &pMgmt->writePool; + pMPool->name = "vnode-write"; + pMPool->max = maxWriteThreads; + if (tMWorkerInit(pMPool) != 0) return -1; + + pMPool = &pMgmt->syncPool; + pMPool->name = "vnode-sync"; + pMPool->max = maxSyncThreads; + if (tMWorkerInit(pMPool) != 0) return -1; + + dDebug("vnode workers is initialized"); return 0; } -static void dndCleanupVnodeReadWorker(SDnode *pDnode) { +static void dndCleanupVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; tWorkerCleanup(&pMgmt->fetchPool); tWorkerCleanup(&pMgmt->queryPool); - dDebug("vnode close worker is initialized"); -} - -static int32_t dndAllocVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); - if (pVnode->pWriteQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - pVnode->pWriteQ = NULL; -} - -static int32_t dndAllocVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); - if (pVnode->pApplyQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - pVnode->pApplyQ = NULL; -} - -static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt * pMgmt = &pDnode->vmgmt; - SMWorkerPool *pPool = &pMgmt->writePool; - pPool->name = "vnode-write"; - pPool->max = pDnode->opt.numOfCores; - if (tMWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("vnode write worker is initialized"); - return 0; -} - -static void dndCleanupVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; tMWorkerCleanup(&pMgmt->writePool); - dDebug("vnode write worker is closed"); -} - -static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); - if (pVnode->pSyncQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); - pVnode->pSyncQ = NULL; -} - -static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) { - int32_t maxThreads = pDnode->opt.numOfCores / 2; - if (maxThreads < 1) maxThreads = 1; - - SVnodesMgmt * pMgmt = &pDnode->vmgmt; - SMWorkerPool *pPool = &pMgmt->syncPool; - pPool->name = "vnode-sync"; - pPool->max = maxThreads; - if (tMWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("vnode sync worker is initialized"); - return 0; -} - -static void dndCleanupVnodeSyncWorker(SDnode *pDnode) { - SVnodesMgmt *pMgmt = &pDnode->vmgmt; tMWorkerCleanup(&pMgmt->syncPool); - dDebug("vnode sync worker is closed"); + dDebug("vnode workers is closed"); +} + +static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + + pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); + pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); + pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); + pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); + pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); + + if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || + pVnode->pQueryQ == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { + SVnodesMgmt *pMgmt = &pDnode->vmgmt; + tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + pVnode->pWriteQ = NULL; + pVnode->pApplyQ = NULL; + pVnode->pSyncQ = NULL; + pVnode->pFetchQ = NULL; + pVnode->pQueryQ = NULL; } int32_t dndInitVnodes(SDnode *pDnode) { dInfo("dnode-vnodes start to init"); - if (dndInitVnodeReadWorker(pDnode) != 0) { - dError("failed to init vnodes read worker since %s", terrstr()); - return -1; - } - - if (dndInitVnodeWriteWorker(pDnode) != 0) { - dError("failed to init vnodes write worker since %s", terrstr()); - return -1; - } - - if (dndInitVnodeSyncWorker(pDnode) != 0) { - dError("failed to init vnodes sync worker since %s", terrstr()); + if (dndInitVnodeWorkers(pDnode) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + dError("failed to init vnode workers since %s", terrstr()); return -1; } @@ -1105,9 +977,7 @@ int32_t dndInitVnodes(SDnode *pDnode) { void dndCleanupVnodes(SDnode *pDnode) { dInfo("dnode-vnodes start to clean up"); dndCloseVnodes(pDnode); - dndCleanupVnodeReadWorker(pDnode); - dndCleanupVnodeWriteWorker(pDnode); - dndCleanupVnodeSyncWorker(pDnode); + dndCleanupVnodeWorkers(pDnode); dInfo("dnode-vnodes is cleaned up"); } diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 8f721dae9a..ce93a14d3f 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -4,8 +4,5 @@ add_subdirectory(qnode) add_subdirectory(bnode) add_subdirectory(snode) add_subdirectory(mnode) -add_subdirectory(db) -add_subdirectory(stb) -add_subdirectory(vgroup) - +add_subdirectory(vnode) add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt deleted file mode 100644 index cb9f1600fc..0000000000 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. DB_SRC) -add_executable(dnode_test_db ${DB_SRC}) -target_link_libraries( - dnode_test_db - PUBLIC sut -) - -add_test( - NAME dnode_test_db - COMMAND dnode_test_db -) diff --git a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt deleted file mode 100644 index b864b0593c..0000000000 --- a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. VGROUP_SRC) -add_executable(dnode_test_vgroup ${VGROUP_SRC}) -target_link_libraries( - dnode_test_vgroup - PUBLIC sut -) - -add_test( - NAME dnode_test_vgroup - COMMAND dnode_test_vgroup -) diff --git a/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt new file mode 100644 index 0000000000..6fb8bb4ba4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/vnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. VNODE_SRC) +add_executable(dnode_test_vnode ${VNODE_SRC}) +target_link_libraries( + dnode_test_vnode + PUBLIC sut +) + +add_test( + NAME dnode_test_vnode + COMMAND dnode_test_vnode +) diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp similarity index 83% rename from source/dnode/mgmt/impl/test/vgroup/vgroup.cpp rename to source/dnode/mgmt/impl/test/vnode/vnode.cpp index 7fa3b4ab61..7233137140 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -11,9 +11,9 @@ #include "sut.h" -class DndTestVgroup : public ::testing::Test { +class DndTestVnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vgroup", 9150); } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vnode", 9150); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,14 +23,14 @@ class DndTestVgroup : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestVgroup::test; +Testbase DndTestVnode::test; -TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { +TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateVnodeMsg); + int32_t contLen = sizeof(SCreateVnodeReq); - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(contLen); + SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -68,9 +68,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SAlterVnodeMsg); + int32_t contLen = sizeof(SAlterVnodeReq); - SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(contLen); + SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -108,9 +108,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SDropVnodeMsg); + int32_t contLen = sizeof(SDropVnodeReq); - SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(contLen); + SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); pReq->vgId = htonl(2); pReq->dnodeId = htonl(1); strcpy(pReq->db, "1.d1"); @@ -118,7 +118,7 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropVnodeMsg); + rpcMsg.contLen = sizeof(SDropVnodeReq); rpcMsg.msgType = TDMT_DND_DROP_VNODE; SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 6d391450b7..9e4656fec8 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); -SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); -SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +SCreateVnodeReq *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +SDropVnodeReq *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 251cbb9217..bf5d01d0a2 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -331,11 +331,11 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); + SCreateVnodeReq *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); if (pMsg == NULL) return -1; action.pCont = pMsg; - action.contLen = sizeof(SCreateVnodeMsg); + action.contLen = sizeof(SCreateVnodeReq); action.msgType = TDMT_DND_CREATE_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -360,11 +360,11 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); + SDropVnodeReq *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); if (pMsg == NULL) return -1; action.pCont = pMsg; - action.contLen = sizeof(SDropVnodeMsg); + action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); @@ -593,11 +593,11 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SAlterVnodeMsg *pMsg = (SAlterVnodeMsg *)mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); + SAlterVnodeReq *pMsg = (SAlterVnodeReq *)mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); if (pMsg == NULL) return -1; action.pCont = pMsg; - action.contLen = sizeof(SAlterVnodeMsg); + action.contLen = sizeof(SAlterVnodeReq); action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -757,11 +757,11 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); + SDropVnodeReq *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); if (pMsg == NULL) return -1; action.pCont = pMsg; - action.contLen = sizeof(SCreateVnodeMsg); + action.contLen = sizeof(SCreateVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index bd17c6d150..e9d35a6e4c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -189,8 +189,8 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg)); +SCreateVnodeReq *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SCreateVnodeReq *pCreate = calloc(1, sizeof(SCreateVnodeReq)); if (pCreate == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -248,8 +248,8 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb return pCreate; } -SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg)); +SDropVnodeReq *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SDropVnodeReq *pDrop = calloc(1, sizeof(SDropVnodeReq)); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 4d37bb368d..3ca35d58a7 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -10,3 +10,5 @@ add_subdirectory(show) add_subdirectory(profile) add_subdirectory(dnode) add_subdirectory(mnode) +add_subdirectory(db) +add_subdirectory(stb) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt new file mode 100644 index 0000000000..f0abdf152c --- /dev/null +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. DB_SRC) +add_executable(mnode_test_db ${DB_SRC}) +target_link_libraries( + mnode_test_db + PUBLIC sut +) + +add_test( + NAME mnode_test_db + COMMAND mnode_test_db +) diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp similarity index 95% rename from source/dnode/mgmt/impl/test/db/db.cpp rename to source/dnode/mnode/impl/test/db/db.cpp index 3a69ae2305..42cf753c7c 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -1,19 +1,19 @@ /** * @file db.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module db-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module db tests + * @version 1.0 + * @date 2022-01-11 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestDb : public ::testing::Test { +class MndTestDb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_db", 9040); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,9 +23,9 @@ class DndTestDb : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestDb::test; +Testbase MndTestDb::test; -TEST_F(DndTestDb, 01_ShowDb) { +TEST_F(MndTestDb, 01_ShowDb) { test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); @@ -51,7 +51,7 @@ TEST_F(DndTestDb, 01_ShowDb) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { +TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { int32_t contLen = sizeof(SCreateDbMsg); @@ -211,7 +211,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { +TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { int32_t contLen = sizeof(SCreateDbMsg); @@ -281,7 +281,7 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->numOfEps, 1); SEpAddrMsg* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9040); + EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } @@ -297,7 +297,7 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->numOfEps, 1); SEpAddrMsg* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9040); + EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } } diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mnode/impl/test/stb/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/stb/CMakeLists.txt rename to source/dnode/mnode/impl/test/stb/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp similarity index 95% rename from source/dnode/mgmt/impl/test/stb/stb.cpp rename to source/dnode/mnode/impl/test/stb/stb.cpp index d3362c7a9b..55cc030122 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -1,19 +1,19 @@ /** * @file stb.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module db-msg tests - * @version 0.1 - * @date 2021-12-17 + * @brief MNODE module stb tests + * @version 1.0 + * @date 2022-01-12 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestStb : public ::testing::Test { +class MndTestStb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9101); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_stb", 9034); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,9 +23,9 @@ class DndTestStb : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestStb::test; +Testbase MndTestStb::test; -TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { +TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { int32_t contLen = sizeof(SCreateDbMsg); diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 55742d5cad..eb4b45bc20 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -24,9 +24,9 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid) { SVnode *pVnode = NULL; // Set default options - if (pVnodeCfg == NULL) { + //if (pVnodeCfg == NULL) { pVnodeCfg = &defaultVnodeOptions; - } + //} // Validate options if (vnodeValidateOptions(pVnodeCfg) < 0) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 7e910d5674..2c1bf71638 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -88,37 +88,37 @@ typedef struct SResultRowPool { SArray* pData; // SArray } SResultRowPool; -struct SQueryAttr; -struct SQueryRuntimeEnv; +struct STaskAttr; +struct STaskRuntimeEnv; struct SUdfInfo; -int32_t getOutputInterResultBufSize(struct SQueryAttr* pQueryAttr); +int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); -size_t getResultRowSize(struct SQueryRuntimeEnv* pRuntimeEnv); +size_t getResultRowSize(struct STaskRuntimeEnv* pRuntimeEnv); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void resetResultRowInfo(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); -int32_t getRowNumForMultioutput(struct SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); +int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char* getPosInResultPage(struct SQueryAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, +static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); @@ -155,7 +155,7 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo); bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 907fb4d2bf..a268215d3d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -21,13 +21,14 @@ #include "tvariant.h" #include "thash.h" -//#include "parser.h" #include "executil.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" #include "tlockfree.h" #include "tpagedfile.h" +#include "planner.h" + struct SColumnFilterElem; @@ -65,7 +66,6 @@ enum { QUERY_OVER = 0x4u, }; - typedef struct SResultRowCell { uint64_t groupId; SResultRow *pRow; @@ -100,7 +100,7 @@ typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list SVariant tag; - STimeWindow win; + STimeWindow win; // todo remove it later STSCursor cur; void* pTable; // for retrieve the page id list SResultRowInfo resInfo; @@ -128,31 +128,34 @@ typedef struct { int64_t sumRunTimes; } SOperatorProfResult; -typedef struct SQueryCostInfo { - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t discardBlocks; - uint64_t elapsedTime; - uint64_t firstStageMergeTime; - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; +typedef struct STaskCostInfo { + int64_t start; + int64_t end; - SArray* queryProfEvents; //SArray - SHashObj* operatorProfResults; //map -} SQueryCostInfo; + uint64_t loadStatisTime; + uint64_t loadFileBlockTime; + uint64_t loadDataInCacheTime; + uint64_t loadStatisSize; + uint64_t loadFileBlockSize; + uint64_t loadDataInCacheSize; + + uint64_t loadDataTime; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t firstStageMergeTime; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; + uint64_t numOfTimeWindows; + + SArray *queryProfEvents; //SArray + SHashObj *operatorProfResults; //map +} STaskCostInfo; typedef struct { int64_t vgroupLimit; @@ -166,7 +169,7 @@ typedef struct { // The basic query information extracted from the SQueryInfo tree to support the // execution of query in a data node. -typedef struct SQueryAttr { +typedef struct STaskAttr { SLimit limit; SLimit slimit; @@ -229,16 +232,40 @@ typedef struct SQueryAttr { STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; SArray *pUdfInfo; // no need to free -} SQueryAttr; +} STaskAttr; typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); struct SOperatorInfo; -typedef struct SQueryRuntimeEnv { +typedef struct STaskIdInfo { + uint64_t queryId; // this is also a request id + uint64_t subplanId; + uint64_t templateId; + uint64_t taskId; // this is a subplan id +} STaskIdInfo; + +typedef struct STaskInfo { + STaskIdInfo id; + char *content; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + pthread_mutex_t lock; // used to synchronize the rsp/query threads +// tsem_t ready; +// int32_t dataReady; // denote if query result is ready or not +// void* rspContext; // response context + char *sql; // query sql string + jmp_buf env; +} STaskInfo; + +typedef struct STaskRuntimeEnv { jmp_buf env; - SQueryAttr* pQueryAttr; + STaskAttr* pQueryAttr; uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not @@ -271,7 +298,7 @@ typedef struct SQueryRuntimeEnv { SRspResultInfo resultInfo; SHashObj *pTableRetrieveTsMap; struct SUdfInfo *pUdfInfo; -} SQueryRuntimeEnv; +} STaskRuntimeEnv; enum { OP_IN_EXECUTING = 1, @@ -287,10 +314,11 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - SQueryRuntimeEnv *pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv; + STaskInfo *pTaskInfo; - struct SOperatorInfo **upstream; // upstream pointer list - int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator + struct SOperatorInfo **pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator __operator_fn_t exec; __optr_cleanup_fn_t cleanup; } SOperatorInfo; @@ -312,8 +340,8 @@ typedef struct SQInfo { int32_t code; // error code to returned to client int64_t owner; // if it is in execution - SQueryRuntimeEnv runtimeEnv; - SQueryAttr query; + STaskRuntimeEnv runtimeEnv; + STaskAttr query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -322,10 +350,10 @@ typedef struct SQInfo { void* rspContext; // response context int64_t startExecTs; // start to exec timestamp char* sql; // query sql string - SQueryCostInfo summary; + STaskCostInfo summary; } SQInfo; -typedef struct SQueryParam { +typedef struct STaskParam { char *sql; char *tagCond; char *colCond; @@ -345,7 +373,7 @@ typedef struct SQueryParam { int32_t tableScanOperator; SArray *pOperator; struct SUdfInfo *pUdfInfo; -} SQueryParam; +} STaskParam; typedef struct STableScanInfo { void *pQueryHandle; @@ -366,9 +394,12 @@ typedef struct STableScanInfo { SSDataBlock block; int32_t numOfOutput; int64_t elapsedTime; - int32_t tableIndex; - int32_t prevGroupId; // previous table group id + + int32_t prevGroupId; // previous table group id + + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + STimeWindow window; } STableScanInfo; typedef struct STagScanInfo { @@ -512,34 +543,34 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); +SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); @@ -561,8 +592,8 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); -void freeParam(SQueryParam *param); -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +void freeParam(STaskParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); @@ -575,13 +606,13 @@ SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pCo SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger); -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); +int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId); void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); +STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); @@ -590,9 +621,9 @@ bool isQueryKilled(SQInfo *pQInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); +void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); -bool onlyQueryTags(SQueryAttr* pQueryAttr); +bool onlyQueryTags(STaskAttr* pQueryAttr); void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -607,8 +638,8 @@ void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); void calculateOperatorProfResults(SQInfo* pQInfo); void queryCostStatis(SQInfo *pQInfo); -void freeQInfo(SQInfo *pQInfo); -void freeQueryAttr(SQueryAttr *pQuery); +void doDestroyTask(SQInfo *pQInfo); +void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ac91f906c7..e8ecffb72c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -30,7 +30,7 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; -int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable) { +int32_t getRowNumForMultioutput(STaskAttr* pQueryAttr, bool topBottomQuery, bool stable) { if (pQueryAttr && (!stable)) { for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { // if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { @@ -42,7 +42,7 @@ int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, boo return 1; } -int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr) { +int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { @@ -86,7 +86,7 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { tfree(pResultRowInfo->pResult); } -void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { +void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } @@ -136,7 +136,7 @@ void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { +void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { if (pResultRow == NULL) { return; } @@ -174,8 +174,8 @@ struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, return NULL; } -size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +size_t getResultRowSize(STaskRuntimeEnv* pRuntimeEnv) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; return 0; // return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); } @@ -393,8 +393,8 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; @@ -488,7 +488,7 @@ int32_t tsDescOrder(const void* p1, const void* p2) { } } -void orderTheResultRows(SQueryRuntimeEnv* pRuntimeEnv) { +void orderTheResultRows(STaskRuntimeEnv* pRuntimeEnv) { __compar_fn_t fn = NULL; if (pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_ASC) { fn = tsAscOrder; @@ -499,7 +499,7 @@ void orderTheResultRows(SQueryRuntimeEnv* pRuntimeEnv) { taosArraySort(pRuntimeEnv->pResultRowArrayList, fn); } -static int32_t mergeIntoGroupResultImplRv(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, uint64_t groupId, int32_t* rowCellInfoOffset) { +static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, uint64_t groupId, int32_t* rowCellInfoOffset) { if (!pGroupResInfo->ordered) { orderTheResultRows(pRuntimeEnv); pGroupResInfo->ordered = true; @@ -528,7 +528,7 @@ static int32_t mergeIntoGroupResultImplRv(SQueryRuntimeEnv *pRuntimeEnv, SGroupR return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, +static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, int32_t* rowCellInfoOffset) { bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQueryAttr); @@ -630,7 +630,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn return code; } -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t* offset) { +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, int32_t* offset) { int64_t st = taosGetTimestampUs(); while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c new file mode 100644 index 0000000000..b98b7fef5c --- /dev/null +++ b/source/libs/executor/src/executorMain.c @@ -0,0 +1,579 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tcache.h" +#include "tglobal.h" +#include "tmsg.h" +#include "exception.h" + +#include "thash.h" +#include "executorimpl.h" +#include "executor.h" +#include "tlosertree.h" +#include "ttypes.h" +#include "query.h" + +typedef struct STaskMgmt { + pthread_mutex_t lock; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; +} STaskMgmt; + +static void taskMgmtKillTaskFn(void* handle, void* param1) { + void** fp = (void**)handle; + qKillTask(*fp); +} + +static void freeqinfoFn(void *qhandle) { + void** handle = qhandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillTask(*handle); + qDestroyTask(*handle); +} + +void freeParam(STaskParam *param) { + tfree(param->sql); + tfree(param->tagCond); + tfree(param->tbnameCond); + tfree(param->pTableIdList); + taosArrayDestroy(param->pOperator); + tfree(param->pExprs); + tfree(param->pSecExprs); + + tfree(param->pExpr); + tfree(param->pSecExpr); + + tfree(param->pGroupColIndex); + tfree(param->pTagColumnInfo); + tfree(param->pGroupbyExpr); + tfree(param->prevResult); +} + +// todo parse json to get the operator tree. + +int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { + assert(pQueryMsg != NULL && tsdb != NULL); + + int32_t code = TSDB_CODE_SUCCESS; +#if 0 + STaskParam param = {0}; + code = convertQueryMsg(pQueryMsg, ¶m); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (pQueryMsg->numOfTables <= 0) { + qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { + qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; + if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, + pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (param.pSecExpr != NULL) { + if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + if (param.colCond != NULL) { + if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); + if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + goto _over; + } + + bool isSTableQuery = false; + STableGroupInfo tableGroupInfo = {0}; + int64_t st = taosGetTimestampUs(); + + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { + isSTableQuery = true; + + // also note there's possibility that only one table in the super table + if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); + + if (code != TSDB_CODE_SUCCESS) { + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); + goto _over; + } + } else { + code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + } else { + assert(0); + } + + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + + assert(pQueryMsg->stableQuery == isSTableQuery); + (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, + param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); + + param.sql = NULL; + param.pExprs = NULL; + param.pSecExprs = NULL; + param.pGroupbyExpr = NULL; + param.pTagColumnInfo = NULL; + param.pFilters = NULL; + + if ((*pTaskInfo) == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _over; + } + param.pUdfInfo = NULL; + + code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); + + _over: + if (param.pGroupbyExpr != NULL) { + taosArrayDestroy(param.pGroupbyExpr->columnInfo); + } + + tfree(param.colCond); + + destroyUdfInfo(param.pUdfInfo); + + taosArrayDestroy(param.pTableIdList); + param.pTableIdList = NULL; + + freeParam(¶m); + + for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { + SColumnInfo* column = pQueryMsg->tableCols + i; + freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); + } + + filterFreeInfo(param.pFilters); + + //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + if (code != TSDB_CODE_SUCCESS) { + *pTaskInfo = NULL; + } +#endif + + // if failed to add ref for all tables in this query, abort current query + return code; +} + +#ifdef TEST_IMPL +// wait moment +int waitMoment(SQInfo* pQInfo){ + if(pQInfo->sql) { + int ms = 0; + char* pcnt = strstr(pQInfo->sql, " count(*)"); + if(pcnt) return 0; + + char* pos = strstr(pQInfo->sql, " t_"); + if(pos){ + pos += 3; + ms = atoi(pos); + while(*pos >= '0' && *pos <= '9'){ + pos ++; + } + char unit_char = *pos; + if(unit_char == 'h'){ + ms *= 3600*1000; + } else if(unit_char == 'm'){ + ms *= 60*1000; + } else if(unit_char == 's'){ + ms *= 1000; + } + } + if(ms == 0) return 0; + printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); + + if(ms < 1000) { + taosMsleep(ms); + } else { + int used_ms = 0; + while(used_ms < ms) { + taosMsleep(1000); + used_ms += 1000; + if(isQueryKilled(pQInfo)){ + printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); + break; + } + } + } + } + return 1; +} +#endif + +bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { + SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); + int64_t threadId = taosGetSelfPthreadId(); + + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + *qId = pQInfo->qId; + if(pQInfo->startExecTs == 0) + pQInfo->startExecTs = taosGetTimestampMs(); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); + return doBuildResCheck(pQInfo); + } + + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); +// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); + return doBuildResCheck(pQInfo); + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pQInfo->runtimeEnv.env); + if (ret != TSDB_CODE_SUCCESS) { + publishQueryAbortEvent(pQInfo, ret); + pQInfo->code = ret; + qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); + return doBuildResCheck(pQInfo); + } + + qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + + bool newgroup = false; + publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + + int64_t st = taosGetTimestampUs(); + pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); + pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); +#ifdef TEST_IMPL + waitMoment(pQInfo); +#endif + publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); + pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); + } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + pRuntimeEnv->resultInfo.total); + } else { + qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, + GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); + } + + return doBuildResCheck(pQInfo); +} + +int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + qError("QInfo invalid qhandle"); + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + *buildRes = false; + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); + return pQInfo->code; + } + + int32_t code = TSDB_CODE_SUCCESS; + + if (tsRetrieveBlockingModel) { + pQInfo->rspContext = pRspContext; + tsem_wait(&pQInfo->ready); + *buildRes = true; + code = pQInfo->code; + } else { + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + pthread_mutex_lock(&pQInfo->lock); + + assert(pQInfo->rspContext == NULL); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize, + GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); + } else { + *buildRes = false; + qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId); + pQInfo->rspContext = pRspContext; + assert(pQInfo->rspContext != NULL); + } + + code = pQInfo->code; + pthread_mutex_unlock(&pQInfo->lock); + } + + return code; +} + +void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + +int32_t qKillTask(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); + setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pQInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); +} + +void qDestroyTask(qTaskInfo_t qHandle) { + SQInfo* pQInfo = (SQInfo*) qHandle; + if (!isValidQInfo(pQInfo)) { + return; + } + + qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId); + queryCostStatis(pQInfo); // print the query cost summary + doDestroyTask(pQInfo); +} + +void* qOpenTaskMgmt(int32_t vgId) { + const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool + + char cacheName[128] = {0}; + sprintf(cacheName, "qhandle_%d", vgId); + + STaskMgmt* pTaskMgmt = calloc(1, sizeof(STaskMgmt)); + if (pTaskMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pTaskMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); + pTaskMgmt->closed = false; + pTaskMgmt->vgId = vgId; + + pthread_mutex_init(&pTaskMgmt->lock, NULL); + + qDebug("vgId:%d, open queryTaskMgmt success", vgId); + return pTaskMgmt; +} + +void qTaskMgmtNotifyClosing(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt* pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = true; + pthread_mutex_unlock(&pQueryMgmt->lock); + + taosCacheRefresh(pQueryMgmt->qinfoPool, taskMgmtKillTaskFn, NULL); +} + +void qQueryMgmtReOpen(void *pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt *pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = false; + pthread_mutex_unlock(&pQueryMgmt->lock); +} + +void qCleanupTaskMgmt(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + STaskMgmt* pQueryMgmt = pQMgmt; + int32_t vgId = pQueryMgmt->vgId; + + assert(pQueryMgmt->closed); + + SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; + pQueryMgmt->qinfoPool = NULL; + + taosCacheCleanup(pqinfoPool); + pthread_mutex_destroy(&pQueryMgmt->lock); + tfree(pQueryMgmt); + + qDebug("vgId:%d, queryMgmt cleanup completed", vgId); +} + +void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo) { + if (pMgmt == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + STaskMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + pthread_mutex_lock(&pQueryMgmt->lock); + if (pQueryMgmt->closed) { + pthread_mutex_unlock(&pQueryMgmt->lock); + qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } else { + void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), + (getMaximumIdleDurationSec()*1000)); + pthread_mutex_unlock(&pQueryMgmt->lock); + + return handle; + } +} + +void** qAcquireTask(void* pMgmt, uint64_t _key) { + STaskMgmt *pQueryMgmt = pMgmt; + + if (pQueryMgmt->closed) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + if (pQueryMgmt->qinfoPool == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } + + void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key)); + if (handle == NULL || *handle == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } else { + return handle; + } +} + +void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle) { + STaskMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + return NULL; + } + + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); + return 0; +} + +#if 0 +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { + int32_t error = TSDB_CODE_SUCCESS; + void** handle = qAcquireTask(pMgmt, qId); + if(handle == NULL) return terrno; + + SQInfo* pQInfo = (SQInfo*)(*handle); + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); + setQueryKilled(pQInfo); + + // wait query stop + int32_t loop = 0; + while (pQInfo->owner != 0) { + taosMsleep(waitMs); + if(loop++ > waitCount){ + error = TSDB_CODE_FAILED; + break; + } + } + + qReleaseTask(pMgmt, (void **)&handle, true); + return error; +} + +#endif \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f119627c69..1d2740f0e0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,24 +12,25 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include #include "os.h" #include "tmsg.h" #include "tglobal.h" #include "ttime.h" - #include "exception.h" + +#include "../../../../contrib/cJson/cJSON.h" #include "executorimpl.h" -#include "thash.h" #include "function.h" #include "tcompare.h" #include "tcompression.h" +#include "thash.h" #include "ttypes.h" +#include "query.h" -#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) +#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) -#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) +#define SET_MAIN_SCAN_FLAG(runtime) ((runtime)->scanFlag = MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) @@ -41,11 +42,6 @@ #define MULTI_KEY_DELIM "-" -#define TIME_WINDOW_COPY(_dst, _src) do {\ - (_dst).skey = (_src).skey;\ - (_dst).ekey = (_src).ekey;\ -} while (0) - enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, @@ -131,40 +127,16 @@ do { \ } \ } while (0) -uint64_t queryHandleId = 0; - int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } -int64_t genQueryId(void) { - int64_t uid = 0; - int64_t did = 0;//tsDnodeId; - - uid = did << 54; - - int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; - - uid |= pid << 44; - - int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; - - uid |= ts << 11; - - int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; - - uid |= sid; - -// //qDebug("gen qid:0x%"PRIx64, uid); - - return uid; -} static int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); return 0; } -static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { +static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { tw->skey += pQueryAttr->interval.sliding * factor; @@ -198,28 +170,28 @@ static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, +static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset); -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static bool hasMainOutput(SQueryAttr *pQueryAttr); +static bool hasMainOutput(STaskAttr *pQueryAttr); static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); -static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); +static int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -//static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); +//static STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win); static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); +static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); @@ -232,32 +204,34 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); +void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); + static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; - if (pOperator->pRuntimeEnv != NULL) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + if (pOperator->pTaskInfo != NULL) { + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); } } -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); +static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); +static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); -static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); -static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, +static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); -SArray* getOrderCheckColumns(SQueryAttr* pQuery); +SArray* getOrderCheckColumns(STaskAttr* pQuery); typedef struct SRowCompSupporter { - SQueryRuntimeEnv *pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv; int16_t dataOffset; __compar_fn_t comFunc; } SRowCompSupporter; @@ -267,7 +241,7 @@ static int compareRowData(const void *a, const void *b, const void *userData) { const SResultRow *pRow2 = (const SResultRow *)b; SRowCompSupporter *supporter = (SRowCompSupporter *)userData; - SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; SFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); SFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); @@ -279,7 +253,7 @@ static int compareRowData(const void *a, const void *b, const void *userData) { return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; } -static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { +static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); size_t size = taosArrayGetSize(columnOrderList); taosArrayDestroy(columnOrderList); @@ -375,7 +349,7 @@ static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput // return (numOfSelectivity > 0 && hasTags); } -static bool isProjQuery(SQueryAttr *pQueryAttr) { +static bool isProjQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { @@ -398,7 +372,7 @@ static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, STaskRuntimeEnv* pRuntimeEnv) { // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; @@ -424,7 +398,7 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } -static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, +static bool chkResultRowFromKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); @@ -462,7 +436,7 @@ static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *p } -static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, +static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); @@ -536,7 +510,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult return pResultRowInfo->pResult[pResultRowInfo->curPos]; } -static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { +static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); } else { @@ -561,7 +535,7 @@ static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWin } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value @@ -609,7 +583,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t } // get the correct time window according to the handled timestamp -static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { +static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value @@ -680,14 +654,14 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, +static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, +static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); @@ -816,7 +790,7 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, STaskAttr* pQueryAttr, TSKEY lastKey) { bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); @@ -827,10 +801,10 @@ static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuer } } -static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, +static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; STableQueryInfo* item = pRuntimeEnv->current; int32_t num = -1; @@ -867,9 +841,9 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc return num; } -static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, +static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; bool hasAggregates = pCtx[0].isAggSet; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -904,7 +878,7 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx } } -static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, +static int32_t getNextQualifiedWindow(STaskAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { getNextTimeWindow(pQueryAttr, pNext); @@ -983,7 +957,7 @@ static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow* pNext return startPos; } -static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { +static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *pWindow) { TSKEY ekey = -1; if (QUERY_IS_ASC_QUERY(pQueryAttr)) { ekey = pWindow->ekey; @@ -1012,20 +986,20 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in } } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, +static void saveDataBlockLastRow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, int32_t rowIndex) { if (pDataBlock == NULL) { return; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } -static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { +static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { TSKEY ts = TSKEY_INITIAL_VAL; bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1126,7 +1100,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { @@ -1136,8 +1110,8 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pQueryAttr->window.skey; @@ -1161,7 +1135,7 @@ static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; SQLFunctionCtx* pCtx = pInfo->pCtx; @@ -1226,8 +1200,8 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1257,8 +1231,8 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLF static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperatorInfo->numOfOutput; TSKEY actualEndKey = tsCols[endRowIndex]; @@ -1289,8 +1263,8 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFun static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; if (!pQueryAttr->timeWindowInterpo) { return; } @@ -1340,9 +1314,9 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1361,7 +1335,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, @@ -1450,9 +1424,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; int32_t numOfOutput = pOperatorInfo->numOfOutput; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); @@ -1469,7 +1443,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SResultRow* pResult = NULL; int32_t forwardStep = 0; @@ -1525,12 +1499,12 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int16_t bytes = pColInfoData->info.bytes; int16_t type = pColInfoData->info.type; @@ -1607,13 +1581,13 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; // primary timestamp column SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); SOptrBasicInfo* pBInfo = &pInfo->binfo; int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; @@ -1692,7 +1666,7 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { +static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; @@ -1746,9 +1720,9 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // in case of timestamp column, always generated results. int32_t functionId = pCtx->functionId; @@ -1843,9 +1817,9 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); if (pFuncCtx == NULL) { @@ -1965,9 +1939,9 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { return NULL; } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { +static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pQueryAttr = pQueryAttr; @@ -2019,30 +1993,30 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // case OP_MultiTableTimeInterval: { // pRuntimeEnv->proot = // createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_AllMultiTableTimeInterval: { // pRuntimeEnv->proot = // createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_TimeWindow: { // pRuntimeEnv->proot = // createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_AllTimeWindow: { // pRuntimeEnv->proot = // createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2050,34 +2024,34 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // pRuntimeEnv->proot = // createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); // -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_SessionWindow: { // pRuntimeEnv->proot = // createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } // case OP_MultiTableAggregate: { // pRuntimeEnv->proot = // createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // break; // } // case OP_Aggregate: { // pRuntimeEnv->proot = // createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); // -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2099,9 +2073,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf // // case OP_StateWindow: { // pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; +// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; // if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); +// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); // } // break; // } @@ -2187,8 +2161,8 @@ _clean: return TSDB_CODE_QRY_OUT_OF_MEMORY; } -static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; @@ -2197,7 +2171,7 @@ static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); } -static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { +static void destroyTsComp(STaskRuntimeEnv *pRuntimeEnv, STaskAttr *pQueryAttr) { if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); if (pColInfoData) { @@ -2210,8 +2184,8 @@ static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) } } -static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); @@ -2271,7 +2245,7 @@ bool isQueryKilled(SQInfo *pQInfo) { void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} -//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { +//static bool isFixedOutputQuery(STaskAttr* pQueryAttr) { // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { // return false; // } @@ -2297,7 +2271,7 @@ void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELL //} // todo refactor with isLastRowQuery -//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { +//bool isPointInterpoQuery(STaskAttr *pQueryAttr) { // for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { // int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; // if (functionId == FUNCTION_INTERP) { @@ -2308,7 +2282,7 @@ void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELL // return false; //} -static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { +static bool isFirstLastRowQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionID == FUNCTION_LAST_ROW) { @@ -2319,7 +2293,7 @@ static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { return false; } -static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { +static bool isCachedLastQuery(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { @@ -2354,7 +2328,7 @@ static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { * The following 4 kinds of query are treated as the tags query * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query */ -bool onlyQueryTags(SQueryAttr* pQueryAttr) { +bool onlyQueryTags(STaskAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; @@ -2373,7 +2347,7 @@ bool onlyQueryTags(SQueryAttr* pQueryAttr) { ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { +void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); @@ -2394,7 +2368,7 @@ void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFir /* * todo add more parameters to check soon.. */ -bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { +bool colIdCheck(STaskAttr *pQueryAttr, uint64_t qId) { // load data column information is incorrect for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { @@ -2408,7 +2382,7 @@ bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { // todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which // the scan order is not matter -static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { +static bool onlyOneQueryType(STaskAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -2425,13 +2399,13 @@ static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t fu return true; } -static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } +static bool onlyFirstQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } -static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } +static bool onlyLastQuery(STaskAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } -static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } +static bool notContainSessionOrStateWindow(STaskAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } -static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { +static int32_t updateBlockLoadStatus(STaskAttr *pQuery, int32_t status) { bool hasFirstLastFunc = false; bool hasOtherFunc = false; @@ -2465,7 +2439,7 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { return status; } -static void doUpdateLastKey(SQueryAttr* pQueryAttr) { +static void doUpdateLastKey(STaskAttr* pQueryAttr) { STimeWindow* win = &pQueryAttr->window; size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); @@ -2485,7 +2459,7 @@ static void doUpdateLastKey(SQueryAttr* pQueryAttr) { } static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { - SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // in case of point-interpolation query, use asc order scan char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 @@ -2580,8 +2554,8 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool } } -static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t MIN_ROWS_PER_PAGE = 4; *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); @@ -2596,8 +2570,8 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -//static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { -// SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +// STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { // return true; @@ -2606,7 +2580,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i // return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); //} -static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { +static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); @@ -2655,7 +2629,7 @@ static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlock return false; } -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { +static int32_t doTSJoinFilter(STaskRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); #if defined(_DEBUG_VIEW) @@ -2781,7 +2755,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { } } -void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, +void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; @@ -2823,7 +2797,7 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf tfree(p); } -void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { +void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { int32_t numOfRows = pBlock->info.rows; int8_t *p = NULL; @@ -2913,22 +2887,35 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; - pBlock->pDataBlock = NULL; - pBlock->pBlockAgg = NULL; - - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int64_t groupId = pRuntimeEnv->current->groupIndex; - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - SQInfo* pQInfo = pRuntimeEnv->qinfo; - SQueryCostInfo* pCost = &pQInfo->summary; +int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { + STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; pCost->totalRows += pBlock->info.rows; + pCost->totalCheckedRows += pBlock->info.rows; + pCost->loadBlocks += 1; + +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + if (pBlock->pDataBlock == NULL) { + return terrno; + } +} + +int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + + pBlock->pDataBlock = NULL; + pBlock->pBlockAgg = NULL; + +// int64_t groupId = pRuntimeEnv->current->groupIndex; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + STaskCostInfo* pCost = &pTaskInfo->cost; + + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; +#if 0 if (pRuntimeEnv->pTsBuf != NULL) { (*status) = BLK_DATA_ALL_NEEDED; @@ -2953,7 +2940,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // Calculate all time windows that are overlapping or contain current data block. // If current data block is contained by all possible time window, do not load current data block. if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { + (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) { (*status) = BLK_DATA_ALL_NEEDED; } @@ -2966,7 +2953,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { SResultRow* pResult = NULL; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); @@ -2995,7 +2982,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } SDataBlockInfo* pBlockInfo = &pBlock->info; - *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); +// *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, @@ -3022,7 +3009,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { SResultRow* pResult = NULL; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); @@ -3074,7 +3061,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); // } } - +#endif return TSDB_CODE_SUCCESS; } @@ -3184,10 +3171,10 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num } void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SExprInfo *pExpr = pOperatorInfo->pExpr; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SExprInfo* pExprInfo = &pExpr[0]; int32_t functionId = getExprFunctionId(pExprInfo); @@ -3242,7 +3229,7 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt } } -void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { +void copyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; pBlock->info.rows = 0; @@ -3293,9 +3280,8 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) } } -static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - +static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { +#if 0 int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); @@ -3314,6 +3300,8 @@ static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { // assert(pCheckInfo->pTable == pTableKeyInfo->pTable); } } +#endif + } void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { @@ -3337,7 +3325,7 @@ int32_t initResultRow(SResultRow *pResultRow) { * +------------+-------------------------------------------+-------------------------------------------+ * offset[0] offset[1] offset[2] */ -void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { +void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { SQLFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; @@ -3461,40 +3449,38 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { +void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { if (status == QUERY_NOT_COMPLETED) { - pRuntimeEnv->status = status; + pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->status |= status; + CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + pTaskInfo->status |= status; } } -static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - - if (pRuntimeEnv->pTsBuf) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } +static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// if (pRuntimeEnv->pTsBuf) { +// SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); +// bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); +// assert(ret); +// } // reverse order time range - SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + SWAP(pTableScanInfo->window.skey, pTableScanInfo->window.ekey, TSKEY); - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + SET_REVERSE_SCAN_FLAG(pTableScanInfo); +// setTaskStatus(pTableScanInfo, QUERY_NOT_COMPLETED); switchCtxOrder(pCtx, numOfOutput); - SWITCH_ORDER(pQueryAttr->order.order); - setupQueryRangeForReverseScan(pRuntimeEnv); + SWITCH_ORDER(pTableScanInfo->order); + setupQueryRangeForReverseScan(pTableScanInfo); } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfOutput = pOperator->numOfOutput; if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { @@ -3539,7 +3525,7 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult } } -static bool hasMainOutput(SQueryAttr *pQueryAttr) { +static bool hasMainOutput(STaskAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); @@ -3551,7 +3537,7 @@ static bool hasMainOutput(SQueryAttr *pQueryAttr) { return false; } -STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { +STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { STableQueryInfo *pTableQueryInfo = buf; pTableQueryInfo->win = win; @@ -3602,7 +3588,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3635,7 +3621,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe } } -void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, +void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3659,7 +3645,7 @@ void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pRe setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } -void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, +void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey) { STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; @@ -3675,7 +3661,7 @@ void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, i pRuntimeEnv->prevGroupId = tableGroupId; } -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3698,8 +3684,8 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF } } -void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SSqlExpr* pExpr = &pExprInfo->base; // if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && @@ -3723,8 +3709,8 @@ void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExpr // } } -int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pRuntimeEnv->pTsBuf != NULL); @@ -3766,9 +3752,9 @@ int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, } // TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { #if 0 - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfExprs = pQueryAttr->numOfOutput; for(int32_t i = 0; i < numOfExprs; ++i) { @@ -3801,8 +3787,8 @@ void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx #endif } -void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 int32_t numOfExprs = pQueryAttr->numOfOutput; for(int32_t i = 0; i < numOfExprs; ++i) { @@ -3842,8 +3828,8 @@ void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunction * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there * is a previous result generated or not. */ -void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; @@ -3887,8 +3873,8 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { * @param result */ -static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3946,7 +3932,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* return 0; } -static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); pBlock->info.rows = 0; @@ -3954,7 +3940,7 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRunti return; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t orderType = TSDB_ORDER_ASC;//(pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); @@ -3969,9 +3955,9 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRunti } } -static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // update the number of result for each, only update the number of rows for the corresponding window result. if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { @@ -4000,8 +3986,8 @@ static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, } static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SSDataBlock* pRes = pRuntimeEnv->outputBuf; @@ -4070,7 +4056,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4187,8 +4173,8 @@ void calculateOperatorProfResults(SQInfo* pQInfo) { } void queryCostStatis(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pQInfo->summary; + STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + STaskCostInfo *pSummary = &pQInfo->summary; uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); @@ -4226,8 +4212,8 @@ void queryCostStatis(SQInfo *pQInfo) { } } -//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); @@ -4262,8 +4248,8 @@ void queryCostStatis(SQInfo *pQInfo) { // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} -//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { // return; @@ -4301,8 +4287,8 @@ void queryCostStatis(SQInfo *pQInfo) { // } //} -//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; // // assert(pQueryAttr->limit.offset == 0); @@ -4352,8 +4338,8 @@ void queryCostStatis(SQInfo *pQInfo) { // return true; //} -//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +//static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // assert(*start <= pRuntimeEnv->current->lastKey); // } else { @@ -4463,18 +4449,18 @@ void queryCostStatis(SQInfo *pQInfo) { //} void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { - if (p->upstream == NULL) { - assert(p->numOfUpstream == 0); + if (p->pDownstream == NULL) { + assert(p->numOfDownstream == 0); } - p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); - p->upstream[p->numOfUpstream++] = pUpstream; + p->pDownstream = realloc(p->pDownstream, POINTER_BYTES * (p->numOfDownstream + 1)); + p->pDownstream[p->numOfDownstream++] = pUpstream; } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); -static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 // TODO set the tags scan handle if (onlyQueryTags(pQueryAttr)) { @@ -4533,9 +4519,9 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, void* param) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; pQueryAttr->tsdb = tsdb; if (tsdb != NULL) { @@ -4616,25 +4602,25 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return code; } - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_NOT_COMPLETED); return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { + if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); + (pTableQueryInfo->lastKey >= pTaskInfo->window.skey) && + (pTableQueryInfo->win.skey >= pTaskInfo->window.skey && pTableQueryInfo->win.ekey <= pTaskInfo->window.ekey)); } else { assert( (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && - (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); + (pTableQueryInfo->lastKey <= pTaskInfo->window.skey) && + (pTableQueryInfo->win.skey <= pTaskInfo->window.skey && pTableQueryInfo->win.ekey >= pTaskInfo->window.ekey)); } } -//STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { +//STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { // STsdbQueryCond cond = { // .colList = pQueryAttr->tableCols, // .order = pQueryAttr->order.order, @@ -4676,7 +4662,7 @@ static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { // } //} -static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { +static void doCloseAllTimeWindow(STaskRuntimeEnv* pRuntimeEnv) { size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); for (int32_t i = 0; i < numOfGroup; ++i) { SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); @@ -4693,44 +4679,37 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; + STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SSDataBlock *pBlock = &pTableScanInfo->block; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; -#if 0 - while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + + while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); +// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); // todo opt - if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - STableQueryInfo** pTableQueryInfo = - (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - if (pTableQueryInfo == NULL) { - break; - } - - pRuntimeEnv->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); - - if (pRuntimeEnv->enableGroupData) { - if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { - *newgroup = true; - } - } - - pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; - } +// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { +// STableQueryInfo** pTableQueryInfo = +// (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); +// if (pTableQueryInfo == NULL) { +// break; +// } +// +// pRuntimeEnv->current = *pTableQueryInfo; +// doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); +// } // this function never returns error? uint32_t status; - int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); +// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pRuntimeEnv->env, code); } @@ -4742,7 +4721,6 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { return pBlock; } -#endif return NULL; } @@ -4750,9 +4728,8 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; - STableScanInfo *pTableScanInfo = pOperator->info; - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableScanInfo *pTableScanInfo = pOperator->info; + STaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4775,14 +4752,14 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - if (pRuntimeEnv->pTsBuf) { - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } + setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + pTableScanInfo->scanFlag = REPEAT_SCAN; +// if (pTaskInfo->pTsBuf) { +// bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); +// assert(ret); +// } +// if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = 0; } @@ -4792,17 +4769,15 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } SSDataBlock *p = NULL; + // todo refactor if (pTableScanInfo->reverseTimes > 0) { - setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - + setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, // GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - pRuntimeEnv->scanFlag = REVERSE_SCAN; - pTableScanInfo->times = 1; pTableScanInfo->current = 0; pTableScanInfo->reverseTimes = 0; @@ -4867,31 +4842,31 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { - assert(repeatTime > 0); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { + assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = pRuntimeEnv->pQueryAttr->order.order; - pInfo->current = 0; -// pInfo->prevGroupId = -1; + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableScanOperator"; -// pOperator->operatorType = OP_TableScan; + pOperator->operatorType = OP_TableScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; return pOperator; } -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pQueryHandle = pTsdbQueryHandle; @@ -4915,7 +4890,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE return pOperator; } -SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pQueryHandle = pTsdbQueryHandle; @@ -4998,7 +4973,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -5019,7 +4994,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime return pOptr; } -SArray* getOrderCheckColumns(SQueryAttr* pQuery) { +SArray* getOrderCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); SArray* pOrderColumns = NULL; @@ -5058,7 +5033,7 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery) { return pOrderColumns; } -SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { +SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); SArray* pOrderColumns = NULL; @@ -5109,7 +5084,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevRow); } -SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); @@ -5172,12 +5147,12 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, +SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, int32_t numOfRows, void *merger) { SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); @@ -5252,9 +5227,9 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); // start to flush data into disk and try do multiway merge sort if (pBlock == NULL) { @@ -5288,7 +5263,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; } -SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { +SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); { @@ -5321,7 +5296,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->cleanup = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -5339,17 +5314,17 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5359,8 +5334,8 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); } -// if (upstream->operatorType == OP_DataBlocksOptScan) { -// STableScanInfo* pScanInfo = upstream->info; +// if (downstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = downstream->info; // order = getTableScanOrder(pScanInfo); // } @@ -5386,7 +5361,7 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); @@ -5398,15 +5373,15 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { return pInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5414,8 +5389,8 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); -// if (upstream->operatorType == OP_DataBlocksOptScan) { -// STableScanInfo* pScanInfo = upstream->info; +// if (downstream->operatorType == OP_DataBlocksOptScan) { +// STableScanInfo* pScanInfo = downstream->info; // order = getTableScanOrder(pScanInfo); // } @@ -5455,7 +5430,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; SProjectOperatorInfo* pProjectInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; @@ -5492,16 +5467,16 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { while(1) { bool prevVal = *newgroup; - // The upstream exec may change the value of the newgroup, so use a local variable instead. - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + // The downstream exec may change the value of the newgroup, so use a local variable instead. + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { assert(*newgroup == false); *newgroup = prevVal; - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); break; } @@ -5547,13 +5522,13 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) { } SLimitOperatorInfo* pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; SSDataBlock* pBlock = NULL; while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { doSetOperatorCompleted(pOperator); @@ -5599,12 +5574,12 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) { } SFilterOperatorInfo* pCondInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; while (1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5631,7 +5606,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { @@ -5641,16 +5616,16 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5669,7 +5644,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5690,7 +5665,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); @@ -5701,16 +5676,16 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5729,7 +5704,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5749,7 +5724,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { } STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { int64_t st = taosGetTimestampUs(); @@ -5765,15 +5740,15 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5792,7 +5767,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5809,7 +5784,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { } STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5820,15 +5795,15 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { return pIntervalInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5847,7 +5822,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5862,13 +5837,13 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { } static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->current; SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); SOptrBasicInfo* pBInfo = &pInfo->binfo; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); int16_t bytes = pColInfoData->info.bytes; int16_t type = pColInfoData->info.type; @@ -5945,7 +5920,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { SStateWindowOperatorInfo* pWindowInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); @@ -5956,14 +5931,14 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { return pBInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5981,7 +5956,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6004,7 +5979,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); @@ -6015,17 +5990,17 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { return pBInfo->pRes; } - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; //pQueryAttr->order.order = TSDB_ORDER_ASC; int32_t order = pQueryAttr->order.order; STimeWindow win = pQueryAttr->window; - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } @@ -6041,7 +6016,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6062,7 +6037,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { SGroupbyOperatorInfo *pInfo = pOperator->info; - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); @@ -6073,12 +6048,12 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { return pInfo->binfo.pRes; } - SOperatorInfo* upstream = pOperator->upstream[0]; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { - publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = upstream->exec(upstream, newgroup); - publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->exec(downstream, newgroup); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; } @@ -6095,7 +6070,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); @@ -6117,7 +6092,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { return pInfo->binfo.pRes; } -static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); @@ -6130,7 +6105,7 @@ static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryR *newgroup = true; } -static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, STaskRuntimeEnv *pRuntimeEnv, bool *newgroup) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { *newgroup = false; doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); @@ -6155,16 +6130,16 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { return NULL; } - SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { return pInfo->pRes; } while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { assert(pBlock != NULL); @@ -6220,7 +6195,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) { } // todo set the attribute of query scan count -static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { +static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr) { for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { @@ -6240,23 +6215,23 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { pOperator->cleanup(pOperator->info, pOperator->numOfOutput); } - if (pOperator->upstream != NULL) { - for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { - destroyOperatorInfo(pOperator->upstream[i]); + if (pOperator->pDownstream != NULL) { + for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + destroyOperatorInfo(pOperator->pDownstream[i]); } - tfree(pOperator->upstream); - pOperator->numOfUpstream = 0; + tfree(pOperator->pDownstream); + pOperator->numOfDownstream = 0; } tfree(pOperator->info); tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); @@ -6265,7 +6240,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -6279,7 +6254,7 @@ SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpera pOperator->exec = doAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6353,7 +6328,7 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = destroyOutputBuf(pInfo->pRes); } -SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); @@ -6374,12 +6349,12 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO pOperator->exec = doSTableAggregate; pOperator->cleanup = destroyAggOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); pInfo->seed = rand(); @@ -6390,7 +6365,7 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); - setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "ProjectOperator"; @@ -6404,7 +6379,7 @@ SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = doProjectOperation; pOperator->cleanup = destroyProjectOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6442,7 +6417,7 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3 return 0; } -SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, +SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); @@ -6462,12 +6437,12 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroyConditionOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { +SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream) { SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; @@ -6480,12 +6455,12 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6505,12 +6480,12 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp pOperator->exec = doIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6530,11 +6505,11 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, pOperator->exec = doAllIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); pInfo->colIndex = -1; pInfo->reptScan = false; @@ -6554,10 +6529,10 @@ SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpe pOperator->exec = doStateWindowAgg; pOperator->cleanup = destroyStateWindowOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); @@ -6579,11 +6554,11 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = doSessionWindowAgg; pOperator->cleanup = destroySWindowOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6603,11 +6578,11 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti pOperator->exec = doSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); @@ -6627,20 +6602,20 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRu pOperator->exec = doAllSTableIntervalAgg; pOperator->cleanup = destroyBasicOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); @@ -6660,17 +6635,17 @@ SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato pOperator->exec = hashGroupbyAggregate; pOperator->cleanup = destroyGroupbyOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); pInfo->multigroupResult = multigroupResult; { - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); STimeWindow w = TSWINDOW_INITIALIZER; @@ -6699,14 +6674,14 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doFill; pOperator->cleanup = destroySFillOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } -SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); pInfo->slimit = pQueryAttr->slimit; @@ -6747,7 +6722,7 @@ SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -6758,7 +6733,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { return NULL; } - SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; STagScanInfo *pInfo = pOperator->info; @@ -6770,7 +6745,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pQueryAttr->numOfOutput == 1); SExprInfo* pExprInfo = &pOperator->pExpr[0]; @@ -6874,7 +6849,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { } if (pOperator->status == OP_EXEC_DONE) { - setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); } pRes->info.rows = count; @@ -6883,7 +6858,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { #endif } -SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); @@ -6968,9 +6943,9 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { SSDataBlock* pBlock = NULL; while(1) { - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); - publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); + publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { doSetOperatorCompleted(pOperator); @@ -7021,7 +6996,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } -SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); pInfo->totalBytes = 0; pInfo->buf = NULL; @@ -7045,7 +7020,7 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat pOperator->pExpr = pExpr; pOperator->cleanup = destroyDistinctOperatorInfo; - appendUpstream(pOperator, upstream); + appendUpstream(pOperator, downstream); return pOperator; } @@ -7199,6 +7174,92 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } +/** + * { + "Id": { + "QueryId": 20, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "InputSchema": [{ + "Type": 9, + "ColId": 1, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 2, + "Bytes": 4 + }, { + "Type": 8, + "ColId": 3, + "Bytes": 20 + }], + "TableScan": { + "TableId": 1, + "TableType": 3, + "Flag": 0, + "Window": { + "StartKey": 0, + "EndKey": 0 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ +int32_t parseTaskInfo(const char* msg, int32_t len) { + cJSON* pJson = cJSON_Parse(msg); + if (NULL == pJson) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); + if (NULL != pSub) { + printf("Id : %s\n", pSub->valuestring); + } + + cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); + if (pNode == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); + if (pNodeName == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + printf("node name is: %s\n", pNodeName->valuestring); + + cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); + if (pNodeSchema == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); + if (pOperator == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); + if (pTableId == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); + if (pTimeWindow == NULL) { + return TSDB_CODE_INVALID_MSG; + } + + + +} + /** * pQueryMsg->head has been converted before this function is called. * @@ -7207,7 +7268,7 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t * @param pExpr * @return */ -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param) { int32_t code = TSDB_CODE_SUCCESS; // if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -7924,7 +7985,7 @@ void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFil return NULL; } -int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { +int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { // if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { // pQueryAttr->numOfFilterCols++; @@ -7943,7 +8004,7 @@ int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { +static void doUpdateExprColumnIndex(STaskAttr *pQueryAttr) { assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { @@ -7980,7 +8041,7 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { } } -void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { +void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) { const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); // the minimum number of rows for projection query @@ -8026,7 +8087,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; - SQueryAttr* pQueryAttr = &pQInfo->query; + STaskAttr* pQueryAttr = &pQInfo->query; pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; pQueryAttr->tableGroupInfo = *pTableGroupInfo; @@ -8145,7 +8206,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S pQueryAttr->window = pQueryMsg->window; updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; STimeWindow window = pQueryAttr->window; int32_t index = 0; @@ -8213,7 +8274,7 @@ _cleanup_qinfo: // filterFreeInfo(pFilters); _cleanup: - freeQInfo(pQInfo); + doDestroyTask(pQInfo); return NULL; } @@ -8231,14 +8292,14 @@ bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger) { int32_t code = TSDB_CODE_SUCCESS; - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; pRuntimeEnv->qinfo = pQInfo; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; STSBuf *pTsBuf = NULL; @@ -8271,7 +8332,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { //qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, // pQueryAttr->window.ekey, pQueryAttr->order.order); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; // todo free memory return TSDB_CODE_SUCCESS; @@ -8279,7 +8340,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { //qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); - setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); +// setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } @@ -8292,7 +8353,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* _error: // table query ref will be decrease during error handling - freeQInfo(pQInfo); + doDestroyTask(pQInfo); return code; } @@ -8373,20 +8434,20 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { return NULL; } -void freeQInfo(SQInfo *pQInfo) { +void doDestroyTask(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } //qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; freeQueryAttr(pQueryAttr); // tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); @@ -8407,8 +8468,8 @@ void freeQInfo(SQInfo *pQInfo) { int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { // the remained number of retrieved rows, not the interpolated result - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // load data from file to msg buffer if (pQueryAttr->tsCompQuery) { @@ -8445,7 +8506,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t // all data returned, set query over if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); @@ -8456,7 +8517,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { //qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); - setQueryStatus(pRuntimeEnv, QUERY_OVER); +// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } return TSDB_CODE_SUCCESS; @@ -8535,8 +8596,8 @@ int32_t checkForQueryBuf(size_t numOfTables) { } bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; SSDataBlock* pRes = pRuntimeEnv->outputBuf; @@ -8569,7 +8630,7 @@ void releaseQueryBuf(size_t numOfTables) { atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } -void freeQueryAttr(SQueryAttr* pQueryAttr) { +void freeQueryAttr(STaskAttr* pQueryAttr) { if (pQueryAttr != NULL) { if (pQueryAttr->fillVal != NULL) { tfree(pQueryAttr->fillVal); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 378af4c1d1..90ad1e15f4 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -19,6 +19,7 @@ #include "index.h" #include "index_fst.h" #include "taos.h" +#include "tchecksum.h" #include "thash.h" #include "tlog.h" diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index d7363f2f4c..1e0a88e17f 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -34,6 +34,7 @@ typedef struct WriterCtx { int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len); int (*flush)(struct WriterCtx* ctx); int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); + int (*size)(struct WriterCtx* ctx); WriterType type; union { struct { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 04dac57d8f..19e9375491 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -34,7 +34,10 @@ void indexInit() { // refactor later indexQhandle = taosInitScheduler(INDEX_QUEUE_SIZE, INDEX_NUM_OF_THREADS, "index"); } -void indexCleanUp() { taosCleanUpScheduler(indexQhandle); } +void indexCleanUp() { + // refacto later + taosCleanUpScheduler(indexQhandle); +} static int uidCompare(const void* a, const void* b) { // add more version compare diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 8bc3776ed9..294c8192e8 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -21,7 +21,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 1024 * 1024 * 2 +#define MEM_THRESHOLD 1024 * 1024 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index bfaeeaaa33..4f782cef26 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -935,7 +935,10 @@ Fst* fstCreate(FstSlice* slice) { uint32_t checkSum = 0; len -= sizeof(checkSum); taosDecodeFixedU32(buf + len, &checkSum); - + if (taosCheckChecksum(buf, len, checkSum)) { + // verify fst + return NULL; + } CompiledAddr rootAddr; len -= sizeof(rootAddr); taosDecodeFixedU64(buf + len, &rootAddr); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 0763aae857..6db5555aa6 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -59,6 +59,13 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } return nRead; } +static int writeCtxGetSize(WriterCtx* ctx) { + if (ctx->type == TFile && ctx->file.readOnly) { + // refactor later + return ctx->file.size; + } + return 0; +} static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { // taosFsyncFile(ctx->file.fd); @@ -109,6 +116,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->readFrom = writeCtxDoReadFrom; + ctx->size = writeCtxGetSize; ctx->offset = 0; ctx->limit = capacity; @@ -159,6 +167,8 @@ int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; + + write->summer = taosCalcChecksum(write->summer, buf, len); return len; } int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { @@ -169,7 +179,10 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { + // opt + return write->summer; +} int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 495c4d4477..4b76402560 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -21,8 +21,11 @@ p * #include "index_fst_counting_writer.h" #include "index_util.h" #include "taosdef.h" +#include "tcoding.h" #include "tcompare.h" +const static uint64_t tfileMagicNumber = 0xdb4775248b80fb57ull; + typedef struct TFileFstIter { FstStreamBuilder* fb; StreamWithState* st; @@ -40,9 +43,12 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); static int tfileWriteHeader(TFileWriter* writer); static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); static int tfileWriteData(TFileWriter* write, TFileValue* tval); +static int tfileWriteFooter(TFileWriter* write); +// handle file corrupt later static int tfileReaderLoadHeader(TFileReader* reader); static int tfileReaderLoadFst(TFileReader* reader); +static int tfileReaderVerify(TFileReader* reader); static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); static SArray* tfileGetFileList(const char* path); @@ -138,8 +144,14 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); if (reader == NULL) { return NULL; } - // T_REF_INC(reader); reader->ctx = ctx; + + if (0 != tfileReaderVerify(reader)) { + tfileReaderDestroy(reader); + indexError("invalid tfile, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + return NULL; + } + // T_REF_INC(reader); if (0 != tfileReaderLoadHeader(reader)) { tfileReaderDestroy(reader); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, @@ -296,6 +308,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); tw->fb = NULL; + + tfileWriteFooter(tw); return 0; } void tfileWriterClose(TFileWriter* tw) { @@ -502,6 +516,14 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) { } return 0; } +static int tfileWriteFooter(TFileWriter* write) { + char buf[sizeof(tfileMagicNumber) + 1] = {0}; + void* pBuf = (void*)buf; + taosEncodeFixedU64((void**)(void*)&pBuf, tfileMagicNumber); + int nwrite = write->ctx->write(write->ctx, buf, strlen(buf)); + assert(nwrite == sizeof(tfileMagicNumber)); + return nwrite; +} static int tfileReaderLoadHeader(TFileReader* reader) { // TODO simple tfile header later char buf[TFILE_HEADER_SIZE] = {0}; @@ -527,9 +549,14 @@ static int tfileReaderLoadFst(TFileReader* reader) { if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; - int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); - indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d ", nread, reader->header.fstOffset, ctx->file.buf, - ctx->file.size); + int size = ctx->size(ctx); + + int64_t ts = taosGetTimestampUs(); + int32_t nread = + ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset); + int64_t cost = taosGetTimestampUs() - ts; + indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread < FST_MAX_SIZE); @@ -558,6 +585,25 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* free(buf); return 0; } +static int tfileReaderVerify(TFileReader* reader) { + // just validate header and Footer, file corrupted also shuild be verified later + WriterCtx* ctx = reader->ctx; + + uint64_t tMagicNumber = 0; + + char buf[sizeof(tMagicNumber) + 1] = {0}; + int size = ctx->size(ctx); + + if (size < sizeof(tMagicNumber) || size <= sizeof(reader->header)) { + return -1; + } else if (ctx->readFrom(ctx, buf, sizeof(tMagicNumber), size - sizeof(tMagicNumber)) != sizeof(tMagicNumber)) { + return -1; + } + + taosDecodeFixedU64(buf, &tMagicNumber); + return tMagicNumber == tfileMagicNumber ? 0 : -1; +} + void tfileReaderRef(TFileReader* reader) { if (reader == NULL) { return; } int ref = T_REF_INC(reader); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 70671a5f3e..a2c0046f9a 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,4 +1,5 @@ +#include #include #include #include @@ -12,7 +13,6 @@ #include "index_tfile.h" #include "tskiplist.h" #include "tutil.h" - void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; @@ -293,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -306,13 +306,41 @@ void validateTFile(char* arg) { } tfCleanup(); } + +void iterTFileReader(char* path, char* ver) { + tfInit(); + + int version = atoi(ver); + TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); + Iterate* iter = tfileIteratorCreate(reader); + bool tn = iter ? iter->next(iter) : false; + int count = 0; + int termCount = 0; + while (tn == true) { + count++; + IterateValue* cv = iter->getValue(iter); + termCount += (int)taosArrayGetSize(cv->val); + printf("col val: %s, size: %d\n", cv->colVal, (int)taosArrayGetSize(cv->val)); + tn = iter->next(iter); + } + printf("total size: %d\n term count: %d\n", count, termCount); + + tfileIteratorDestroy(iter); + tfCleanup(); +} + int main(int argc, char* argv[]) { // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } + if (argc > 2) { + // opt + iterTFileReader(argv[1], argv[2]); + } // checkFstCheckIterator(); // checkFstLongTerm(); // checkFstPrefixSearch(); - checkMillonWriteAndReadOfFst(); + // checkMillonWriteAndReadOfFst(); + return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5438f88b76..4f3330b7b3 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -665,14 +665,19 @@ class IndexObj { size_t numOfTable = 100 * 10000) { std::string tColVal = colVal; size_t colValSize = tColVal.size(); + int skip = 100; + numOfTable /= skip; for (int i = 0; i < numOfTable; i++) { - tColVal[i % colValSize] = 'a' + i % 26; + for (int k = 0; k < 10 && k < colVal.size(); k++) { + // opt + tColVal[rand() % colValSize] = 'a' + k % 26; + } SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); - for (size_t i = 0; i < 10; i++) { - int ret = Put(terms, i); + for (size_t j = 0; j < skip; j++) { + int ret = Put(terms, j); assert(ret == 0); } indexMultiTermDestroy(terms); @@ -939,10 +944,11 @@ TEST_F(IndexEnv2, testIndex_read_performance) { TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/multi_tag"; if (index->Init(path) != 0) {} - index->WriteMultiMillonData("tag1", "Hello", 100 * 10000); - index->WriteMultiMillonData("tag2", "Test", 100 * 10000); - index->WriteMultiMillonData("tag3", "Test", 100 * 10000); - index->WriteMultiMillonData("tag4", "Test", 100 * 10000); + int64_t st = taosGetTimestampUs(); + int32_t num = 1000 * 10000; + index->WriteMultiMillonData("tag1", "xxxxxxxxxxxxxxx", num); + std::cout << "numOfRow: " << num << "\ttime cost:" << taosGetTimestampUs() - st << std::endl; + // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); } TEST_F(IndexEnv2, testLongComVal) { std::string path = "/tmp/long_colVal"; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 5cabbb5e3b..faa8c526a0 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3952,7 +3952,8 @@ int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmt pQueryInfo->pTableMetaInfo[0]->pTableMeta = pmt; pQueryInfo->pTableMetaInfo[0]->name = *name; pQueryInfo->numOfTables = 1; - + pQueryInfo->pTableMetaInfo[0]->tagColList = taosArrayInit(4, POINTER_BYTES); + code = setTableVgroupList(pCtx, name, &pQueryInfo->pTableMetaInfo[0]->vgroupList); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(data.pTableMeta); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 63d2e9b855..2a50752c88 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId); -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep); int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index bbb84223ac..7c427efb5a 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -75,24 +75,16 @@ int32_t dsinkNameToDsinkType(const char* name) { return DSINK_Unknown; } -static SDataSink* initDataSink(int32_t type, int32_t size) { - SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); - sink->info.type = type; - sink->info.name = dsinkTypeToDsinkName(type); - return sink; -} - -static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { - SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher)); - return (SDataSink*)dispatcher; -} - -static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks) { - SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter)); - inserter->numOfTables = pBlocks->numOfTables; - inserter->size = pBlocks->size; - SWAP(inserter->pData, pBlocks->pData, char*); - return (SDataSink*)inserter; +static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { + dst->pSchema = malloc(sizeof(SSlotSchema) * src->numOfCols); + if (NULL == dst->pSchema) { + return false; + } + memcpy(dst->pSchema, src->pSchema, sizeof(SSlotSchema) * src->numOfCols); + dst->numOfCols = src->numOfCols; + dst->resultRowSize = src->resultRowSize; + dst->precision = src->precision; + return true; } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { @@ -102,6 +94,10 @@ static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataB return false; } memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; + for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; + } return true; } @@ -120,13 +116,37 @@ static bool cloneExprArray(SArray** dst, SArray* src) { return (TSDB_CODE_SUCCESS == copyAllExprInfo(*dst, src, true) ? true : false); } +static SDataSink* initDataSink(int32_t type, int32_t size, const SPhyNode* pRoot) { + SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); + sink->info.type = type; + sink->info.name = dsinkTypeToDsinkName(type); + if (NULL !=pRoot && !copySchema(&sink->schema, &pRoot->targetSchema)) { + tfree(sink); + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + return sink; +} + +static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks, const SPhyNode* pRoot) { + SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot); + inserter->numOfTables = pBlocks->numOfTables; + inserter->size = pBlocks->size; + SWAP(inserter->pData, pBlocks->pData, char*); + return (SDataSink*)inserter; +} + +static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, const SPhyNode* pRoot) { + SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher), pRoot); + return (SDataSink*)dispatcher; +} + static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { SPhyNode* node = (SPhyNode*)validPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { free(node); - return NULL; + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); } return node; } @@ -149,7 +169,7 @@ static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { // todo - return MASTER_SCAN; + return MAIN_SCAN; } static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { @@ -237,9 +257,10 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); + subplan->msgType = TDMT_VND_QUERY; vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execNode); subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); - subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode); + subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } return pCxt->nextId.templateId++; @@ -248,6 +269,7 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); node->srcTemplateId = srcTemplateId; + node->pSrcEndPoints = validPointer(taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr))); return (SPhyNode*)node; } @@ -313,7 +335,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); vgroupInfoToEpSet(&blocks->vg, &subplan->execNode); - subplan->pDataSink = createDataInserter(pCxt, blocks); + subplan->pDataSink = createDataInserter(pCxt, blocks, NULL); subplan->pNode = NULL; subplan->type = QUERY_TYPE_MODIFY; subplan->msgType = pPayload->msgType; @@ -332,7 +354,7 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { subplan->msgType = TDMT_VND_QUERY; subplan->pNode = createPhyNode(pCxt, pRoot); - subplan->pDataSink = createDataDispatcher(pCxt, pRoot); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); } // todo deal subquery } @@ -359,6 +381,24 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - //todo +void setExchangSourceNode(uint64_t templateId, SQueryNodeAddr* pEp, SPhyNode* pNode) { + if (NULL == pNode) { + return; + } + if (OP_Exchange == pNode->info.type) { + SExchangePhyNode* pExchange = (SExchangePhyNode*)pNode; + if (templateId == pExchange->srcTemplateId) { + taosArrayPush(pExchange->pSrcEndPoints, pEp); + } + } + if (pNode->pChildren != NULL) { + size_t size = taosArrayGetSize(pNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + setExchangSourceNode(templateId, pEp, taosArrayGetP(pNode->pChildren, i)); + } + } +} + +void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* pEp) { + setExchangSourceNode(templateId, pEp, subplan->pNode); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 54bddd0e3f..f80a631413 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -88,8 +88,8 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return TSDB_CODE_SUCCESS; } -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - return setSubplanExecutionNode(subplan, templateId, ep); +void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { + setSubplanExecutionNode(subplan, templateId, ep); } int32_t qSubPlanToString(const SSubplan *subplan, char** str, int32_t* len) { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 270f255ec0..ada9b247ce 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -96,7 +96,7 @@ typedef struct SSchJob { SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* - int8_t status; + int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; int32_t userFetch; @@ -113,10 +113,10 @@ typedef struct SSchJob { #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) -#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) +#define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) -#define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) +#define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st) #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) #define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3f8c75a78c..3ac08b5c42 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -23,6 +23,7 @@ static SSchedulerMgmt schMgmt = {0}; int32_t schValidateStatus(SSchJob *pJob, int8_t oriStatus, int8_t newStatus) { int32_t code = 0; +/* if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -77,6 +78,7 @@ int32_t schValidateStatus(SSchJob *pJob, int8_t oriStatus, int8_t newStatus) { qError("invalid task status:%d", oriStatus); return TSDB_CODE_QRY_APP_ERROR; } +*/ return TSDB_CODE_SUCCESS; @@ -461,6 +463,7 @@ int32_t schProcessOnDataFetched(SSchJob *job) { int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { bool moved = false; + int32_t code = 0; SCH_ERR_RET(schMoveTaskToSuccList(pJob, pTask, &moved)); if (!moved) { @@ -522,7 +525,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&par->childReady, 1); - SCH_ERR_RET(qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->execAddr)); + qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->execAddr); if (SCH_TASK_READY_TO_LUNCH(par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); @@ -539,7 +542,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) SCH_ERR_RET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); if (!needRetry) { - SCH_TASK_ELOG("task failed[%x], no more retry", errCode); + SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode); if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { SCH_ERR_RET(schMoveTaskToFailList(pJob, pTask, &moved)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eaaaf28297..3ea564722b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -277,9 +277,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_NOT_DEPLOYED, "Bnode not deployed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_INVALID_OPTION, "Bnode option invalid") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_READ_FILE_ERROR, "Read bnode.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR, "Write bnode.json error") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnode directories") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED, "Vnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_NOT_DEPLOYED, "Vnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_INVALID_OPTION, "Vnode option invalid") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_READ_FILE_ERROR, "Read vnodes.json error") TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_WRITE_FILE_ERROR, "Write vnodes.json error") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_VNODE_TOO_MANY_VNODES, "Too many vnodes") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress")