From 19e3f91df51daf284d88ff5a94000a860b43e877 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 00:55:56 -0800 Subject: [PATCH 01/34] minor changes --- include/common/tmsg.h | 20 ++++++++++---------- source/dnode/mgmt/impl/inc/dndInt.h | 4 ++-- source/dnode/mgmt/impl/src/dndMgmt.c | 4 ++-- source/dnode/mgmt/impl/src/dnode.c | 6 +++--- source/dnode/mnode/impl/src/mndDb.c | 15 ++++++++------- source/dnode/mnode/impl/src/mndFunc.c | 10 +++++----- source/dnode/mnode/impl/test/db/db.cpp | 16 ++++++++-------- source/libs/parser/src/dCDAstProcess.c | 8 ++++---- source/libs/qcom/src/querymsg.c | 4 ++-- tools/shell/src/backup/tnettest.c | 10 +++++----- 10 files changed, 49 insertions(+), 48 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 632d99878c..8009f8999e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -667,25 +667,25 @@ typedef struct { int8_t walLevel; int8_t quorum; int8_t cacheLastRow; -} SAlterDbMsg; +} SAlterDbReq; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int8_t ignoreNotExists; -} SDropDbMsg; +} SDropDbReq; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int32_t vgVersion; -} SUseDbMsg; +} SUseDbReq; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; -} SSyncDbMsg; +} SSyncDbReq; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; -} SCompactDbMsg; +} SCompactDbReq; typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -699,16 +699,16 @@ typedef struct { int32_t commentSize; int32_t codeSize; char pCont[]; -} SCreateFuncMsg; +} SCreateFuncReq; typedef struct { char name[TSDB_FUNC_NAME_LEN]; -} SDropFuncMsg; +} SDropFuncReq; typedef struct { int32_t numOfFuncs; char pFuncNames[]; -} SRetrieveFuncMsg; +} SRetrieveFuncReq; typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -932,7 +932,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroup; int32_t vgid[]; -} SCompactMsg; +} SCompactReq; typedef struct SShowRsp { int64_t showId; @@ -1042,7 +1042,7 @@ typedef struct { int8_t align[7]; char name[TSDB_STEP_NAME_LEN]; char desc[TSDB_STEP_DESC_LEN]; -} SStartupMsg; +} SStartupReq; // mq related typedef struct { diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index d5c9dd57dc..afdd678213 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -167,7 +167,7 @@ typedef struct SDnode { SBnodeMgmt bmgmt; SVnodesMgmt vmgmt; STransMgmt tmgmt; - SStartupMsg startup; + SStartupReq startup; } SDnode; EStat dndGetStat(SDnode *pDnode); @@ -175,7 +175,7 @@ void dndSetStat(SDnode *pDnode, EStat stat); char *dndStatStr(EStat stat); void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); -void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup); +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 7d071be944..6244d83dad 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -473,12 +473,12 @@ static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) { void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("startup req is received"); - SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg)); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); dndGetStartup(pDnode, pStartup); dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; + SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)}; rpcSendResponse(&rpcRsp); } diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 362838eba0..0e4e2545b5 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -45,14 +45,14 @@ char *dndStatStr(EStat stat) { } void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) { - SStartupMsg *pStartup = &pDnode->startup; + SStartupReq *pStartup = &pDnode->startup; tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); pStartup->finished = 0; } -void dndGetStartup(SDnode *pDnode, SStartupMsg *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupMsg)); +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1a5ea53d3c..c7360471e0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -194,7 +194,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); - if (pDb == NULL) { + if (pDb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; } return pDb; @@ -495,7 +495,7 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { +static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { @@ -646,7 +646,7 @@ UPDATE_DB_OVER: static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SAlterDbMsg *pAlter = pReq->rpcMsg.pCont; + SAlterDbReq *pAlter = pReq->rpcMsg.pCont; pAlter->totalBlocks = htonl(pAlter->totalBlocks); pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); @@ -793,7 +793,7 @@ DROP_DB_OVER: static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SDropDbMsg *pDrop = pReq->rpcMsg.pCont; + SDropDbReq *pDrop = pReq->rpcMsg.pCont; mDebug("db:%s, start to drop", pDrop->db); @@ -823,7 +823,7 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; - SUseDbMsg *pUse = pReq->rpcMsg.pCont; + SUseDbReq *pUse = pReq->rpcMsg.pCont; pUse->vgVersion = htonl(pUse->vgVersion); SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); @@ -836,6 +836,7 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); SUseDbRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { + mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -890,7 +891,7 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SSyncDbMsg *pSync = pReq->rpcMsg.pCont; + SSyncDbReq *pSync = pReq->rpcMsg.pCont; SDbObj *pDb = mndAcquireDb(pMnode, pSync->db); if (pDb == NULL) { mError("db:%s, failed to process sync db req since %s", pSync->db, terrstr()); @@ -903,7 +904,7 @@ static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SCompactDbMsg *pCompact = pReq->rpcMsg.pCont; + SCompactDbReq *pCompact = pReq->rpcMsg.pCont; SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db); if (pDb == NULL) { mError("db:%s, failed to process compact db req since %s", pCompact->db, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 1c809a9b9a..c22c7b5885 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -26,7 +26,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate); static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); @@ -156,7 +156,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNe return 0; } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate) { +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate) { SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); pFunc->createdTime = taosGetTimestampMs(); pFunc->funcType = pCreate->funcType; @@ -264,7 +264,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SCreateFuncMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateFuncReq *pCreate = pMsg->rpcMsg.pCont; pCreate->outputLen = htonl(pCreate->outputLen); pCreate->bufSize = htonl(pCreate->bufSize); pCreate->sigature = htobe64(pCreate->sigature); @@ -323,7 +323,7 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont; + SDropFuncReq *pDrop = pMsg->rpcMsg.pCont; mDebug("func:%s, start to drop", pDrop->name); @@ -353,7 +353,7 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SRetrieveFuncMsg *pRetrieve = pMsg->rpcMsg.pCont; + SRetrieveFuncReq *pRetrieve = pMsg->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index fd860c87d6..1a089e2932 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -136,9 +136,9 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - int32_t contLen = sizeof(SAlterDbMsg); + int32_t contLen = sizeof(SAlterDbReq); - SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(contLen); + SAlterDbReq* pReq = (SAlterDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->totalBlocks = htonl(12); pReq->daysToKeep0 = htonl(300); @@ -205,9 +205,9 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - int32_t contLen = sizeof(SDropDbMsg); + int32_t contLen = sizeof(SDropDbReq); - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); @@ -261,9 +261,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { CheckBinary("d2", TSDB_DB_NAME_LEN - 1); { - int32_t contLen = sizeof(SUseDbMsg); + int32_t contLen = sizeof(SUseDbReq); - SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(contLen); + SUseDbReq* pReq = (SUseDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->vgVersion = htonl(-1); @@ -314,9 +314,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { } { - int32_t contLen = sizeof(SDropDbMsg); + int32_t contLen = sizeof(SDropDbReq); - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 06729813c1..947941b2cf 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -842,11 +842,11 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c goto _error; } - SUseDbMsg* pUseDbMsg = (SUseDbMsg*)calloc(1, sizeof(SUseDbMsg)); + SUseDbReq* pUseDbMsg = (SUseDbReq*)calloc(1, sizeof(SUseDbReq)); tNameExtractFullName(&n, pUseDbMsg->db); pDcl->pMsg = (char*)pUseDbMsg; - pDcl->msgLen = sizeof(SUseDbMsg); + pDcl->msgLen = sizeof(SUseDbReq); pDcl->msgType = TDMT_MND_USE_DB; break; } @@ -895,14 +895,14 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c goto _error; } - SDropDbMsg* pDropDbMsg = (SDropDbMsg*)calloc(1, sizeof(SDropDbMsg)); + SDropDbReq* pDropDbMsg = (SDropDbReq*)calloc(1, sizeof(SDropDbReq)); code = tNameExtractFullName(&name, pDropDbMsg->db); pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T); pDcl->msgType = TDMT_MND_DROP_DB; - pDcl->msgLen = sizeof(SDropDbMsg); + pDcl->msgLen = sizeof(SDropDbReq); pDcl->pMsg = (char*)pDropDbMsg; break; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 093e42a3d2..16cabd6483 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -59,7 +59,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms SBuildUseDBInput* bInput = (SBuildUseDBInput *)input; - int32_t estimateSize = sizeof(SUseDbMsg); + int32_t estimateSize = sizeof(SUseDbReq); if (NULL == *msg || msgSize < estimateSize) { tfree(*msg); *msg = rpcMallocCont(estimateSize); @@ -68,7 +68,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms } } - SUseDbMsg *bMsg = (SUseDbMsg *)*msg; + SUseDbReq *bMsg = (SUseDbReq *)*msg; strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); bMsg->db[sizeof(bMsg->db) - 1] = 0; diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index d6904ddc04..7b5dbd2405 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -338,7 +338,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { return pRpcConn; } -static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupMsg *pStep) { +static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi, SStartupReq *pStep) { SRpcEpSet epSet; SRpcMsg reqMsg; SRpcMsg rspMsg; @@ -374,7 +374,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p } int32_t code = 0; - if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupMsg)) { + if (pStep != NULL && rspMsg.pCont != NULL && rspMsg.contLen > 0 && rspMsg.contLen <= sizeof(SStartupReq)) { memcpy(pStep, rspMsg.pCont, rspMsg.contLen); code = 1; } @@ -384,8 +384,8 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p return code; } -static int32_t taosNetParseStartup(SStartupMsg *pCont) { - SStartupMsg *pStep = pCont; +static int32_t taosNetParseStartup(SStartupReq *pCont) { + SStartupReq *pStep = pCont; uInfo("step:%s desc:%s", pStep->name, pStep->desc); if (pStep->finished) { @@ -398,7 +398,7 @@ static int32_t taosNetParseStartup(SStartupMsg *pCont) { static void taosNetTestStartup(char *host, int32_t port) { uInfo("check startup, host:%s port:%d\n", host, port); - SStartupMsg *pStep = malloc(sizeof(SStartupMsg)); + SStartupReq *pStep = malloc(sizeof(SStartupReq)); while (1) { int32_t code = taosNetCheckRpc(host, port + TSDB_PORT_DNODEDNODE, 20, 0, pStep); if (code > 0) { From 93e94127e287a22eb4e92d5a6683a4d0a67f78ab Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 10 Jan 2022 18:22:12 +0800 Subject: [PATCH 02/34] feature/qnode --- include/libs/qcom/query.h | 1 + source/libs/scheduler/inc/schedulerInt.h | 16 +- source/libs/scheduler/src/scheduler.c | 804 ++++++++++++------ source/libs/scheduler/test/schedulerTests.cpp | 14 +- 4 files changed, 540 insertions(+), 295 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index da88366f11..92bbb074f1 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -35,6 +35,7 @@ enum { JOB_TASK_STATUS_CANCELLING, JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_DROPPING, + JOB_TASK_STATUS_FREEING, }; typedef struct STableComInfo { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 270f255ec0..4195376812 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -61,14 +61,17 @@ typedef struct SSchLevel { typedef struct SSchTask { uint64_t taskId; // task id + SRWLatch lock; // task lock SSchLevel *level; // level SSubplan *plan; // subplan char *msg; // operator tree int32_t msgLen; // msg length int8_t status; // task status - SQueryNodeAddr execAddr; // task actual executed node address + int32_t lastMsgType; // last sent msg type + SQueryNodeAddr succeedAddr; // task executed success node address int8_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr + SArray *execAddrs; // all tried node for current task, element is SQueryNodeAddr SQueryProfileSummary summary; // task execution summary int32_t childReady; // child task ready number SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* @@ -96,22 +99,24 @@ typedef struct SSchJob { SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* + int32_t ref; int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; - int32_t userFetch; + int8_t userFetch; int32_t remoteFetch; SSchTask *fetchTask; int32_t errCode; - void *res; + void *res; //TODO free it or not int32_t resNumOfRows; SQueryProfileSummary summary; } SSchJob; -#define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE #define SCH_TASK_READY_TO_LUNCH(task) (atomic_load_32(&(task)->childReady) >= taosArrayGetSize((task)->children)) + #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) +#define SCH_TASK_NO_NEED_DROP(task) ((task)->plan->type == QUERY_TYPE_MODIFY) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) @@ -127,6 +132,7 @@ typedef struct SSchJob { #define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) @@ -137,7 +143,7 @@ typedef struct SSchJob { static int32_t schLaunchTask(SSchJob *job, SSchTask *task); -static int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType); +static int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); #ifdef __cplusplus } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 25b8ec981e..c13aacb2ad 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -20,9 +20,75 @@ static SSchedulerMgmt schMgmt = {0}; -int32_t schValidateStatus(SSchJob *pJob, int8_t oriStatus, int8_t newStatus) { + +int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { + pTask->plan = pPlan; + pTask->level = pLevel; + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); + pTask->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + pTask->execAddrs = taosArrayInit(SCH_MAX_CONDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + if (NULL == pTask->execAddrs) { + SCH_TASK_ELOG("taosArrayInit %d exec addrs failed", SCH_MAX_CONDIDATE_EP_NUM); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + +void schFreeTask(SSchTask* pTask) { + if (pTask->candidateAddrs) { + taosArrayDestroy(pTask->candidateAddrs); + } + + // TODO NEED TO VERFY WITH ASYNC_SEND MEMORY FREE + //tfree(pTask->msg); + + if (pTask->children) { + taosArrayDestroy(pTask->children); + } + + if (pTask->parents) { + taosArrayDestroy(pTask->parents); + } +} + + +int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { + int32_t lastMsgType = atomic_load_32(&pTask->lastMsgType); + + switch (msgType) { + case TDMT_VND_CREATE_TABLE_RSP: + case TDMT_VND_SUBMIT_RSP: + case TDMT_VND_QUERY_RSP: + case TDMT_VND_RES_READY_RSP: + case TDMT_VND_FETCH_RSP: + case TDMT_VND_DROP_TASK: + if (lastMsgType != (msgType - 1)) { + SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%d, rspType:%d", lastMsgType, msgType); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { + SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + break; + default: + SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); + + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; + int8_t oriStatus = SCH_GET_JOB_STATUS(pJob); + /* if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); @@ -80,6 +146,10 @@ int32_t schValidateStatus(SSchJob *pJob, int8_t oriStatus, int8_t newStatus) { } */ + SCH_SET_JOB_STATUS(pJob, newStatus); + + SCH_JOB_DLOG("status updated from %d to %d", oriStatus, newStatus); + return TSDB_CODE_SUCCESS; _return: @@ -137,6 +207,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + } else { + if (0 != pLevel->level) { + SCH_TASK_ELOG("invalid task info, level:%d, parentNum:%d", pLevel->level, parentNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } } for (int32_t n = 0; n < parentNum; ++n) { @@ -166,19 +241,28 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { return TSDB_CODE_SUCCESS; } -int32_t schInitTask(SSchJob* pJob, SSchTask *pTask, SSubplan* pPlan, SSchLevel *pLevel) { - pTask->plan = pPlan; - pTask->level = pLevel; - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); - pTask->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + +int32_t schRecordTaskSucceedNode(SSchTask *pTask) { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); + + assert(NULL != addr); + + pTask->succeedAddr = *addr; return TSDB_CODE_SUCCESS; } -void schFreeTask(SSchTask* pTask) { - taosArrayDestroy(pTask->candidateAddrs); + +int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr) { + if (NULL == taosArrayPush(pTask->execAddrs, addr)) { + SCH_TASK_ELOG("taosArrayPush addr to execAddr list failed, errno:%d", errno); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; } + int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; @@ -257,7 +341,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - schInitTask(pJob, &task, plan, pLevel); + SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); void *p = taosArrayPush(pLevel->subTasks, &task); if (NULL == p) { @@ -346,46 +430,67 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { if (HASH_NODE_EXIST(code)) { - SCH_TASK_ELOG("task already in exec list, code:%x", code); + SCH_TASK_ELOG("task already in execTask list, code:%x", code); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SCH_TASK_ELOG("taosHashPut task to exec list failed, errno:%d", errno); + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task added to exec list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); + SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); return TSDB_CODE_SUCCESS; } -int32_t schMoveTaskToSuccList(SSchJob *job, SSchTask *task, bool *moved) { - if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { - qError("remove task taskId:0x%"PRIx64" from execTasks failed, reqId:0x%"PRIx64, task->taskId, job->queryId); - return TSDB_CODE_SUCCESS; +int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { + if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { + SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); } - if (0 != taosHashPut(job->succTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { - qError("taosHashPut failed"); + int32_t code = taosHashPut(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + *moved = true; + + SCH_TASK_ELOG("task already in succTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } *moved = true; + + SCH_TASK_DLOG("task moved to succTask list, numOfTasks:%d", taosHashGetSize(pJob->succTasks)); return TSDB_CODE_SUCCESS; } -int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { - if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { - qWarn("remove task[%"PRIx64"] from execTasks failed, it may not exist", task->taskId); +int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { + *moved = false; + + if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { + SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); } - if (0 != taosHashPut(job->failTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { - qError("taosHashPut failed"); + int32_t code = taosHashPut(pJob->failTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + *moved = true; + + SCH_TASK_WLOG("task already in failTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } *moved = true; + + SCH_TASK_DLOG("task moved to failTask list, numOfTasks:%d", taosHashGetSize(pJob->failTasks)); return TSDB_CODE_SUCCESS; } @@ -393,6 +498,8 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info + // TODO set condidateIdx + // TODO record failed but tried task *needRetry = false; @@ -400,60 +507,70 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } -int32_t schFetchFromRemote(SSchJob *job) { +int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; - if (atomic_val_compare_exchange_32(&job->remoteFetch, 0, 1) != 0) { - qInfo("prior fetching not finished"); + if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { + SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); return TSDB_CODE_SUCCESS; } - SCH_ERR_JRET(schBuildAndSendMsg(job, job->fetchTask, TDMT_VND_FETCH)); + if (atomic_load_ptr(&pJob->res)) + + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); return TSDB_CODE_SUCCESS; _return: - atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); return code; } -int32_t schProcessOnJobPartialSuccess(SSchJob *job) { - job->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; - - bool needFetch = job->userFetch; - if ((!SCH_JOB_NEED_FETCH(&job->attr)) && job->attr.syncSchedule) { - tsem_post(&job->rspSem); - } - - if (needFetch) { - SCH_ERR_RET(schFetchFromRemote(job)); - } - - return TSDB_CODE_SUCCESS; -} - +// Note: no more error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { - int8_t status = SCH_GET_JOB_STATUS(pJob); - - if (schValidateStatus(pJob, status, JOB_TASK_STATUS_FAILED)) { - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); + + if (errCode) { + atomic_store_32(&pJob->errCode, errCode); } - - SCH_SET_JOB_STATUS(pJob, JOB_TASK_STATUS_FAILED); - - atomic_store_32(&pJob->errCode, errCode); - atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - - if (pJob->userFetch || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { tsem_post(&pJob->rspSem); } - return TSDB_CODE_SUCCESS; + SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + + assert(0); } + +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { + int32_t code = 0; + + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); + + if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) { + tsem_post(&pJob->rspSem); + } + + if (atomic_load_8(&pJob->userFetch)) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + SCH_ERR_RET(schProcessOnJobFailure(pJob, code)); + + SCH_RET(code); +} + + int32_t schProcessOnDataFetched(SSchJob *job) { atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); @@ -461,25 +578,77 @@ int32_t schProcessOnDataFetched(SSchJob *job) { } -int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { + +// Note: no more error processing, handled in function internal +int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { + bool needRetry = false; bool moved = false; + int32_t taskDone = 0; int32_t code = 0; + + SCH_TASK_DLOG("taskOnFailure, code:%x", errCode); - SCH_ERR_RET(schMoveTaskToSuccList(pJob, pTask, &moved)); - if (!moved) { - SCH_TASK_ELOG(" task may already moved, status:%d", pTask->status); + SCH_ERR_JRET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); + + if (!needRetry) { + SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode); + + if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { + code = schMoveTaskToFailList(pJob, pTask, &moved); + if (code && moved) { + SCH_ERR_RET(errCode); + } + } + + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); + + if (SCH_TASK_NEED_WAIT_ALL(pTask)) { + SCH_LOCK(SCH_WRITE, &pTask->level->lock); + pTask->level->taskFailed++; + taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; + SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); + + atomic_store_32(&pJob->errCode, errCode); + + if (taskDone < pTask->level->taskNum) { + SCH_TASK_DLOG("not all tasks done, done:%d, all:%d", taskDone, pTask->level->taskNum); + SCH_ERR_RET(errCode); + } + } + } else { + // Note: no more error processing, already handled + SCH_ERR_RET(schLaunchTask(pJob, pTask)); + return TSDB_CODE_SUCCESS; } +_return: + + SCH_ERR_RET(schProcessOnJobFailure(pJob, errCode)); + + SCH_ERR_RET(errCode); +} + + +// Note: no more error processing, handled in function internal +int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { + bool moved = false; + int32_t code = 0; + SSchTask *pErrTask = pTask; + + SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); + + code = schMoveTaskToSuccList(pJob, pTask, &moved); + if (code && moved) { + SCH_ERR_RET(code); + } + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + + SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; if (parentNum == 0) { - if (pTask->level->level != 0) { - SCH_TASK_ELOG("no parent task level error, level:%d", pTask->level->level); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - int32_t taskDone = 0; if (SCH_TASK_NEED_WAIT_ALL(pTask)) { @@ -490,22 +659,23 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone < pTask->level->taskNum) { SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); + return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { assert(0); } if (pTask->level->taskFailed > 0) { - pJob->status = JOB_TASK_STATUS_FAILED; - SCH_ERR_RET(schProcessOnJobFailure(pJob, TSDB_CODE_QRY_APP_ERROR)); - - return TSDB_CODE_SUCCESS; + SCH_RET(schProcessOnJobFailure(pJob, 0)); + } else { + SCH_RET(schProcessOnJobPartialSuccess(pJob)); } } else { - pJob->resNode = pTask->execAddr; + pJob->resNode = pTask->succeedAddr; } pJob->fetchTask = pTask; + SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); return TSDB_CODE_SUCCESS; @@ -522,14 +692,18 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); - + pErrTask = par; + atomic_add_fetch_32(&par->childReady, 1); - code = qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->execAddr); + SCH_LOCK(SCH_WRITE, &par->lock); + code = qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr); if (code) { + SCH_UNLOCK(SCH_WRITE, &par->lock); SCH_TASK_ELOG("qSetSubplanExecutionNode failed, code:%x, templateId:%"PRIx64, code, pTask->plan->id.templateId); - SCH_ERR_RET(code); + SCH_ERR_JRET(code); } + SCH_UNLOCK(SCH_WRITE, &par->lock); if (SCH_TASK_READY_TO_LUNCH(par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); @@ -537,158 +711,142 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } return TSDB_CODE_SUCCESS; + +_return: + + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pErrTask, code)); + + SCH_ERR_RET(code); } -int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { - bool needRetry = false; - bool moved = false; - int32_t taskDone = 0; - SCH_ERR_RET(schTaskCheckAndSetRetry(pJob, pTask, errCode, &needRetry)); - - if (!needRetry) { - SCH_TASK_ELOG("task failed and no more retry, code:%x", errCode); - - if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { - SCH_ERR_RET(schMoveTaskToFailList(pJob, pTask, &moved)); - if (!moved) { - SCH_TASK_ELOG("task may already moved, status:%d", pTask->status); - } - } - - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_FAILED); - - if (SCH_TASK_NEED_WAIT_ALL(pTask)) { - SCH_LOCK(SCH_WRITE, &pTask->level->lock); - pTask->level->taskFailed++; - taskDone = pTask->level->taskSucceed + pTask->level->taskFailed; - SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); - - if (taskDone < pTask->level->taskNum) { - qDebug("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); - return TSDB_CODE_SUCCESS; - } - } - - SCH_ERR_RET(schProcessOnJobFailure(pJob, errCode)); - - return errCode; - } - - SCH_ERR_RET(schLaunchTask(pJob, pTask)); - - return TSDB_CODE_SUCCESS; -} - -int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) { int32_t code = 0; + SCH_ERR_JRET(schValidateTaskReceivedMsgType(pJob, pTask, msgType)); + switch (msgType) { case TDMT_VND_CREATE_TABLE_RSP: { - if (rspCode != TSDB_CODE_SUCCESS) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + + break; } - - break; - } case TDMT_VND_SUBMIT_RSP: { - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; - job->resNumOfRows += rsp->affectedRows; + #if 0 //TODO OPEN THIS + SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } + + pJob->resNumOfRows += rsp->affectedRows; + #else + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); + } + #endif + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + break; } case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); - if (code) { - goto _task_error; - } + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); } + + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); + break; } case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); } + + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + break; } case TDMT_VND_FETCH_RSP: { - SCH_ERR_JRET(rspCode); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - job->res = rsp; - if (rsp) { - job->resNumOfRows = rsp->numOfRows; + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } + + atomic_store_ptr(&pJob->res, rsp); + atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); + + SCH_ERR_JRET(schProcessOnDataFetched(pJob)); - SCH_ERR_JRET(schProcessOnDataFetched(job)); break; } case TDMT_VND_DROP_TASK: { - + // SHOULD NEVER REACH HERE + assert(0); + break; } default: - qError("unknown msg type:%d received", msgType); - return TSDB_CODE_QRY_INVALID_INPUT; + SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); + + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } return TSDB_CODE_SUCCESS; -_task_error: - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, code)); - return TSDB_CODE_SUCCESS; - _return: - code = schProcessOnJobFailure(job, code); - return code; + + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); + + SCH_RET(code); } int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; + SSchJob *pJob = NULL; + SSchTask *pTask = NULL; SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { - qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId); + qError("QID:%"PRIx64" taosHashGet queryId not exist", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - int32_t s = taosHashGetSize((*job)->execTasks); + pJob = *job; + + atomic_add_fetch_32(&pJob->ref, 1); + + int32_t s = taosHashGetSize(pJob->execTasks); assert(s != 0); - SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId)); + SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { - qError("failed to get task, taskId:%"PRIx64" not exist, reqId:0x%"PRIx64, pParam->taskId, (*job)->queryId); + qError("QID:%"PRIx64",TID:%"PRIx64" taosHashGet taskId not exist", pParam->queryId, pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } + + pTask = *task; + + SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode); - schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode); + SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: + + if (pJob) { + atomic_sub_fetch_32(&pJob->ref, 1); + } + tfree(param); SCH_RET(code); } @@ -715,7 +873,7 @@ int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) { SSchCallbackParam *pParam = (SSchCallbackParam *)param; - qDebug("drop task rsp received, queryId:%"PRIx64 ",taksId:%"PRIx64 ",code:%d", pParam->queryId, pParam->taskId, code); + qDebug("QID:%"PRIx64",TID:%"PRIx64" drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); } int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { @@ -739,7 +897,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { *fp = schHandleDropCallback; break; default: - qError("unknown msg type:%d", msgType); + qError("unknown msg type for callback, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } @@ -771,12 +929,15 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t pMsgSendInfo->msgInfo.pData = msg; pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = fp; int64_t transporterId = 0; - SCH_ERR_JRET(asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo)); + code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); + if (code) { + qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); + SCH_ERR_JRET(code); + } return TSDB_CODE_SUCCESS; @@ -798,15 +959,19 @@ void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { } } - -int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { +int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; void *msg = NULL; int32_t code = 0; + bool isCandidateAddr = false; SEpSet epSet; - - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - + + if (NULL == addr) { + addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); + + isCandidateAddr = true; + } + schConvertAddrToEpSet(addr, &epSet); switch (msgType) { @@ -887,18 +1052,25 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { break; } default: - SCH_TASK_ELOG("unknown msg type:%d", msgType); + SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } + atomic_store_32(&pTask->lastMsgType, msgType); SCH_ERR_JRET(schAsyncSendMsg(pJob->transport, &epSet, pJob->queryId, pTask->taskId, msgType, msg, msgSize)); + if (isCandidateAddr) { + SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr)); + } + return TSDB_CODE_SUCCESS; _return: + atomic_store_32(&pTask->lastMsgType, -1); + tfree(msg); SCH_RET(code); } @@ -913,13 +1085,19 @@ static FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { || status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING); } + +// Note: no more error processing, handled in function internal int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { int8_t status = 0; int32_t code = 0; if (schJobNeedToStop(pJob, &status)) { SCH_TASK_ELOG("no need to launch task cause of job status, job status:%d", status); - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + + code = atomic_load_32(&pJob->errCode); + SCH_ERR_RET(code); + + SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); } SSubplan *plan = pTask->plan; @@ -941,56 +1119,68 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } - SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, plan->msgType)); + SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); return TSDB_CODE_SUCCESS; _return: - code = schProcessOnTaskFailure(pJob, pTask, code); + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); SCH_RET(code); } int32_t schLaunchJob(SSchJob *pJob) { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); + + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); for (int32_t i = 0; i < level->taskNum; ++i) { SSchTask *pTask = taosArrayGet(level->subTasks, i); SCH_ERR_RET(schLaunchTask(pJob, pTask)); } - - pJob->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } -void schDropJobAllTasks(SSchJob *job) { - void *pIter = taosHashIterate(job->succTasks, NULL); +void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { + if (NULL == pTask->execAddrs) { + SCH_TASK_DLOG("no exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + return; + } + + int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); + + if (size <= 0) { + SCH_TASK_DLOG("empty exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + return; + } + + SQueryNodeAddr *addr = NULL; + for (int32_t i = 0; i < size; ++i) { + addr = (SQueryNodeAddr *)taosArrayGet(pTask->execAddrs, i); + + schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); + } +} + +void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { + void *pIter = taosHashIterate(list, NULL); while (pIter) { - SSchTask *task = *(SSchTask **)pIter; + SSchTask *pTask = *(SSchTask **)pIter; - int32_t msgType = task->plan->msgType; - if (msgType == TDMT_VND_CREATE_TABLE || msgType == TDMT_VND_SUBMIT) { - break; + if (!SCH_TASK_NO_NEED_DROP(pTask)) { + schDropTaskOnExecutedNode(pJob, pTask); } + + pIter = taosHashIterate(list, pIter); + } +} - schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK); - pIter = taosHashIterate(job->succTasks, pIter); - } - - pIter = taosHashIterate(job->failTasks, NULL); - while (pIter) { - SSchTask *task = *(SSchTask **)pIter; - - int32_t msgType = task->plan->msgType; - if (msgType == TDMT_VND_CREATE_TABLE || msgType == TDMT_VND_SUBMIT) { - break; - } - - schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK); - pIter = taosHashIterate(job->succTasks, pIter); - } +void schDropJobAllTasks(SSchJob *pJob) { + schDropTaskInHashList(pJob, pJob->execTasks); + schDropTaskInHashList(pJob, pJob->succTasks); + schDropTaskInHashList(pJob, pJob->failTasks); } int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** job, bool syncSchedule) { @@ -1066,6 +1256,13 @@ _return: SCH_RET(code); } +int32_t schCancelJob(SSchJob *pJob) { + //TODO + + //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + +} + int32_t schedulerInit(SSchedulerCfg *cfg) { if (schMgmt.jobs) { @@ -1104,10 +1301,12 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, true)); + SSchJob *job = NULL; + + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, true)); + + *pJob = job; - SSchJob *job = *(SSchJob **)pJob; - pRes->code = atomic_load_32(&job->errCode); pRes->numOfRows = job->resNumOfRows; @@ -1119,93 +1318,129 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - return schExecJobImpl(transport, nodeList, pDag, pJob, false); -} + SSchJob *job = NULL; + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, false)); -int32_t scheduleFetchRows(void *pJob, void **data) { - if (NULL == pJob || NULL == data) { - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SSchJob *job = pJob; - int32_t code = 0; - - if (!SCH_JOB_NEED_FETCH(&job->attr)) { - qError("no need to fetch data"); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (job->status == JOB_TASK_STATUS_FAILED) { - job->res = NULL; - SCH_RET(atomic_load_32(&job->errCode)); - } - - if (job->status == JOB_TASK_STATUS_SUCCEED) { - job->res = NULL; - return TSDB_CODE_SUCCESS; - } - - if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { - qError("prior fetching not finished"); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (job->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(job)); - } - - tsem_wait(&job->rspSem); - - if (job->status == JOB_TASK_STATUS_FAILED) { - code = atomic_load_32(&job->errCode); - } - - if (job->res && ((SRetrieveTableRsp *)job->res)->completed) { - job->status = JOB_TASK_STATUS_SUCCEED; - } - - *data = job->res; - job->res = NULL; - -_return: - atomic_val_compare_exchange_32(&job->userFetch, 1, 0); - - SCH_RET(code); -} - -int32_t scheduleCancelJob(void *pJob) { - //TODO - - //TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST + *pJob = job; return TSDB_CODE_SUCCESS; } -void scheduleFreeJob(void *pJob) { - if (NULL == pJob) { + +int32_t scheduleFetchRows(void *job, void **data) { + if (NULL == job || NULL == data) { + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SSchJob *pJob = job; + int32_t code = 0; + + atomic_add_fetch_32(&pJob->ref, 1); + + if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { + SCH_JOB_ELOG("no need to fetch data, status:%d", SCH_GET_JOB_STATUS(pJob)); + atomic_sub_fetch_32(&pJob->ref, 1); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { + SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); + atomic_sub_fetch_32(&pJob->ref, 1); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + int8_t status = SCH_GET_JOB_STATUS(pJob); + + if (status == JOB_TASK_STATUS_FAILED) { + *data = atomic_load_ptr(&pJob->res); + atomic_store_ptr(&pJob->res, NULL); + SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); + } else if (status == JOB_TASK_STATUS_SUCCEED) { + *data = atomic_load_ptr(&pJob->res); + atomic_store_ptr(&pJob->res, NULL); + goto _return; + } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); + } + + tsem_wait(&pJob->rspSem); + + status = SCH_GET_JOB_STATUS(pJob); + + if (status == JOB_TASK_STATUS_FAILED) { + code = atomic_load_32(&pJob->errCode); + } + + if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { + SCH_ERR_JRET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); + } + + while (true) { + *data = atomic_load_ptr(&pJob->res); + + if (*data != atomic_val_compare_exchange_ptr(&pJob->res, *data, NULL)) { + continue; + } + + break; + } + +_return: + + atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); + + atomic_sub_fetch_32(&pJob->ref, 1); + + SCH_RET(code); +} + +int32_t scheduleCancelJob(void *job) { + SSchJob *pJob = (SSchJob *)job; + + atomic_add_fetch_32(&pJob->ref, 1); + + int32_t code = schCancelJob(pJob); + + atomic_sub_fetch_32(&pJob->ref, 1); + + SCH_RET(code); +} + +void scheduleFreeJob(void *job) { + if (NULL == job) { return; } - SSchJob *job = pJob; + SSchJob *pJob = job; - if (job->status > 0) { - if (0 != taosHashRemove(schMgmt.jobs, &job->queryId, sizeof(job->queryId))) { - qError("remove job:%"PRIx64"from mgmt failed", job->queryId); // maybe already freed - return; - } - - if (job->status == JOB_TASK_STATUS_EXECUTING) { - scheduleCancelJob(pJob); - } - - schDropJobAllTasks(job); + if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { + SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); + return; } - job->subPlans = NULL; // it is a reference to pDag->pSubplans - int32_t numOfLevels = taosArrayGetSize(job->levels); + while (true) { + int32_t ref = atomic_load_32(&pJob->ref); + if (0 == ref) { + break; + } else if (ref > 0) { + usleep(1); + } else { + assert(0); + } + } + + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { + schCancelJob(pJob); + } + + schDropJobAllTasks(pJob); + + pJob->subPlans = NULL; // it is a reference to pDag->pSubplans + + int32_t numOfLevels = taosArrayGetSize(pJob->levels); for(int32_t i = 0; i < numOfLevels; ++i) { - SSchLevel *pLevel = taosArrayGet(job->levels, i); + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for(int32_t j = 0; j < numOfTasks; ++j) { @@ -1216,12 +1451,15 @@ void scheduleFreeJob(void *pJob) { taosArrayDestroy(pLevel->subTasks); } - taosHashCleanup(job->execTasks); - taosHashCleanup(job->failTasks); - taosHashCleanup(job->succTasks); - taosArrayDestroy(job->levels); + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); - tfree(job); + taosArrayDestroy(pJob->levels); + + tfree(pJob->res); + + tfree(pJob); } void schedulerDestroy(void) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 790778b736..9bc08fa24b 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -36,7 +36,7 @@ namespace { -extern "C" int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); void schtBuildQueryDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000001; @@ -188,7 +188,7 @@ void *schtSendRsp(void *param) { SShellSubmitRspMsg rsp = {0}; rsp.affectedRows = 10; - schProcessRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); + schHandleResponseMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); pIter = taosHashIterate(job->execTasks, pIter); } @@ -233,7 +233,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -244,7 +244,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -255,7 +255,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SQueryTableRsp rsp = {0}; - code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -266,7 +266,7 @@ TEST(queryTest, normalCase) { SSchTask *task = *(SSchTask **)pIter; SResReadyRsp rsp = {0}; - code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); pIter = taosHashIterate(job->execTasks, pIter); @@ -275,7 +275,7 @@ TEST(queryTest, normalCase) { SRetrieveTableRsp rsp = {0}; rsp.completed = 1; rsp.numOfRows = 10; - code = schProcessRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); + code = schHandleResponseMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0); ASSERT_EQ(code, 0); From 8cb0204632a0561099a9974eb628166e09929449 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 03:41:33 -0800 Subject: [PATCH 03/34] minor changes --- tests/script/general/db/basic1.sim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/script/general/db/basic1.sim b/tests/script/general/db/basic1.sim index 44d53917f2..52af7d93ea 100644 --- a/tests/script/general/db/basic1.sim +++ b/tests/script/general/db/basic1.sim @@ -85,4 +85,23 @@ if $data02 != 2 then return -1 endi +return +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +system sh/exec.sh -n dnode1 -s start + +sql show databases + +if $rows != 1 then + return -1 +endi + +sql_error use d1 + +sql use d4 +sql show vgroups + +if $rows != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 2d53617d5c885eccad52b928d9ace840f8a8cc21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Jan 2022 19:48:21 +0800 Subject: [PATCH 04/34] [td-11818] support select * on child table. --- include/common/common.h | 6 + include/dnode/vnode/tsdb/tsdb.h | 134 ++ include/libs/executor/executor.h | 26 +- include/libs/planner/planner.h | 2 +- source/client/src/clientImpl.c | 20 +- source/client/test/clientTests.cpp | 900 ++++---- source/dnode/vnode/impl/src/vnodeQuery.c | 2 +- source/dnode/vnode/tsdb/CMakeLists.txt | 1 + source/dnode/vnode/tsdb/src/tsdbCommit.c | 2 +- source/dnode/vnode/tsdb/src/tsdbRead.c | 2138 +++++++++---------- source/libs/executor/CMakeLists.txt | 31 +- source/libs/executor/inc/dataSinkMgt.h | 2 +- source/libs/executor/inc/executil.h | 4 +- source/libs/executor/inc/executorimpl.h | 50 +- source/libs/executor/src/executil.c | 8 +- source/libs/executor/src/executorMain.c | 241 +-- source/libs/executor/src/executorimpl.c | 323 ++- source/libs/executor/test/CMakeLists.txt | 18 + source/libs/executor/test/executorTests.cpp | 221 ++ source/libs/parser/src/parser.c | 3 + source/libs/planner/inc/plannerInt.h | 2 +- source/libs/planner/src/physicalPlan.c | 11 +- source/libs/planner/src/physicalPlanJson.c | 12 +- source/libs/planner/src/planner.c | 36 +- source/libs/planner/test/phyPlanTests.cpp | 5 +- source/libs/qworker/CMakeLists.txt | 26 +- source/libs/qworker/src/qworker.c | 11 +- source/libs/scheduler/src/scheduler.c | 87 +- 28 files changed, 2298 insertions(+), 2024 deletions(-) create mode 100644 source/libs/executor/test/CMakeLists.txt diff --git a/include/common/common.h b/include/common/common.h index 0ae06e5a7f..9b8a465442 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -62,6 +62,12 @@ typedef struct SConstantItem { SVariant value; } SConstantItem; +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 3aeef4d013..f3d9f71346 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -18,6 +18,7 @@ #include "mallocator.h" #include "meta.h" +#include "common.h" #ifdef __cplusplus extern "C" { @@ -39,6 +40,10 @@ typedef struct STable { STSchema *pSchema; } STable; +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid @@ -58,6 +63,22 @@ typedef struct STsdbCfg { int8_t compression; } STsdbCfg; +// query condition to build multi-table data block iterator +typedef struct STsdbQueryCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} STsdbQueryCond; + +typedef struct { + void *pTable; + TSKEY lastKey; + uint64_t uid; +} STableKeyInfo; + // STsdb STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); void tsdbClose(STsdb *); @@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); +typedef void* tsdbReadHandleT; +/** + * Get the data block iterator, starting from position according to the query condition + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition + * @param qinfo query info handle from query processor + * @return + */ +tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, + void *pRef); + +/** + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. + * @return + */ +//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +// SMemRef *pRef); + + +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); + +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); + +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); + +/** + * move to next block if exists + * + * @param pTsdbReadHandle + * @return + */ +bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); + +/** + * Get current data block information + * + * @param pTsdbReadHandle + * @param pBlockInfo + * @return + */ +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); + +/** + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 + * @return + */ +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); + +/** + * + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. + * + * @param pTsdbReadHandle query handle + * @param pColumnIdList required data columns id list + * @return + */ +SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); + +/** + * destroy the created table group list, which is generated by tag query + * @param pGroupList + */ +void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ +int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); + +/** + * + * @param tsdb + * @param pTableIdList + * @param pGroupInfo + * @return + */ +int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); + +/** + * clean up the query handle + * @param queryHandle + */ +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 17c11b5d09..cf3ff101aa 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -22,23 +22,25 @@ extern "C" { typedef void* qTaskInfo_t; -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param pTaskInfo - * @return - */ -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); + /** + * Create the exec task object according to task json + * @param tsdb + * @param vgId + * @param pTaskInfoMsg + * @param pTaskInfo + * @param qId + * @return + */ +int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main query execution function, including query on both table and multiple tables, + * the main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * * @param qinfo * @return */ -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); +bool qExecTask(qTaskInfo_t qTask); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -81,7 +83,7 @@ int32_t qKillTask(qTaskInfo_t qinfo); * @param qinfo * @return */ -int32_t qIsQueryCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t qinfo); /** * destroy query info structure @@ -113,7 +115,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param numOfIndex * @return */ -int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); +//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); /** * Update the table id list of a given query. diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index c5da68f0a6..e58fa351d0 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -150,7 +150,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d18142cebf..26f97cc6ef 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -197,7 +197,25 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + + SSchema *pSchema = NULL; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + + int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &pResInfo->numOfCols, pRequest->requestId); + if (code != 0) { + return code; + } + + if (pQueryNode->type == TSDB_SQL_SELECT) { + pResInfo->fields = calloc(1, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + pResInfo->fields[i].bytes = pSchema[i].bytes; + pResInfo->fields[i].type = pSchema[i].type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } + } + + return code; } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 73c9fc5e9f..af5ad874f8 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -24,7 +24,6 @@ #include "../inc/clientInt.h" #include "taos.h" -#include "tglobal.h" namespace { void showDB(TAOS* pConn) { @@ -57,449 +56,449 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -// TEST(testCase, drop_db_test) { +//TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // } -// taos_free_result(pRes); // -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } // taos_free_result(pRes); // taos_close(pConn); //} - -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } - - taos_close(pConn); -} - -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } - - taosHashCleanup(phash); -} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} +// +//// TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +////} +// +//TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, show_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, generated_request_id_test) { +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); +//} // TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use test1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - return; - } +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// return; +// } - taos_free_result(pRes); +// taos_free_result(pRes); - pRes = taos_query(pConn, "select * from tm0"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + +// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +// taos_free_result(pRes); + + pRes = taos_query(pConn, "select * from tu"); if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); ASSERT_TRUE(false); } diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index a32ee50df5..0f3ef9a015 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 65dc95a9d8..e38ba1c466 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -13,6 +13,7 @@ else(0) "src/tsdbReadImpl.c" "src/tsdbFile.c" "src/tsdbFS.c" + "src/tsdbRead.c" ) endif(0) diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index 2be032a335..5aab343c4b 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c index da5481dae1..906046ed9a 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/tsdb/src/tsdbRead.c @@ -13,18 +13,23 @@ * along with this program. If not, see . */ +#include "tsdb.h" +#include "tsdbDef.h" +#include "tsdbFS.h" +#include "tsdbLog.h" +#include "tsdbReadImpl.h" +#include "ttime.h" +#include "exception.h" #include "os.h" -#include "tdataformat.h" -#include "tskiplist.h" -#include "tulog.h" #include "talgo.h" #include "tcompare.h" -#include "exception.h" +#include "tdataformat.h" +#include "tskiplist.h" #include "taosdef.h" #include "tlosertree.h" #include "tsdbint.h" -#include "texpr.h" +#include "tmsg.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -34,8 +39,7 @@ ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) + .uid = (_checkInfo)->tableId}) enum { TSDB_QUERY_TYPE_ALL = 1, @@ -62,7 +66,7 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SDFileSet* fileGroup; int32_t slot; - int32_t tid; + uint64_t uid; SArray* pLoadedCols; } SDataBlockLoadInfo; @@ -79,13 +83,13 @@ enum { typedef struct STableCheckInfo { - STableId tableId; + uint64_t tableId; TSKEY lastKey; STable* pTableObj; SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks - uint8_t chosen:2; // indicate which iterator should move forward + uint8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -111,8 +115,8 @@ typedef struct SIOCostSummary { int64_t headFileLoadTime; } SIOCostSummary; -typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; +typedef struct STsdbReadHandle { + STsdb* pTsdb; SQueryFilePos cur; // current position int16_t order; STimeWindow window; // the primary query time window that applies to all queries @@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle { STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size - SMemRef *pMemRef; +// STsdb +// STsdbMemTable * pMemTable; SArray *defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ @@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle { SArray *prev; // previous row which is before than time window SArray *next; // next row which is after the query time window SIOCostSummary cost; -} STsdbQueryHandle; +} STsdbReadHandle; typedef struct STableGroupSupporter { int32_t numOfCols; @@ -154,23 +159,23 @@ typedef struct STableGroupSupporter { } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); -static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); -static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); +static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList); +static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); +//static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); +static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); -static void* doFreeColumnInfoData(SArray* pColumnInfoData); -static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); +//static void* doFreeColumnInfoData(SArray* pColumnInfoData); +//static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; - pBlockLoadInfo->tid = -1; + pBlockLoadInfo->uid = 0; pBlockLoadInfo->fileGroup = NULL; } @@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); +static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle); assert(numOfCols <= TSDB_MAX_COLUMNS); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } return pIdList; } -static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); +static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle); // check if the primary time stamp column needs to load int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); @@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { - assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); - - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); - } - - taosArrayDestroy(psTable); +static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { +// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); +// +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pTsdbReadHandle->pMemRef->ref++ == 0) { +// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); +// } +// +// taosArrayDestroy(psTable); } -static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { // it has been freed - return; - } - - if (--pMemRef->ref == 0) { - tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot)); - } - - pQueryHandle->pMemRef = NULL; +static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { +// assert(pTsdbReadHandle != NULL); +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pMemRef == NULL) { // it has been freed +// return; +// } +// +// if (--pMemRef->ref == 0) { +// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); +// } +// +// pTsdbReadHandle->pMemRef = NULL; } -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; +// +// int64_t rows = 0; +// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; +// if (pMemTable == NULL) { return rows; } +// +//// STableData* pMem = NULL; +//// STableData* pIMem = NULL; +// +//// SMemTable* pMemT = pMemRef->snapshot.mem; +//// SMemTable* pIMemT = pMemRef->snapshot.imem; +// +// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// for (int32_t i = 0; i < size; ++i) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); +// +//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +//// pMem = pMemT->tData[pCheckInfo->tableId]; +//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +//// } +//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +//// pIMem = pIMemT->tData[pCheckInfo->tableId]; +//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +//// } +// } +// return rows; +//} - int64_t rows = 0; - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { return rows; } - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pMemRef->snapshot.mem; - SMemTable* pIMemT = pMemRef->snapshot.imem; - - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows : 0; - } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows : 0; - } - } - return rows; -} - -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1 && pMeta != NULL); + assert(sizeOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; - assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || - info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); +// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || +// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - info.tableId.tid = info.pTableObj->tableId.tid; - info.tableId.uid = info.pTableObj->tableId.uid; + info.tableId = pKeyInfo->uid; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { - info.lastKey = pQueryHandle->window.skey; + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { + info.lastKey = pTsdbReadHandle->window.skey; } - assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); + assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); } else { - assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); + assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey); } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" 0x%"PRIx64, pQueryHandle, info.tableId.uid, - info.tableId.tid, info.lastKey, pQueryHandle->qId); + tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId); } } - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - +// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); size_t gsize = taosArrayGetSize(pTableCheckInfo); - for (int32_t i = 0; i < gsize; ++i) { STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); taosArrayPush(pTable, &pInfo->pTableObj); @@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return pTableCheckInfo; } -static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables >= 1); // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->lastKey = pQueryHandle->window.skey; + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pTsdbReadHandle->window.skey; pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); pCheckInfo->initBuf = false; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.skey); } else { - assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.skey); } } } @@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } -static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); +static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); - STimeWindow* w = &pQueryHandle->window; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + STimeWindow* w = &pTsdbReadHandle->window; + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. -static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { +static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { STsdbCfg* pCfg = &pTsdb->config; int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick } -static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { - pQueryHandle->window = pCond->twindow; +static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) { + pTsdbReadHandle->window = pCond->twindow; bool updateTs = false; - int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (startTs > pQueryHandle->window.skey) { - pQueryHandle->window.skey = startTs; + int64_t startTs = getEarliestValidTimestamp(pTsdbReadHandle->pTsdb); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (startTs > pTsdbReadHandle->window.skey) { + pTsdbReadHandle->window.skey = startTs; pCond->twindow.skey = startTs; updateTs = true; } } else { - if (startTs > pQueryHandle->window.ekey) { - pQueryHandle->window.ekey = startTs; + if (startTs > pTsdbReadHandle->window.ekey) { + pTsdbReadHandle->window.ekey = startTs; pCond->twindow.ekey = startTs; updateTs = true; } @@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p if (updateTs) { tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 - ", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, - pQueryHandle->window.ekey, pQueryHandle->qId); + ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); } } -static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); - if (pQueryHandle == NULL) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + if (pReadHandle == NULL) { goto _end; } - pQueryHandle->order = pCond->order; - pQueryHandle->pTsdb = tsdb; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = INT32_MIN; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->qId = qId; - pQueryHandle->allocSize = 0; - pQueryHandle->locateStart = false; - pQueryHandle->pMemRef = pMemRef; - pQueryHandle->loadType = pCond->type; + pReadHandle->order = pCond->order; + pReadHandle->pTsdb = tsdb; + pReadHandle->type = TSDB_QUERY_TYPE_ALL; + pReadHandle->cur.fid = INT32_MIN; + pReadHandle->cur.win = TSWINDOW_INITIALIZER; + pReadHandle->checkFiles = true; + pReadHandle->activeIndex = 0; // current active table index + pReadHandle->qId = qId; + pReadHandle->allocSize = 0; + pReadHandle->locateStart = false; + pReadHandle->loadType = pCond->type; - pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; - pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; + pReadHandle->outputCapacity = 4096;//((STsdb*)tsdb)->config.maxRowsPerFileBlock; + pReadHandle->loadExternalRow = pCond->loadExternalRows; + pReadHandle->currentLoadExternalRows = pCond->loadExternalRows; - if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { goto _end; } - assert(pCond != NULL && pMemRef != NULL); - setQueryTimewindow(pQueryHandle, pCond); + assert(pCond != NULL); + setQueryTimewindow(pReadHandle, pCond); if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); - if (pQueryHandle->statis == NULL) { + pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + if (pReadHandle->statis == NULL) { goto _end; } // todo: use list instead of array? - pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->pColumns == NULL) { + pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); + if (pReadHandle->pColumns == NULL) { goto _end; } @@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { goto _end; } - taosArrayPush(pQueryHandle->pColumns, &colInfo); - pQueryHandle->statis[i].colId = colInfo.info.colId; + taosArrayPush(pReadHandle->pColumns, &colInfo); + pReadHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); +// STsdbMeta* pMeta = NULL;//tsdbGetMeta(tsdb); +// assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); - if (pQueryHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); + pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock); + if (pReadHandle->pDataCols == NULL) { + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _end; } - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (TsdbQueryHandleT) pQueryHandle; + return (tsdbReadHandleT)pReadHandle; _end: - tsdbCleanupQueryHandle(pQueryHandle); +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); - if (pQueryHandle == NULL) { +tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) { + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (pTsdbReadHandle == NULL) { return NULL; } - if (emptyQueryTimewindow(pQueryHandle)) { - return (TsdbQueryHandleT*) pQueryHandle; + if (emptyQueryTimewindow(pTsdbReadHandle)) { + return (tsdbReadHandleT*) pTsdbReadHandle; } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); - SArray* psTable = NULL; // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); taosArrayDestroy(psTable); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbMayTakeMemSnapshot(pQueryHandle, psTable); +// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable); - tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); - return (TsdbQueryHandleT) pQueryHandle; + tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId); + return (tsdbReadHandleT) pTsdbReadHandle; } -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - if (pCond->order != pQueryHandle->order) { - pQueryHandle->order = pCond->order; - SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + if (pCond->order != pTsdbReadHandle->order) { + pTsdbReadHandle->order = pCond->order; + SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; } - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); - resetCheckInfo(pQueryHandle); + resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); SArray* pTable = NULL; - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); +// STsdbMeta* pMeta = tsdbGetMeta(pTsdbReadHandle->pTsdb); - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); +// pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { +tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + int32_t code = checkForCachedLastRow(pTsdbReadHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { +#if 0 +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLast(pQueryHandle); + int32_t code = checkForCachedLast(pTsdbReadHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { +#endif +SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { assert(pHandle != NULL); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SArray* res = taosArrayInit(size, POINTER_BYTES); for(int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); taosArrayPush(res, &pCheckInfo->pTableObj); } @@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { - taosArrayPush(px, pInfo); - pNew->numOfTables += 1; - break; - } +// if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { +// taosArrayPush(px, pInfo); +// pNew->numOfTables += 1; +// break; +// } } // there are no data in this group @@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { +tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* } } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); - pQueryHandle->loadExternalRow = true; - pQueryHandle->currentLoadExternalRows = true; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + pTsdbReadHandle->loadExternalRow = true; + pTsdbReadHandle->currentLoadExternalRows = true; - return pQueryHandle; + return pTsdbReadHandle; } -static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - +static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pCheckInfo) { if (pCheckInfo->initBuf) { return true; } @@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { - return false; - } +// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) { +// return false; +// } +// +// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); +// + STbData** pMem = NULL; + STbData** pIMem = NULL; - assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; - SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + TKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->mem != NULL) { + pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pMem != NULL) { pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->imem != NULL) { + pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pIMem != NULL) { pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } if (!imemEmpty) { @@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } @@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { return hasNext; } -static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - if (!pCheckInfo->initBuf) { initTableMemIterator(pHandle, pCheckInfo); } @@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { } pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); + tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, + pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { +static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) { int32_t code = 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index); pCheckInfo->numOfBlocks = 0; - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { code = terrno; return code; } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx; // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) { return 0; // no data blocks in the file belongs to pCheckInfo->pTable } @@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { return terrno; } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int return 0; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { +static int32_t getFileCompInfo(STsdbReadHandle* pTsdbReadHandle, int32_t* numOfBlocks) { // load all the comp offset value for all tables in this file int32_t code = TSDB_CODE_SUCCESS; *numOfBlocks = 0; - pQueryHandle->cost.headFileLoad += 1; + pTsdbReadHandle->cost.headFileLoad += 1; int64_t s = taosGetTimestampUs(); size_t numOfTables = 0; - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); - } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { - numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks); + } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); for (int32_t i = 0; i < numOfTables; ++i) { - code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks); if (code != TSDB_CODE_SUCCESS) { int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } } @@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); - int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); + STSchema *pSchema = NULL;//tsdbGetTableSchema(pCheckInfo->pTableObj); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; + int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); + int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle))); if (ret != TSDB_CODE_SUCCESS) { int32_t c = terrno; assert(c != TSDB_CODE_SUCCESS); goto _error; } - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; + pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup; + pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot; + pBlockLoadInfo->uid = pCheckInfo->pTableObj->uid; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for(int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc } int64_t elapsedTime = (taosGetTimestampUs() - st); - pQueryHandle->cost.blockLoadTime += elapsedTime; + pTsdbReadHandle->cost.blockLoadTime += elapsedTime; tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId); return terrno; } -static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); +static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ - SQueryFilePos* cur = &pQueryHandle->cur; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ + SQueryFilePos* cur = &pTsdbReadHandle->cur; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; - /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); + /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); if (key != TSKEY_INITIAL_VAL) { - tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); + tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId); } else { - tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); } - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; - TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); - cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); - pQueryHandle->realNumOfRows = cur->rows; + TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle); + pTsdbReadHandle->realNumOfRows = cur->rows; // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } @@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p // return error, add test cases - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { return code; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p * * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ - assert(pQueryHandle->outputCapacity >= binfo.rows); - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); + assert(pTsdbReadHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo); - if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { - pQueryHandle->realNumOfRows = binfo.rows; + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) { + pTsdbReadHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; cur->win = binfo.window; cur->mixBlock = false; cur->blockCompleted = true; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { cur->lastKey = binfo.window.ekey + 1; cur->pos = binfo.rows; } else { @@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p cur->pos = -1; } } else { // partially copy to dest buffer - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } assert(cur->blockCompleted); if (cur->rows == binfo.rows) { tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId); } else { tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId); } } @@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); if (asc) { // query ended in/started from current block - if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } assert(pCheckInfo->lastKey <= pBlock->keyLast); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc? 0:(pBlock->numOfRows - 1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block - if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } assert(pCheckInfo->lastKey >= pBlock->keyFirst); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc? 0:(pBlock->numOfRows-1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } - *exists = pQueryHandle->realNumOfRows > 0; + *exists = pTsdbReadHandle->realNumOfRows > 0; return code; } @@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c return numOfRows; } - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while(i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c int32_t bytes = pColInfo->info.bytes; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c i++; } - pQueryHandle->cur.win.ekey = tsArray[end]; - pQueryHandle->cur.lastKey = tsArray[end] + step; + pTsdbReadHandle->cur.win.ekey = tsArray[end]; + pTsdbReadHandle->cur.lastKey = tsArray[end] + step; return numOfRows + num; } // Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, - SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, +static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, + SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { char* pData = NULL; STSchema* pSchema; @@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } + if(isRow1DataRow) { numOfColsOfRow1 = schemaNCols(pSchema1); } else { @@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(row2) { isRow2DataRow = isDataRow(row2); if (pSchema2 == NULL) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } if(isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0, j = 0, k = 0; while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, SET_DOUBLE_PTR(pData, value); break; case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { *(TSKEY *)pData = tdGetKey(*(TKEY *)value); } else { *(TSKEY *)pData = *(TSKEY *)value; @@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(forceSetNull) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { + if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { return; } // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; + if (numOfRows < pTsdbReadHandle->outputCapacity) { + int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, +static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { int32_t remain = endPos - startPos + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = (pTsdbReadHandle->outputCapacity - numOfExisted) + startPos - 1; } else { *end = endPos; } @@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos *start = startPos; } else { int32_t remain = (startPos - endPos) + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = startPos + 1 - (pTsdbReadHandle->outputCapacity - numOfExisted); } else { *end = endPos; } @@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos } } -static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void updateInfoAfterMerge(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; + pTsdbReadHandle->realNumOfRows = numOfRows; cur->rows = numOfRows; cur->pos = endPos; } -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; if (cur->rows > 0) { - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey); } else { - assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); + assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); } else { - cur->win = pQueryHandle->window; + cur->win = pTsdbReadHandle->window; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - cur->lastKey = pQueryHandle->window.ekey + step; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + cur->lastKey = pTsdbReadHandle->window.ekey + step; } } -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); int32_t pos = cur->pos; int32_t start = cur->pos; int32_t end = endPos; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(start, end, int32_t); } - assert(pQueryHandle->outputCapacity >= (end - start + 1)); - int32_t numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, 0, start, end); + assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); + int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end); // the time window should always be ascending order: skey <= ekey cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; @@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->blockCompleted = true; // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pQueryHandle, numOfRows, numOfCols); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } -int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { +int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + } else if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey <= pBlockInfo->window.skey) { endPos = 0; cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); } else { assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pTsdbReadHandle->window.ekey, order); cur->mixBlock = true; } @@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { - SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataBlockInfo blockInfo = {0};//GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; - initTableMemIterator(pQueryHandle, pCheckInfo); + initTableMemIterator(pTsdbReadHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); // for search the endPos, so the order needs to reverse - int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); + pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; @@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &blockInfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { SMemRow row2 = NULL; - SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); + SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); if (row1 == NULL) { break; } TSKEY key = memRowKey(row1); - if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { - doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); } if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t qstart = 0, qend = 0; - getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); + getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend); pos += (qend - qstart + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[qend]:tsArray[qstart]; cur->lastKey = cur->win.ekey + step; } - } while (numOfRows < pQueryHandle->outputCapacity); + } while (numOfRows < pTsdbReadHandle->outputCapacity); - if (numOfRows < pQueryHandle->outputCapacity) { + if (numOfRows < pTsdbReadHandle->outputCapacity) { /** * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && - ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && - !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && + ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && + !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } int32_t start = -1, end = -1; - getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); + getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } @@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } - moveDataToFront(pQueryHandle, numOfRows, numOfCols); - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; } -static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { size_t size = sizeof(STableBlockInfo) * numOfBlocks; - if (pQueryHandle->allocSize < size) { - pQueryHandle->allocSize = (int32_t)size; - char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); + if (pTsdbReadHandle->allocSize < size) { + pTsdbReadHandle->allocSize = (int32_t)size; + char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; + pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*) tmp; } - memset(pQueryHandle->pDataBlockInfo, 0, size); + memset(pTsdbReadHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; @@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t numOfQualTables = 0; for (int32_t j = 0; j < numOfTables; ++j) { - STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j); if (pTableCheck->numOfBlocks <= 0) { continue; } @@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO // since there is only one table qualified, blocks are not sorted if (numOfQualTables == 1) { - memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); + memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pQueryHandle, cnt, - pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt, + pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pQueryHandle, cnt, - numOfQualTables, pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt, + numOfQualTables, pTsdbReadHandle->qId); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; - pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; + pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; // set data block index overflow, in order to disable the offset comparator if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pQueryHandle, cnt, pQueryHandle->qId); + tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); return TSDB_CODE_SUCCESS; } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists); -static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlockRv(STsdbReadHandle* pTsdbReadHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; + SQueryFilePos* cur = &pTsdbReadHandle->cur; while(1) { - int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); if (code != TSDB_CODE_SUCCESS || *exists) { return code; } - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // all data blocks in current file has been checked already, try next file if exists - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // next block of the same file cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; - pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; } } } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { - pQueryHandle->numOfBlocks = 0; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + pTsdbReadHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; - assert(pQueryHandle->numOfBlocks == 0); + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; + assert(pTsdbReadHandle->numOfBlocks == 0); break; } - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist } // todo return error code to query engine - if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pTsdbReadHandle, numOfBlocks, &pTsdbReadHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { + assert(numOfBlocks >= pTsdbReadHandle->numOfBlocks); + if (pTsdbReadHandle->numOfBlocks > 0) { break; } } // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { - assert(pQueryHandle->pFileGroup == NULL); + assert(pTsdbReadHandle->pFileGroup == NULL); } cur->fid = INT32_MIN; // denote that there are no data in file anymore @@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist return code; } - assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); - cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fid; + assert(pTsdbReadHandle->pFileGroup != NULL && pTsdbReadHandle->numOfBlocks > 0); + cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 0:pTsdbReadHandle->numOfBlocks-1; + cur->fid = pTsdbReadHandle->pFileGroup->fid; - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pBlockInfo, exists); + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pBlockInfo, exists); } static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { @@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); } -static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; +static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; - assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + SQueryFilePos* cur = &pTsdbReadHandle->cur; + assert(cur->slot < pTsdbReadHandle->numOfBlocks && cur->slot >= 0); cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; } - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; +#if 0 +int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); pTableBlockInfo->numOfFiles += 1; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { numOfBlocks = 0; - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; break; } pTableBlockInfo->numOfFiles += 1; - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); if (numOfBlocks == 0) { continue; } for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); SBlock* pBlock = pCheckInfo->pCompInfo->blocks; for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { @@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist return code; } +#endif -static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); + SQueryFilePos* cur = &pTsdbReadHandle->cur; // find the start data block in file - if (!pQueryHandle->locateStart) { - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + if (!pTsdbReadHandle->locateStart) { + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // check if current file block is all consumed - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pQueryHandle, cur->slot, cur->pos, - pQueryHandle->qId); - int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = (pQueryHandle->realNumOfRows > 0); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos, + pTsdbReadHandle->qId); + int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo); + *exists = (pTsdbReadHandle->realNumOfRows > 0); if (code != TSDB_CODE_SUCCESS || *exists) { return code; @@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is empty, try next block in file // all data blocks in current file has been checked already, try next file if exists - if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { - return getFirstFileDataBlock(pQueryHandle, exists); + if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { - moveToNextDataBlockInCurrentFile(pQueryHandle); - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pNext, exists); + moveToNextDataBlockInCurrentFile(pTsdbReadHandle); + STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pNext, exists); } } } -static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - while (pQueryHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pQueryHandle)) { + while (pTsdbReadHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pTsdbReadHandle)) { return true; } - pQueryHandle->activeIndex += 1; + pTsdbReadHandle->activeIndex += 1; } // no data in memtable or imemtable, decrease the memory reference. // TODO !! -// tsdbMayUnTakeMemSnapshot(pQueryHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); return false; } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // starts from the buffer in case of descending timestamp order check data blocks - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int32_t i = 0; while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && - (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { - break; - } +// if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && +// (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { +// break; +// } i++; } @@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle return; } - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + info.lastKey = pTsdbReadHandle->window.skey; + taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { + STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; int16_t rv = -1; STSchema* pSchema = NULL; do { - SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); if (row == NULL) { break; } TSKEY key = memRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); + if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey); break; } @@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); rv = memRowVersion(row); } - mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) { int32_t emptySize = maxRowsToRead - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle, + elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId); return numOfRows; } static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); @@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - tfree(pInfo->q); - } else { - taosHashCleanup((SHashObj *)(pInfo->q)); - } +// tQueryInfo* pInfo = (tQueryInfo*)param; +// if (pInfo->optr != TSDB_RELATION_IN) { +// tfree(pInfo->q); +// } else { +// taosHashCleanup((SHashObj *)(pInfo->q)); +// } free(param); } -static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { - if (pQueryHandle->checkFiles) { +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 + +static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - tsdbRetrieveDataBlock((TsdbQueryHandleT*) pQueryHandle, NULL); - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); - assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } - pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. + pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. return exists; } - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; } - if (hasMoreDataInCache(pQueryHandle)) { - pQueryHandle->currentLoadExternalRows = false; + if (hasMoreDataInCache(pTsdbReadHandle)) { + pTsdbReadHandle->currentLoadExternalRows = false; return true; } // current result is empty - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { - SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && pTsdbReadHandle->cur.rows == 0) { +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable; - doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); - doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef); - bool result = tsdbGetExternalRow(pQueryHandle); + bool result = tsdbGetExternalRow(pTsdbReadHandle); - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - pQueryHandle->currentLoadExternalRows = false; +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); + pTsdbReadHandle->currentLoadExternalRows = false; return result; } @@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { return false; } -static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { +static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0 && numOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; + SQueryFilePos* cur = &pTsdbReadHandle->cur; SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); + if (++pTsdbReadHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); +// if (ret != TSDB_CODE_SUCCESS) { +// return false; +// } + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); tfree(pRow); // update the last key value @@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { -static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - int32_t numOfRows = 0; - assert(numOfTables > 0 && tgNumOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; - TSKEY priKey = TSKEY_INITIAL_VAL; - int32_t priIdx = -1; - SColumnInfoData* pColInfo = NULL; +//static bool loadCachedLast(STsdbReadHandle* pTsdbReadHandle) { +// // the last row is cached in buffer, return it directly. +// // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER +// int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// int32_t numOfRows = 0; +// assert(numOfTables > 0 && tgNumOfCols > 0); +// SQueryFilePos* cur = &pTsdbReadHandle->cur; +// TSKEY priKey = TSKEY_INITIAL_VAL; +// int32_t priIdx = -1; +// SColumnInfoData* pColInfo = NULL; +// +// while (++pTsdbReadHandle->activeIndex < numOfTables) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// STable* pTable = pCheckInfo->pTableObj; +// char* pData = NULL; +// +// int32_t numOfCols = pTable->maxColNum; +// +// if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { +// tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->uid, pTable->tableId); +// continue; +// } +// +// int32_t i = 0, j = 0; +// while(i < tgNumOfCols && j < numOfCols) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// if (pTable->lastCols[j].colId < pColInfo->info.colId) { +// j++; +// continue; +// } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { +// i++; +// continue; +// } +// +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// if (pTable->lastCols[j].bytes > 0) { +// void* value = pTable->lastCols[j].pData; +// switch (pColInfo->info.type) { +// case TSDB_DATA_TYPE_BINARY: +// case TSDB_DATA_TYPE_NCHAR: +// memcpy(pData, value, varDataTLen(value)); +// break; +// case TSDB_DATA_TYPE_NULL: +// case TSDB_DATA_TYPE_BOOL: +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_UTINYINT: +// *(uint8_t *)pData = *(uint8_t *)value; +// break; +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_USMALLINT: +// *(uint16_t *)pData = *(uint16_t *)value; +// break; +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_UINT: +// *(uint32_t *)pData = *(uint32_t *)value; +// break; +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_UBIGINT: +// *(uint64_t *)pData = *(uint64_t *)value; +// break; +// case TSDB_DATA_TYPE_FLOAT: +// SET_FLOAT_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_DOUBLE: +// SET_DOUBLE_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_TIMESTAMP: +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +// priKey = tdGetKey(*(TKEY *)value); +// priIdx = i; +// +// i++; +// j++; +// continue; +// } else { +// *(TSKEY *)pData = *(TSKEY *)value; +// } +// break; +// default: +// memcpy(pData, value, pColInfo->info.bytes); +// } +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == i) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +//// *(TSKEY *)pData = pTable->lastCols[j].ts; +// continue; +// } +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// assert(numOfRows < pTsdbReadHandle->outputCapacity); +// } +// +// i++; +// j++; +// } +// +// // leave the real ts column as the last row, because last function only (not stable) use the last row as res +// if (priKey != TSKEY_INITIAL_VAL) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// *(TSKEY *)pData = priKey; +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == priIdx) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// } +// +// if (numOfRows > 0) { +// cur->rows = numOfRows; +// cur->mixBlock = true; +// +// return true; +// } +// } +// +// return false; +//} - while (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - STable* pTable = pCheckInfo->pTableObj; - char* pData = NULL; - - int32_t numOfCols = pTable->maxColNum; - - if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { - tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid); - continue; - } - - int32_t i = 0, j = 0; - while(i < tgNumOfCols && j < numOfCols) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pTable->lastCols[j].colId < pColInfo->info.colId) { - j++; - continue; - } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { - i++; - continue; - } - - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - if (pTable->lastCols[j].bytes > 0) { - void* value = pTable->lastCols[j].pData; - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - priKey = tdGetKey(*(TKEY *)value); - priIdx = i; - - i++; - j++; - continue; - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == i) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = pTable->lastCols[j].ts; - continue; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - assert(numOfRows < pQueryHandle->outputCapacity); - } - - i++; - j++; - } - - // leave the real ts column as the last row, because last function only (not stable) use the last row as res - if (priKey != TSKEY_INITIAL_VAL) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - *(TSKEY *)pData = priKey; - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == priIdx) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - } - - if (numOfRows > 0) { - cur->rows = numOfRows; - cur->mixBlock = true; - - return true; - } - } - - return false; -} - - -static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0); int64_t stime = taosGetTimestampUs(); - while(pQueryHandle->activeIndex < numOfTables) { - if (loadBlockOfActiveTable(pQueryHandle)) { + while(pTsdbReadHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pTsdbReadHandle)) { return true; } - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); pCheckInfo->numOfBlocks = 0; - pQueryHandle->activeIndex += 1; - pQueryHandle->locateStart = false; - pQueryHandle->checkFiles = true; - pQueryHandle->cur.rows = 0; - pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + pTsdbReadHandle->activeIndex += 1; + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->cur.rows = 0; + pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow; terrno = TSDB_CODE_SUCCESS; int64_t elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; } return false; } // handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); return false; } @@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t elapsedTime = stime; // TODO refactor: remove "type" - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { - if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { - return loadCachedLastRow(pQueryHandle); - } else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { - return loadCachedLast(pQueryHandle); + if (pTsdbReadHandle->type == TSDB_QUERY_TYPE_LAST) { + if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { +// return loadCachedLastRow(pTsdbReadHandle); + } else if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { +// return loadCachedLast(pTsdbReadHandle); } } - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - return loadDataBlockFromTableSeq(pQueryHandle); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pTsdbReadHandle); } else { // loadType == RR and Offset Order - if (pQueryHandle->checkFiles) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + pTsdbReadHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; } // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); + bool ret = doHasDataInBuffer(pTsdbReadHandle); terrno = TSDB_CODE_SUCCESS; elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; return ret; } } -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { - STsdbQueryHandle* pSecQueryHandle = NULL; +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) { +// STsdbReadHandle* pSecQueryHandle = NULL; +// +// if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) { +// return TSDB_CODE_SUCCESS; +// } +// +// if (type == TSDB_NEXT_ROW && pTsdbReadHandle->next) { +// return TSDB_CODE_SUCCESS; +// } +// +// // prepare the structure +// int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->prev == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } else { +// pTsdbReadHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->next == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } +// +// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// +// SColumnInfoData colInfo = {{0}, 0}; +// colInfo.info = pCol->info; +// colInfo.pData = calloc(1, pCol->info.bytes); +// if (colInfo.pData == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// taosArrayPush(row, &colInfo); +// } +// +// // load the previous row +// STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; +// if (type == TSDB_PREV_ROW) { +// cond.order = TSDB_ORDER_DESC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN}; +// } else { +// cond.order = TSDB_ORDER_ASC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; +// } +// +// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// if (cond.colList == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// for (int32_t i = 0; i < cond.numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); +// } +// +// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef); +// tfree(cond.colList); +// +// // current table, only one table +// STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// +// SArray* psTable = NULL; +// pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); +// if (pSecQueryHandle->pTableCheckInfo == NULL) { +// taosArrayDestroy(psTable); +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// +// tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); +// if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { +// // no result in current query, free the corresponding result rows structure +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// } else { +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); +// } +// +// goto out_of_memory; +// } +// +// SDataBlockInfo blockInfo = {{0}, 0}; +// tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); +// tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); +// +// row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev:pTsdbReadHandle->next; +// int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(row, i); +// SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); +// memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); +// } +// +//out_of_memory: +// tsdbCleanupQueryHandle(pSecQueryHandle); +// return terrno; +//} - if (type == TSDB_PREV_ROW && pQueryHandle->prev) { - return TSDB_CODE_SUCCESS; - } - - if (type == TSDB_NEXT_ROW && pQueryHandle->next) { - return TSDB_CODE_SUCCESS; - } - - // prepare the structure - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); - - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->prev == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } else { - pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->next == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } - - SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCol->info; - colInfo.pData = calloc(1, pCol->info.bytes); - if (colInfo.pData == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - taosArrayPush(row, &colInfo); - } - - // load the previous row - STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; - if (type == TSDB_PREV_ROW) { - cond.order = TSDB_ORDER_DESC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; - } else { - cond.order = TSDB_ORDER_ASC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; - } - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - for (int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); - } - - pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); - tfree(cond.colList); - - // current table, only one table - STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - - SArray* psTable = NULL; - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - taosArrayDestroy(psTable); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - - tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); - if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { - // no result in current query, free the corresponding result rows structure - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - } else { - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - } - - goto out_of_memory; - } - - SDataBlockInfo blockInfo = {{0}, 0}; - tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - - row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; - int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(row, i); - SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); - } - -out_of_memory: - tsdbCleanupQueryHandle(pSecQueryHandle); - return terrno; -} - -bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - SQueryFilePos* cur = &pQueryHandle->cur; +bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + SQueryFilePos* cur = &pTsdbReadHandle->cur; cur->fid = INT32_MIN; cur->mixBlock = true; - if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) { cur->rows = 0; return false; } - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); - SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i); memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { - int32_t code = TSDB_CODE_SUCCESS; +//int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { +// int32_t code = TSDB_CODE_SUCCESS; +// +// TSDB_RLOCK_TABLE(pTable); +// +// if (!pTable->lastRow) { +// code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; +// goto out; +// } +// +// if (pRes) { +// *pRes = tdMemRowDup(pTable->lastRow); +// if (*pRes == NULL) { +// code = TSDB_CODE_TDB_OUT_OF_MEMORY; +// } +// } +// +//out: +// TSDB_RUNLOCK_TABLE(pTable); +// return code; +//} - TSDB_RLOCK_TABLE(pTable); - - if (!pTable->lastRow) { - code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; - goto out; - } - - if (pRes) { - *pRes = tdMemRowDup(pTable->lastRow); - if (*pRes == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - } - } - -out: - TSDB_RUNLOCK_TABLE(pTable); - return code; +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { + return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { - return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; +int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { + assert(pTsdbReadHandle != NULL && groupList != NULL); + +// TSKEY key = TSKEY_INITIAL_VAL; +// +// SArray* group = taosArrayGetP(groupList->pGroupList, 0); +// assert(group != NULL); +// +// STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +// +// int32_t code = 0; +// +// if (((STable*)pInfo->pTable)->lastRow) { +// code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); +// if (code != TSDB_CODE_SUCCESS) { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; +// } else { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; +// } +// } +// +// // update the tsdb query time range +// if (pTsdbReadHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { +// pTsdbReadHandle->window = TSWINDOW_INITIALIZER; +// pTsdbReadHandle->checkFiles = false; +// pTsdbReadHandle->activeIndex = -1; // start from -1 +// } + + return TSDB_CODE_SUCCESS; } -int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { - assert(pQueryHandle != NULL && groupList != NULL); - - TSKEY key = TSKEY_INITIAL_VAL; - - SArray* group = taosArrayGetP(groupList->pGroupList, 0); - assert(group != NULL); - - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); int32_t code = 0; - - if (((STable*)pInfo->pTable)->lastRow) { - code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; - } else { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; - } - } +// if (pTsdbReadHandle->pTsdb && atomic_load_8(&pTsdbReadHandle->pTsdb->hasCachedLastColumn)){ +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; +// } // update the tsdb query time range - if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { - pQueryHandle->window = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 - } - - return code; -} - -int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - - int32_t code = 0; - - if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; - } - - // update the tsdb query time range - if (pQueryHandle->cachelastrow) { - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = -1; // start from -1 } return code; @@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; + TSKEY lastKey = 0;//((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; @@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // keyInfo.pTable may be NULL here. if (pInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pInfo->pTable); +// tsdbUnRefTable(pInfo->pTable); } } @@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; - STable* pTable = NULL; + + uint64_t uid = 0; // there are data in file if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; - pTable = pBlockInfo->pTableCheckInfo->pTableObj; + uid = pBlockInfo->pTableCheckInfo->tableId; } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; + uid = pCheckInfo->tableId; } - pDataBlockInfo->uid = pTable->tableId.uid; - pDataBlockInfo->tid = pTable->tableId.tid; - pDataBlockInfo->rows = cur->rows; + pDataBlockInfo->uid = uid; + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } @@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { + STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { @@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; - assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; @@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; @@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { if (pHandle->cur.mixBlock) { return pHandle->pColumns; } else { - SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); + SDataBlockInfo binfo = {0};/*GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);*/ assert(pHandle->realNumOfRows <= binfo.rows); // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && - pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { + pBlockLoadInfo->uid == pCheckInfo->pTableObj->tid) { return pHandle->pColumns; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; @@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } } - +#if 0 void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return 0; } +#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { return 1; } else { ASSERT(false); @@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; taosArrayPush(g, &info); - tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { STable** prev = taosArrayGet(pTableList, i - 1); @@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); - tsdbRefTable(*p); - assert((*p)->type == TSDB_CHILD_TABLE); +// assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; @@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } +#if 0 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - tsdbRefTable(pKeyInfo->pTable); STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; taosArrayPush(sa, &info); @@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } if (pTable->type != TSDB_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId, pTable->name->data); terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client @@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, - pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, + pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons return terrno; } -int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab } assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - tsdbRefTable(pTable); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; pGroupInfo->numOfTables = 1; @@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab return terrno; } -int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl return terrno; } - tsdbRefTable(pTable); - STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; taosArrayPush(group, &info); } @@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - if (pQueryHandle == NULL) { +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; + if (pTsdbReadHandle == NULL) { return; } - pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); + pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); - taosArrayDestroy(pQueryHandle->defaultLoadColumn); - tfree(pQueryHandle->pDataBlockInfo); - tfree(pQueryHandle->statis); + taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); + tfree(pTsdbReadHandle->pDataBlockInfo); + tfree(pTsdbReadHandle->statis); - if (!emptyQueryTimewindow(pQueryHandle)) { - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (!emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { - assert(pQueryHandle->pTableCheckInfo == NULL); + assert(pTsdbReadHandle->pTableCheckInfo == NULL); } - if (pQueryHandle->pTableCheckInfo != NULL) { - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->pTableCheckInfo != NULL) { + pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); } - tsdbDestroyReadH(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pTsdbReadHandle->rhelper); - tdFreeDataCols(pQueryHandle->pDataCols); - pQueryHandle->pDataCols = NULL; + tdFreeDataCols(pTsdbReadHandle->pDataCols); + pTsdbReadHandle->pDataCols = NULL; - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); + pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); - SIOCostSummary* pCost = &pQueryHandle->cost; + SIOCostSummary* pCost = &pTsdbReadHandle->cost; tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); + pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId); - tfree(pQueryHandle); + tfree(pTsdbReadHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { @@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } +#endif \ No newline at end of file diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index ba941ab22d..04b5fab4bf 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,12 +1,27 @@ aux_source_directory(src EXECUTOR_SRC) -add_library(executor ${EXECUTOR_SRC}) +#add_library(executor ${EXECUTOR_SRC}) + + +#target_link_libraries( +# executor +# PRIVATE os util common function parser planner qcom tsdb +#) + +add_library(executor STATIC ${EXECUTOR_SRC}) +#set_target_properties(executor PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" +# ) +target_link_libraries(executor + PRIVATE os util common function parser planner qcom tsdb + ) + target_include_directories( - executor - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + executor + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - executor - PRIVATE os util common function parser planner qcom -) \ No newline at end of file +#if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/dataSinkMgt.h b/source/libs/executor/inc/dataSinkMgt.h index d13423b25d..d0057a213a 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/source/libs/executor/inc/dataSinkMgt.h @@ -33,7 +33,7 @@ struct SDataSink; struct SSDataBlock; typedef struct SDataSinkMgtCfg { - uint32_t maxDataBlockNum; + uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2c1bf71638..79766b22ac 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); +//int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a268215d3d..86d708856f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,15 +29,8 @@ #include "tpagedfile.h" #include "planner.h" - struct SColumnFilterElem; -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -51,19 +44,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, + // when this task starts to execute, this status will set + TASK_NOT_COMPLETED = 0x1u, - /* query is over + /* Task is over * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 2. when all data within queried time window, it is also denoted as query_completed */ - QUERY_COMPLETED = 0x2u, + TASK_COMPLETED = 0x2u, /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x4u, + TASK_OVER = 0x4u, }; typedef struct SResultRowCell { @@ -129,6 +122,7 @@ typedef struct { } SOperatorProfResult; typedef struct STaskCostInfo { + int64_t created; int64_t start; int64_t end; @@ -246,13 +240,14 @@ typedef struct STaskIdInfo { uint64_t taskId; // this is a subplan id } STaskIdInfo; -typedef struct STaskInfo { +typedef struct SExecTaskInfo { STaskIdInfo id; char *content; uint32_t status; STimeWindow window; STaskCostInfo cost; int64_t owner; // if it is in execution + int32_t code; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -260,8 +255,9 @@ typedef struct STaskInfo { // int32_t dataReady; // denote if query result is ready or not // void* rspContext; // response context char *sql; // query sql string - jmp_buf env; -} STaskInfo; + jmp_buf env; // + struct SOperatorInfo *pRoot; +} SExecTaskInfo; typedef struct STaskRuntimeEnv { jmp_buf env; @@ -269,7 +265,7 @@ typedef struct STaskRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -314,8 +310,8 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - STaskRuntimeEnv *pRuntimeEnv; - STaskInfo *pTaskInfo; + STaskRuntimeEnv *pRuntimeEnv; // todo remove it + SExecTaskInfo *pTaskInfo; struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator @@ -376,7 +372,7 @@ typedef struct STaskParam { } STaskParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -544,7 +540,7 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -572,11 +568,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); +//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +//SSDataBlock* doSLimit(void* param, bool* newgroup); -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -617,14 +613,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); -bool isQueryKilled(SQInfo *pQInfo); +bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); -void destroyUdfInfo(struct SUdfInfo* pUdfInfo); +//void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -644,5 +640,7 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e8ecffb72c..5253add5af 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { -// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int64_t endt = taosGetTimestampMs(); -// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, endt - startt); _end: @@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun break; } -// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_TASKID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } // int64_t elapsedTime = taosGetTimestampUs() - st; -// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index b98b7fef5c..778521ae2d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#include +#include "exception.h" #include "os.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" -#include "exception.h" #include "thash.h" #include "executorimpl.h" @@ -66,152 +67,24 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -// todo parse json to get the operator tree. +int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { + assert(tsdb != NULL && pSubplan != NULL); -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - STaskParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); + int32_t code = doCreateExecTaskInfo(pSubplan, (SExecTaskInfo**) pTaskInfo, tsdb); if (code != TSDB_CODE_SUCCESS) { - goto _over; + goto _error; } - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pTaskInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000}; + code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { - *pTaskInfo = NULL; + goto _error; } -#endif + DataSinkHandle pHandle = NULL; + code = dsCreateDataSinker(pSubplan->pDataSink, &pHandle); + + _error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -250,7 +123,7 @@ int waitMoment(SQInfo* pQInfo){ while(used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isQueryKilled(pQInfo)){ + if(isTaskKilled(pQInfo)){ printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -261,68 +134,66 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); +bool qExecTask(qTaskInfo_t qinfo) { + SExecTaskInfo * pTaskInfo = (SExecTaskInfo *)qinfo; int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*) curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return false; } - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); + if(pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); } - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); -// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" it is already killed, abort", GET_TASKID(pTaskInfo)); +// return doBuildResCheck(pTaskInfo); } +// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; +// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { +// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); +// setTaskStatus(pTaskInfo, TASK_COMPLETED); +// return doBuildResCheck(pTaskInfo); +// } + // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); + int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); + publishQueryAbortEvent(pTaskInfo, ret); + pTaskInfo->code = ret; + qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); +// return doBuildResCheck(pTaskInfo); } - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + qDebug("QInfo:0x%"PRIx64" query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); + void* pOutput = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + // todo put the result into sink node. - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); + + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed", GET_TASKID(pTaskInfo)); +// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { +// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, +// pRuntimeEnv->resultInfo.total); } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); +// qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pTaskInfo->qId, +// GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); } - - return doBuildResCheck(pQInfo); +// return doBuildResCheck(pTaskInfo); } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { @@ -398,13 +269,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) { } int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); + return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); } void qDestroyTask(qTaskInfo_t qHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1d2740f0e0..b5cd9f9287 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -26,6 +26,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); - static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } } @@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn int16_t type = pColInfoData->info.type; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); return; } @@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->param[2].i = pQueryAttr->window.ekey; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; } else if (functionId == FUNCTION_ARITHM) { - pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); +// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); } } @@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; @@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT } } - //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); // group by normal column, sliding window query, interval query are handled by interval query processor // interval (down sampling operation) @@ -2164,8 +2163,8 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; +// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); - destroyUdfInfo(pRuntimeEnv->pUdfInfo); +// destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { return pQInfo->rspContext != NULL; } -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { +bool isTaskKilled(SExecTaskInfo *pTaskInfo) { + if (IS_QUERY_KILLED(pTaskInfo)) { return true; } // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { + if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec()) + /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) { - assert(pQInfo->startExecTs != 0); - //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 + assert(pTaskInfo->cost.start != 0); +// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } @@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; @@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; + } else { + return TSDB_CODE_SUCCESS; } } -int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; pBlock->pDataBlock = NULL; @@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window @@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); // if (pBlock->pDataBlock == NULL) { // return terrno; // } @@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { + if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->status |= status; } } @@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt SWITCH_ORDER(pTableScanInfo->order); setupQueryRangeForReverseScan(pTableScanInfo); + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { @@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI // // int16_t tagType = pCtx[0].tag.nType; // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); // } else { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.i); // } // } @@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz); } else { - //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i); } return -1; @@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // Keep the cursor info of current table pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } @@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p int32_t start = 0; int32_t step = -1; - //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); if (orderType == TSDB_ORDER_ASC) { @@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p } } - //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); pBlock->info.rows = numOfResult; return 0; } @@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} @@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { +// if (isTaskKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; // pTableQueryInfo->lastKey += step; // -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, // pQuery->limit.offset); // } else { // find the appropriated start position in current block // updateOffsetVal(pRuntimeEnv, &blockInfo); @@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) { // pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index // // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, // pQueryAttr->current->lastKey); // // return key; @@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } #endif return terrno; @@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { // case OP_TableBlockInfoScan: { -// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_TableSeqScan: { -// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_DataBlocksOptScan: { -// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); +// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); // break; // } // case OP_TableScan: { -// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); +// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); // break; // } default: { // do nothing @@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { +static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && @@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SSDataBlock *pBlock = &pTableScanInfo->block; STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; - while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + if (isTaskKilled(pOperator->pTaskInfo)) { + longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; -// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); + longjmp(pOperator->pTaskInfo->env, code); } // current block is ignored according to filter result by block statistics data, continue load the next block @@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { return NULL; } else { break; @@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; // if (pTaskInfo->pTsBuf) { @@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pResultRowInfo->curPos = 0; } - //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } SSDataBlock *p = NULL; @@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pTableScanInfo->reverseTimes > 0) { setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; -// pTableScanInfo->order = cond.order; + qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = pResultRowInfo->size - 1; @@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = order; @@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; @@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doGlobalAggregate; +// pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; appendUpstream(pOperator, downstream); @@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; +// pOperator->exec = doMultiwayMergeSort; pOperator->cleanup = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { assert(*newgroup == false); *newgroup = prevVal; - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } @@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); @@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); +// doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); pInfo->numOfFilterCols = numOfFilter; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; -// pOperator->operatorType = OP_SLimit; + pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; +// pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; @@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { count += 1; } - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); *(int64_t*)pColInfo->pData = pInfo->totalTables; count = 1; pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo @@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; } - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); } pRes->info.rows = count; @@ -7174,90 +7175,52 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -/** - * { - "Id": { - "QueryId": 20, - "TemplateId": 0, - "SubplanId": 0 - }, - "Node": { - "Name": "TableScan", - "InputSchema": [{ - "Type": 9, - "ColId": 1, - "Bytes": 8 - }, { - "Type": 4, - "ColId": 2, - "Bytes": 4 - }, { - "Type": 8, - "ColId": 3, - "Bytes": 20 - }], - "TableScan": { - "TableId": 1, - "TableType": 3, - "Flag": 0, - "Window": { - "StartKey": 0, - "EndKey": 0 - } - } - }, - "DataSink": { - "Name": "Dispatch", - "Dispatch": { - } - } +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { + SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + + pthread_mutex_init(&pTaskInfo->lock, NULL); + pTaskInfo->cost.created = taosGetTimestampMs(); + return pTaskInfo; } - */ -int32_t parseTaskInfo(const char* msg, int32_t len) { - cJSON* pJson = cJSON_Parse(msg); - if (NULL == pJson) { - return TSDB_CODE_INVALID_MSG; + +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { + if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { + if (pPhyNode->info.type == OP_TableScan) { + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + pTaskInfo->pRoot = pOperatorInfo; + } } +} - cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); - if (NULL != pSub) { - printf("Id : %s\n", pSub->valuestring); - } +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { + STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + cond.twindow.skey = INT64_MIN; + cond.twindow.ekey = INT64_MAX; + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); - if (pNode == NULL) { - return TSDB_CODE_INVALID_MSG; - } + // todo set the correct table column info + cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; + cond.colList[0].bytes = sizeof(uint64_t); + cond.colList[0].colId = 1; - cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); - if (pNodeName == NULL) { - return TSDB_CODE_INVALID_MSG; - } + cond.colList[1].type = TSDB_DATA_TYPE_INT; + cond.colList[1].bytes = sizeof(int32_t); + cond.colList[1].colId = 2; - printf("node name is: %s\n", pNodeName->valuestring); - - cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); - if (pNodeSchema == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); - if (pOperator == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); - if (pTableId == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); - if (pTimeWindow == NULL) { - return TSDB_CODE_INVALID_MSG; - } + STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + taosArrayPush(pa, &info); + taosArrayPush(group.pGroupList, &pa); + *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); + tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + return TSDB_CODE_SUCCESS; } /** @@ -7704,7 +7667,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; - code = initUdfInfo(pUdfInfo); +// code = initUdfInfo(pUdfInfo); if (code) { return code; } @@ -7996,8 +7959,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); +// doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, +// &pQueryAttr->pFilterInfo, qId); pQueryAttr->createFilterOperator = true; @@ -8505,7 +8468,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t } // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt new file mode 100644 index 0000000000..ece84207c7 --- /dev/null +++ b/source/libs/executor/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + executorTest + PUBLIC os util common transport gtest taos qcom executor function planner +) + +TARGET_INCLUDE_DIRECTORIES( + executorTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/executor/inc" +) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index e69de29bb2..5f5fddbe28 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "trpc.h" +#include "stub.h" +#include "executor.h" + +/** +{ + "Id": { + "QueryId": 1.3108161807422521e+19, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "Targets": [{ + "Base": { + "Schema": { + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 1, + "Type": 9, + "Bytes": 8 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 9, + "ColId": 1, + "Bytes": 8 + } + } + }, { + "Base": { + "Schema": { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 2, + "Type": 4, + "Bytes": 4 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 4, + "ColId": 2, + "Bytes": 4 + } + } + }], + "InputSchema": [{ + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }], + "TableScan": { + "TableId": 1, + "TableType": 2, + "Flag": 0, + "Window": { + "StartKey": -9.2233720368547758e+18, + "EndKey": 9.2233720368547758e+18 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, build_executor_tree_Test) { + + + const char* msg = "{\n" + "\t\"Id\":\t{\n" + "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" + "\t\t\"TemplateId\":\t0,\n" + "\t\t\"SubplanId\":\t0\n" + "\t},\n" + "\t\"Node\":\t{\n" + "\t\t\"Name\":\t\"TableScan\",\n" + "\t\t\"Targets\":\t[{\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}],\n" + "\t\t\"InputSchema\":\t[{\n" + "\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t}],\n" + "\t\t\"TableScan\":\t{\n" + "\t\t\t\"TableId\":\t1,\n" + "\t\t\t\"TableType\":\t2,\n" + "\t\t\t\"Flag\":\t0,\n" + "\t\t\t\"Window\":\t{\n" + "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" + "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" + "\t\t\t}\n" + "\t\t}\n" + "\t},\n" + "\t\"DataSink\":\t{\n" + "\t\t\"Name\":\t\"Dispatch\",\n" + "\t\t\"Dispatch\":\t{\n" + "\t\t}\n" + "\t}\n" + "}"; + + SExecTaskInfo* pTaskInfo = nullptr; + int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo); +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index f440e6cdfe..5f17fcfaee 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; + } else { + terrno = code; + return code; } } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2a50752c88..be266bd415 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -62,7 +62,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfExpr; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index a38d110d5f..7ef434ccf5 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->numOfCols = pPlanNode->numOfExpr; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); if (NULL == dataBlockSchema->pSchema) { return false; } - memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; - for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + for (int32_t i = 0; i < pPlanNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pPlanNode->pExpr, i); + memcpy(&dataBlockSchema->pSchema[i], &pExprInfo->base.resSchema, sizeof(SSlotSchema)); + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; } + return true; } @@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); } - static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d67ff956b9..5175845fee 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - return fromItem(jArray, func, array, itemSize, *size); + if (*array == NULL) { + *array = calloc(*size, itemSize); + } + + return fromItem(jArray, func, *array, itemSize, *size); } static char* getString(const cJSON* json, const char* name) { @@ -728,7 +732,7 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); } if (res) { - res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); + res = fromRawArray(json, jkPnodeSchema, schemaFromJson, (void**)&node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); } if (res) { res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); @@ -889,6 +893,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); + + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f80a631413..de3454ddbb 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -56,35 +56,43 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { - SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pNode, &logicPlan); +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId) { + SQueryPlanNode* pLogicPlan; + int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - // - if (logicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(logicPlan, &str); -// printf("%s\n", str); + if (pLogicPlan->info.type != QNODE_MODIFY) { + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } - code = optimizeQueryPlan(logicPlan); + int32_t numOfOutput = pLogicPlan->numOfExpr; + *pSchema = calloc(numOfOutput, sizeof(SSchema)); + *numOfResCols = numOfOutput; + + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pLogicPlan->pExpr, i); + memcpy(&(*pSchema)[i], pExprInfo->pExpr->pSchema, sizeof(SSchema)); + } + + code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - code = createDag(logicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, requestId); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 3f6f5ab77b..0d3820cc77 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -62,7 +62,10 @@ protected: } SQueryDag* dag = nullptr; uint64_t requestId = 20; - code = qCreateQueryDag(query, &dag, requestId); + SSchema *schema = NULL; + uint32_t numOfOutput = 0; + + code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index ea3e97057b..a3db9c6992 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,31 @@ aux_source_directory(src QWORKER_SRC) -add_library(qworker ${QWORKER_SRC}) +#add_library(qworker ${QWORKER_SRC}) +#target_include_directories( +# qworker +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +#) +# +#target_link_libraries( +# qworker +# PRIVATE os util transport planner qcom executor +#) + +add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( qworker PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - qworker - PRIVATE os util transport planner qcom -) +#set_target_properties(qworker PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# ) + +target_link_libraries(qworker + PRIVATE os util transport planner qcom executor + ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 808c1e19f9..56c60bc7f2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,9 +1,10 @@ #include "qworker.h" -#include "tname.h" +#include "executor.h" #include "planner.h" #include "query.h" #include "qworkerInt.h" #include "tmsg.h" +#include "tname.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -1011,7 +1012,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { bool queryDone = false; bool queryRsped = false; bool needStop = false; - SSubplan *plan = NULL; + struct SSubplan *plan = NULL; QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { @@ -1025,8 +1026,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(code); } - //TODO call executer to init subquery - code = 0; // return error directly + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); //TODO call executer to init subquery if (code) { @@ -1040,7 +1041,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { queryRsped = true; //TODO call executer to execute subquery - code = 0; + code = qExecTask(pTaskInfo); void *data = NULL; queryDone = false; //TODO call executer to execute subquery diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ac08b5c42..e4285dc444 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -181,7 +181,6 @@ void schFreeTask(SSchTask* pTask) { int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; - pJob->queryId = pDag->queryId; if (pDag->numOfSubplans <= 0) { @@ -256,7 +255,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + schInitTask(pJob, &task, plan, pLevel); void *p = taosArrayPush(pLevel->subTasks, &task); @@ -279,7 +278,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - if (planToTask) { taosHashCleanup(planToTask); } @@ -592,60 +590,59 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms break; } case TDMT_VND_SUBMIT_RSP: { - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; - job->resNumOfRows += rsp->affectedRows; + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; + job->resNumOfRows += rsp->affectedRows; - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); - if (code) { - goto _task_error; - } + SQueryTableRsp *rsp = (SQueryTableRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_RES_READY_RSP: { - SResReadyRsp *rsp = (SResReadyRsp *)msg; - - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_FETCH_RSP: { - SCH_ERR_JRET(rspCode); - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + SCH_ERR_JRET(rspCode); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - job->res = rsp; - if (rsp) { - job->resNumOfRows = rsp->numOfRows; - } - - SCH_ERR_JRET(schProcessOnDataFetched(job)); - break; + job->res = rsp; + if (rsp) { + job->resNumOfRows = rsp->numOfRows; } + + SCH_ERR_JRET(schProcessOnDataFetched(job)); + break; + } case TDMT_VND_DROP_TASK: { - - } + } default: qError("unknown msg type:%d received", msgType); return TSDB_CODE_QRY_INVALID_INPUT; @@ -1111,7 +1108,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void } int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { - if (NULL == transport || /*NULL == nodeList || */NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { + if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } From 867f6ba8f39a1b5e77e279621c5d74e53d0e729d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 10 Jan 2022 06:59:10 -0500 Subject: [PATCH 05/34] TD-12678 datasink to json --- source/libs/planner/src/physicalPlanJson.c | 37 ++++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d67ff956b9..992360e5ea 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -198,6 +198,29 @@ static bool schemaFromJson(const cJSON* json, void* obj) { return true; } +static const char* jkDataBlockSchemaSlotSchema = "SlotSchema"; +static const char* jkDataBlockSchemaResultRowSize = "resultRowSize"; +static const char* jkDataBlockSchemaPrecision = "Precision"; + +static bool dataBlockSchemaToJson(const void* obj, cJSON* json) { + const SDataBlockSchema* schema = (const SDataBlockSchema*)obj; + bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema), schema->numOfCols); + if (res) { + res = cJSON_AddNumberToObject(json, jkDataBlockSchemaResultRowSize, schema->resultRowSize); + } + if (res) { + res = cJSON_AddNumberToObject(json, jkDataBlockSchemaPrecision, schema->precision); + } + return res; +} + +static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { + SDataBlockSchema* schema = (SDataBlockSchema*)obj; + schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); + schema->precision = getNumber(json, jkDataBlockSchemaPrecision); + return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, schema->pSchema, sizeof(SSlotSchema), &schema->numOfCols); +} + static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; static const char* jkColumnFilterInfoUpperRelOptr = "UpperRelOptr"; static const char* jkColumnFilterInfoFilterstr = "Filterstr"; @@ -708,7 +731,7 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { res = addArray(jNode, jkPnodeConditions, exprInfoToJson, phyNode->pConditions); } if (res) { - res = addRawArray(jNode, jkPnodeSchema, schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols); + res = addObject(jNode, jkPnodeSchema, dataBlockSchemaToJson, &phyNode->targetSchema); } if (res) { res = addArray(jNode, jkPnodeChildren, phyNodeToJson, phyNode->pChildren); @@ -728,7 +751,7 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); } if (res) { - res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); + res = fromObject(json, jkPnodeSchema, dataBlockSchemaFromJson, &node->targetSchema, true); } if (res) { res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); @@ -786,6 +809,7 @@ static bool specificDataSinkFromJson(const cJSON* json, void* obj) { } static const char* jkDataSinkName = "Name"; +static const char* jkDataSinkSchema = "Schema"; static bool dataSinkToJson(const void* obj, cJSON* json) { const SDataSink* dsink = (const SDataSink*)obj; @@ -793,6 +817,9 @@ static bool dataSinkToJson(const void* obj, cJSON* json) { if (res) { res = addObject(json, dsink->info.name, specificDataSinkToJson, dsink); } + if (res) { + res = addObject(json, jkDataSinkSchema, dataBlockSchemaToJson, &dsink->schema); + } return res; } @@ -800,7 +827,11 @@ static bool dataSinkFromJson(const cJSON* json, void* obj) { SDataSink* dsink = (SDataSink*)obj; dsink->info.name = getString(json, jkDataSinkName); dsink->info.type = dsinkNameToDsinkType(dsink->info.name); - return fromObject(json, dsink->info.name, specificDataSinkFromJson, dsink, true); + bool res = fromObject(json, jkDataSinkSchema, dataBlockSchemaFromJson, &dsink->schema, true); + if (res) { + res = fromObject(json, dsink->info.name, specificDataSinkFromJson, dsink, true); + } + return res; } static const char* jkIdQueryId = "QueryId"; From a326b29021eb76cfb36484e8cc063c93a993bc49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 04:44:11 -0800 Subject: [PATCH 06/34] minor changes --- include/common/tmsg.h | 300 ++++++------------ include/libs/planner/planner.h | 2 +- include/libs/transport/trpc.h | 10 +- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mgmt/impl/test/sut/inc/sut.h | 4 +- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 2 +- source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndCluster.c | 4 +- source/dnode/mnode/impl/src/mndConsumer.c | 10 +- source/dnode/mnode/impl/src/mndDb.c | 10 +- source/dnode/mnode/impl/src/mndDnode.c | 8 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndProfile.c | 8 +- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 52 +-- source/dnode/mnode/impl/src/mndTopic.c | 18 +- source/dnode/mnode/impl/src/mndUser.c | 4 +- source/dnode/mnode/impl/src/mndVgroup.c | 68 ++-- source/dnode/mnode/impl/src/mnode.c | 4 +- source/dnode/mnode/impl/test/db/db.cpp | 12 +- source/dnode/mnode/impl/test/show/show.cpp | 2 +- source/dnode/mnode/impl/test/stb/stb.cpp | 18 +- source/dnode/vnode/impl/inc/vnodeRequest.h | 6 +- source/dnode/vnode/impl/src/vnodeQuery.c | 10 +- source/dnode/vnode/impl/src/vnodeRequest.c | 4 +- source/dnode/vnode/tsdb/inc/tsdbMemTable.h | 2 +- source/dnode/vnode/tsdb/src/tsdbMemTable.c | 2 +- source/libs/catalog/test/catalogTests.cpp | 32 +- source/libs/executor/inc/executorimpl.h | 8 +- source/libs/executor/src/executorMain.c | 2 +- source/libs/executor/src/executorimpl.c | 14 +- source/libs/parser/inc/astToMsg.h | 6 +- source/libs/parser/src/astToMsg.c | 22 +- source/libs/parser/src/dCDAstProcess.c | 6 +- source/libs/planner/src/physicalPlanJson.c | 8 +- source/libs/qcom/src/querymsg.c | 10 +- source/libs/qworker/src/qworker.c | 10 +- source/libs/qworker/test/qworkerTests.cpp | 56 ++-- source/libs/scheduler/src/scheduler.c | 14 +- source/libs/scheduler/test/schedulerTests.cpp | 2 +- 43 files changed, 334 insertions(+), 440 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8009f8999e..fdf64b7af2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -133,36 +133,36 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) -typedef struct SKv { +typedef struct { int32_t keyLen; int32_t valueLen; void* key; void* value; } SKv; -typedef struct SClientHbKey { +typedef struct { int32_t connId; int32_t hbType; } SClientHbKey; -typedef struct SClientHbReq { +typedef struct { SClientHbKey connKey; SHashObj* info; // hash } SClientHbReq; -typedef struct SClientHbBatchReq { +typedef struct { int64_t reqId; SArray* reqs; // SArray } SClientHbBatchReq; -typedef struct SClientHbRsp { +typedef struct { SClientHbKey connKey; int32_t status; int32_t bodyLen; void* body; } SClientHbRsp; -typedef struct SClientHbBatchRsp { +typedef struct { int64_t reqId; int64_t rspId; SArray* rsps; // SArray @@ -220,13 +220,13 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) return buf; } -typedef struct SBuildTableMetaInput { +typedef struct { int32_t vgId; char* dbName; char* tableFullName; } SBuildTableMetaInput; -typedef struct SBuildUseDBInput { +typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int32_t vgVersion; } SBuildUseDBInput; @@ -234,17 +234,12 @@ typedef struct SBuildUseDBInput { #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta -typedef struct { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; -} SEpAddrMsg; - typedef struct { char fqdn[TSDB_FQDN_LEN]; uint16_t port; } SEpAddr; -typedef struct SMsgHead { +typedef struct { int32_t contLen; int32_t vgId; } SMsgHead; @@ -262,7 +257,7 @@ typedef struct SSubmitBlk { } SSubmitBlk; // Submit message for this TSDB -typedef struct SSubmitMsg { +typedef struct { SMsgHead header; int64_t version; int32_t length; @@ -301,7 +296,7 @@ typedef struct { int32_t failedRows; // number of failed records (exclude duplicate records) int32_t numOfFailedBlocks; SShellSubmitRspBlock failedBlocks[]; -} SShellSubmitRspMsg; +} SShellSubmitRsp; typedef struct SSchema { int8_t type; @@ -310,98 +305,24 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -typedef struct { - int32_t contLen; - int32_t vgId; - int8_t tableType; - int16_t numOfColumns; - int16_t numOfTags; - int32_t tid; - int32_t sversion; - int32_t tversion; - int32_t tagDataLen; - int32_t sqlDataLen; - uint64_t uid; - uint64_t superTableUid; - uint64_t createdTime; - char tableFname[TSDB_TABLE_FNAME_LEN]; - char stbFname[TSDB_TABLE_FNAME_LEN]; - char data[]; -} SMDCreateTableMsg; - -// typedef struct { -// int32_t len; // one create table message -// char tableName[TSDB_TABLE_FNAME_LEN]; -// int16_t numOfColumns; -// int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string -// int8_t igExists; -// int8_t rspMeta; -// int8_t reserved[16]; -// char schema[]; -//} SCreateTableMsg; - -typedef struct { - char tableName[TSDB_TABLE_FNAME_LEN]; - int16_t numOfColumns; - int16_t numOfTags; - int8_t igExists; - int8_t rspMeta; - char schema[]; -} SCreateCTableMsg; - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; int32_t numOfTags; int32_t numOfColumns; SSchema pSchema[]; -} SCreateStbMsg, SCreateTableMsg; +} SMCreateStbReq; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; -} SDropStbMsg; +} SMDropStbReq; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t alterType; SSchema schema; -} SAlterStbMsg; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - uint64_t suid; -} SVDropStbReq; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int8_t type; /* operation type */ - int32_t numOfCols; /* number of schema */ - int32_t numOfTags; - char data[]; -} SAlterTableMsg; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; -} SDropTableMsg; - -typedef struct { - SMsgHead head; - int64_t uid; - int32_t tid; - int16_t tversion; - int16_t colId; - int8_t type; - int16_t bytes; - int32_t tagValLen; - int16_t numOfTags; - int32_t schemaLen; - char data[]; -} SUpdateTableTagValMsg; +} SMAlterStbReq; typedef struct { int32_t pid; @@ -470,28 +391,13 @@ typedef struct { } SCreateUserReq, SAlterUserReq; typedef struct { - int32_t contLen; - int32_t vgId; - int32_t tid; - uint64_t uid; - char tableFname[TSDB_TABLE_FNAME_LEN]; -} SMDDropTableMsg; - -typedef struct { - int32_t contLen; - int32_t vgId; - uint64_t uid; - char tableFname[TSDB_TABLE_FNAME_LEN]; -} SDropSTableMsg; - -typedef struct SColIndex { int16_t colId; // column id int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag int16_t flag; // denote if it is a tag or a normal column char name[TSDB_DB_FNAME_LEN]; } SColIndex; -typedef struct SColumnFilterInfo { +typedef struct { int16_t lowerRelOptr; int16_t upperRelOptr; int16_t filterstr; // denote if current column is char(binary/nchar) @@ -512,7 +418,7 @@ typedef struct SColumnFilterInfo { }; } SColumnFilterInfo; -typedef struct SColumnFilterList { +typedef struct { int16_t numOfFilters; union { int64_t placeholder; @@ -523,14 +429,14 @@ typedef struct SColumnFilterList { * for client side struct, we only need the column id, type, bytes are not necessary * But for data in vnode side, we need all the following information. */ -typedef struct SColumnInfo { +typedef struct { int16_t colId; int16_t type; int16_t bytes; SColumnFilterList flist; } SColumnInfo; -typedef struct STableIdInfo { +typedef struct { uint64_t uid; TSKEY key; // last accessed ts, for subscription } STableIdInfo; @@ -547,7 +453,7 @@ typedef struct { int32_t tsOrder; // ts comp block order } STsBufInfo; -typedef struct SInterval { +typedef struct { int32_t tz; // query client timezone char intervalUnit; char slidingUnit; @@ -606,7 +512,7 @@ typedef struct { int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; -} SQueryTableMsg; +} SQueryTableReq; typedef struct { int32_t code; @@ -623,7 +529,7 @@ typedef struct { int8_t free; } SRetrieveTableReq; -typedef struct SRetrieveTableRsp { +typedef struct { int64_t useconds; int8_t completed; // all results are returned to client int8_t precision; @@ -655,7 +561,7 @@ typedef struct { int8_t update; int8_t cacheLastRow; int8_t ignoreExist; -} SCreateDbMsg; +} SCreateDbReq; typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -768,7 +674,7 @@ typedef struct { typedef struct { int32_t reserved; -} STransMsg; +} STransReq; typedef struct { int32_t dnodeId; @@ -843,7 +749,7 @@ typedef struct { SMsgHead header; char dbFname[TSDB_DB_FNAME_LEN]; char tableFname[TSDB_TABLE_FNAME_LEN]; -} STableInfoMsg; +} STableInfoReq; typedef struct { int8_t metaClone; // create local clone of the cached table meta @@ -851,7 +757,7 @@ typedef struct { int32_t numOfTables; int32_t numOfUdfs; char tableNames[]; -} SMultiTableInfoMsg; +} SMultiTableInfoReq; typedef struct SVgroupInfo { int32_t vgId; @@ -859,19 +765,19 @@ typedef struct SVgroupInfo { uint32_t hashEnd; int8_t inUse; int8_t numOfEps; - SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; typedef struct { int32_t vgId; int8_t numOfEps; - SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupMsg; typedef struct { int32_t numOfVgroups; SVgroupMsg vgroups[]; -} SVgroupsMsg, SVgroupsInfo; +} SVgroupsInfo; typedef struct { char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name @@ -888,9 +794,9 @@ typedef struct { uint64_t tuid; int32_t vgId; SSchema pSchema[]; -} STableMetaMsg; +} STableMetaRsp; -typedef struct SMultiTableMeta { +typedef struct { int32_t numOfTables; int32_t numOfVgroup; int32_t numOfUdf; @@ -934,9 +840,9 @@ typedef struct { int32_t vgid[]; } SCompactReq; -typedef struct SShowRsp { +typedef struct { int64_t showId; - STableMetaMsg tableMeta; + STableMetaRsp tableMeta; } SShowRsp; typedef struct { @@ -975,17 +881,6 @@ typedef struct { int32_t dnodeId; } SMCreateBnodeReq, SMDropBnodeReq, SDCreateBnodeReq, SDDropBnodeReq; -typedef struct { - int32_t dnodeId; - int32_t vgId; - int32_t tid; -} SConfigTableMsg; - -typedef struct { - int32_t dnodeId; - int32_t vgId; -} SConfigVnodeMsg; - typedef struct { char sql[TSDB_SHOW_SQL_LEN]; int32_t queryId; @@ -1083,46 +978,6 @@ typedef struct { } SSubmitReqReader; typedef struct { - /* data */ -} SCreateTableReq; - -typedef struct { - /* data */ -} SCreateTableRsp; - -typedef struct { - /* data */ -} SDropTableReq; - -typedef struct { - /* data */ -} SDropTableRsp; - -typedef struct { - /* data */ -} SAlterTableReq; - -typedef struct { - /* data */ -} SAlterTableRsp; - -typedef struct { - /* data */ -} SDropStableReq; - -typedef struct { - /* data */ -} SDropStableRsp; - -typedef struct { - /* data */ -} SUpdateTagValReq; - -typedef struct { - /* data */ -} SUpdateTagValRsp; - -typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; @@ -1131,59 +986,59 @@ typedef struct SSubQueryMsg { char msg[]; } SSubQueryMsg; -typedef struct SResReadyMsg { +typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; -} SResReadyMsg; +} SResReadyReq; -typedef struct SResReadyRsp { +typedef struct { int32_t code; } SResReadyRsp; -typedef struct SResFetchMsg { +typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; -} SResFetchMsg; +} SResFetchReq; -typedef struct SSchTasksStatusMsg { +typedef struct { SMsgHead header; uint64_t sId; -} SSchTasksStatusMsg; +} SSchTasksStatusReq; -typedef struct STaskStatus { +typedef struct { uint64_t queryId; uint64_t taskId; int8_t status; } STaskStatus; -typedef struct SSchedulerStatusRsp { +typedef struct { uint32_t num; STaskStatus status[]; } SSchedulerStatusRsp; -typedef struct STaskCancelMsg { +typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; -} STaskCancelMsg; +} STaskCancelReq; -typedef struct STaskCancelRsp { +typedef struct { int32_t code; } STaskCancelRsp; -typedef struct STaskDropMsg { +typedef struct { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; -} STaskDropMsg; +} STaskDropReq; -typedef struct STaskDropRsp { +typedef struct { int32_t code; } STaskDropRsp; @@ -1334,18 +1189,18 @@ typedef struct { void* executor; int32_t sqlLen; char* sql; -} SCreateTopicMsg; +} SCreateTopicReq; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; -} SDropTopicMsg; +} SDropTopicReq; typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t alterType; SSchema schema; -} SAlterTopicMsg; +} SAlterTopicReq; typedef struct { SMsgHead head; @@ -1356,13 +1211,13 @@ typedef struct { char* executor; int32_t sqlLen; char* sql; -} SDCreateTopicMsg; +} SDCreateTopicReq; typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; uint64_t tuid; -} SDDropTopicMsg; +} SDDropTopicReq; typedef struct SVCreateTbReq { uint64_t ver; // use a general definition @@ -1402,24 +1257,63 @@ void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); int tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq); void* tSVCreateTbBatchReqDeserialize(void* buf, SVCreateTbBatchReq* pReq); -typedef struct SVCreateTbRsp { +typedef struct { + SMsgHead head; } SVCreateTbRsp; -typedef struct SVShowTablesReq { +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; +} SVAlterTbReq; + +typedef struct { + SMsgHead head; +} SVAlterTbRsp; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; +} SVDropTbReq; + +typedef struct { + SMsgHead head; +} SVDropTbRsp; + +typedef struct { + SMsgHead head; + int64_t uid; + int32_t tid; + int16_t tversion; + int16_t colId; + int8_t type; + int16_t bytes; + int32_t tagValLen; + int16_t numOfTags; + int32_t schemaLen; + char data[]; +} SUpdateTagValReq; + +typedef struct { + SMsgHead head; +} SUpdateTagValRsp; + +typedef struct { SMsgHead head; } SVShowTablesReq; -typedef struct SVShowTablesRsp { +typedef struct { int64_t id; - STableMetaMsg metaInfo; + STableMetaRsp metaInfo; } SVShowTablesRsp; -typedef struct SVShowTablesFetchReq { +typedef struct { SMsgHead head; int32_t id; } SVShowTablesFetchReq; -typedef struct SVShowTablesFetchRsp { +typedef struct { int64_t useconds; int8_t completed; // all results are returned to client int8_t precision; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index c5da68f0a6..da70f21498 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -114,7 +114,7 @@ typedef struct SProjectPhyNode { typedef struct SExchangePhyNode { SPhyNode node; uint64_t srcTemplateId; // template id of datasource suplans - SArray *pSrcEndPoints; // SEpAddrMsg, scheduler fill by calling qSetSuplanExecutionNode + SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; typedef struct SSubplanId { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index e28098dfbf..25e295f980 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -101,11 +101,11 @@ typedef struct SMsgSendInfo { SDataBuf msgInfo; } SMsgSendInfo; -typedef struct SQueryNodeAddr{ - int32_t nodeId; //vgId or qnodeId - int8_t inUse; - int8_t numOfEps; - SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +typedef struct SQueryNodeAddr { + int32_t nodeId; // vgId or qnodeId + int8_t inUse; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SQueryNodeAddr; bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index fb1b971049..85f3fb06a7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -130,7 +130,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { SShowRsp* pShow = (SShowRsp *)pMsg->pData; pShow->showId = htobe64(pShow->showId); - STableMetaMsg *pMetaMsg = &(pShow->tableMeta); + STableMetaRsp *pMetaMsg = &(pShow->tableMeta); pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); SSchema* pSchema = pMetaMsg->pSchema; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index fd9c1cd93e..9f724faeb9 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -53,7 +53,7 @@ class Testbase { void SendShowMetaReq(int8_t showType, const char* db); void SendShowRetrieveReq(); - STableMetaMsg* GetShowMeta(); + STableMetaRsp* GetShowMeta(); SRetrieveTableRsp* GetRetrieveRsp(); int32_t GetMetaNum(); @@ -74,7 +74,7 @@ class Testbase { private: int64_t showId; - STableMetaMsg* pMeta; + STableMetaRsp* pMeta; SRetrieveTableRsp* pRetrieveRsp; char* pData; int32_t pos; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 65c7d67254..ba530dbdd0 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -179,6 +179,6 @@ const char* Testbase::GetShowBinary(int32_t len) { int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } -STableMetaMsg* Testbase::GetShowMeta() { return pMeta; } +STableMetaRsp* Testbase::GetShowMeta() { return pMeta; } SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 89fb2e3189..d2107b9d07 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -32,7 +32,7 @@ extern "C" { typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index dbe925992d..189cbfea6d 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -31,7 +31,7 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq); static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq); static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); @@ -391,7 +391,7 @@ static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 92b9017627..5f80672369 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -26,7 +26,7 @@ static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); @@ -163,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchema; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 828eb4a5b7..54e640d8b7 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -39,7 +39,7 @@ static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg); static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); @@ -395,7 +395,7 @@ static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoReq *pInfo = pMsg->rpcMsg.pCont; mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname); @@ -417,9 +417,9 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { taosRLockLatch(&pConsumer->lock); int32_t totalCols = pConsumer->numOfColumns + pConsumer->numOfTags; - int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); - STableMetaMsg *pMeta = rpcMallocCont(contLen); + STableMetaRsp *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { taosRUnLockLatch(&pConsumer->lock); mndReleaseDb(pMnode, pDb); @@ -483,7 +483,7 @@ static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumO return 0; } -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c7360471e0..f2f8931aa1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -35,7 +35,7 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq); static int32_t mndProcessUseDbReq(SMnodeMsg *pReq); static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq); static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq); -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); @@ -378,7 +378,7 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbMsg *pCreate, SUserObj *pUser) { +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pReq, SCreateDbReq *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN); memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); @@ -449,7 +449,7 @@ CREATE_DB_OVER: static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - SCreateDbMsg *pCreate = pReq->rpcMsg.pCont; + SCreateDbReq *pCreate = pReq->rpcMsg.pCont; pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); @@ -858,7 +858,7 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { pInfo->numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEpAddrMsg *pEpArrr = &pInfo->epAddr[gid]; + SEpAddr *pEpArrr = &pInfo->epAddr[gid]; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); @@ -915,7 +915,7 @@ static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { return 0; } -static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 1b62a47c91..4bc570c11d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -53,10 +53,10 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessStatusReq(SMnodeMsg *pReq); -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); @@ -582,7 +582,7 @@ static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); } -static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchema; @@ -656,7 +656,7 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index c22c7b5885..d406247bc1 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -31,7 +31,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); @@ -395,7 +395,7 @@ static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 2c09b29872..75ed5b0a1e 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -33,7 +33,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pReq); static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveMnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); @@ -579,7 +579,7 @@ static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fcc2eec028..fced3facbe 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -51,9 +51,9 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); static int32_t mndProcessConnectReq(SMnodeMsg *pReq); static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq); static int32_t mndProcessKillConnReq(SMnodeMsg *pReq); -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); @@ -389,7 +389,7 @@ static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { } } -static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -518,7 +518,7 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 88cade08ed..6077a95a7b 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -31,7 +31,7 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq); static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq); static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); @@ -391,7 +391,7 @@ static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index dabd1f0142..7688ea76ab 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -31,7 +31,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq); static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq); static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp); static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); @@ -393,7 +393,7 @@ static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index b01c291ce5..7d77e29d74 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -32,14 +32,14 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb); -static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg); +static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg); +static int32_t mndProcesSMAlterStbReq(SMnodeMsg *pMsg); +static int32_t mndProcesSMDropStbReq(SMnodeMsg *pMsg); static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg); static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg); static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg); static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -52,9 +52,9 @@ int32_t mndInitStb(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndStbActionUpdate, .deleteFp = (SdbDeleteFp)mndStbActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessCreateStbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessAlterStbMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessDropStbMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcesSMCreateStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcesSMAlterStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcesSMDropStbReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessCreateStbInRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessAlterStbInRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessDropStbInRsp); @@ -264,10 +264,10 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb return buf; } -static SVDropStbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SVDropStbReq); +static SVDropTbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t contLen = sizeof(SVDropTbReq); - SVDropStbReq *pDrop = calloc(1, contLen); + SVDropTbReq *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -276,12 +276,12 @@ static SVDropStbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj pDrop->head.contLen = htonl(contLen); pDrop->head.vgId = htonl(pVgroup->vgId); memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pDrop->suid = htobe64(pStb->uid); + // pDrop->suid = htobe64(pStb->uid); return pDrop; } -static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) { +static int32_t mndCheckCreateStbMsg(SMCreateStbReq *pCreate) { pCreate->numOfColumns = htonl(pCreate->numOfColumns); pCreate->numOfTags = htonl(pCreate->numOfTags); int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; @@ -398,7 +398,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SVDropStbReq *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); + SVDropTbReq *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -409,7 +409,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pMsg; - action.contLen = sizeof(SVDropStbReq); + action.contLen = sizeof(SVDropTbReq); action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); @@ -423,7 +423,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) { +static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); tstrncpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -494,9 +494,9 @@ CREATE_STB_OVER: return code; } -static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { +static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SCreateStbMsg *pCreate = pMsg->rpcMsg.pCont; + SMCreateStbReq *pCreate = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to create", pCreate->name); @@ -551,7 +551,7 @@ static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { +static int32_t mndCheckAlterStbMsg(SMAlterStbReq *pAlter) { SSchema *pSchema = &pAlter->schema; pSchema->colId = htonl(pSchema->colId); pSchema->bytes = htonl(pSchema->bytes); @@ -578,9 +578,9 @@ static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; } -static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { +static int32_t mndProcesSMAlterStbReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SAlterStbMsg *pAlter = pMsg->rpcMsg.pCont; + SMAlterStbReq *pAlter = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to alter", pAlter->name); @@ -692,9 +692,9 @@ DROP_STB_OVER: return 0; } -static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { +static int32_t mndProcesSMDropStbReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SDropStbMsg *pDrop = pMsg->rpcMsg.pCont; + SMDropStbReq *pDrop = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to drop", pDrop->name); @@ -729,7 +729,7 @@ static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoReq *pInfo = pMsg->rpcMsg.pCont; mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); @@ -750,9 +750,9 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { taosRLockLatch(&pStb->lock); int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); - STableMetaMsg *pMeta = rpcMallocCont(contLen); + STableMetaRsp *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { taosRUnLockLatch(&pStb->lock); mndReleaseDb(pMnode, pDb); @@ -818,7 +818,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs return 0; } -static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 57b7c16c39..acdc718f20 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -35,7 +35,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); @@ -186,10 +186,10 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } -static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { - int32_t contLen = sizeof(SDDropTopicMsg); +static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { + int32_t contLen = sizeof(SDDropTopicReq); - SDDropTopicMsg *pDrop = calloc(1, contLen); + SDDropTopicReq *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -274,7 +274,7 @@ static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SDropTopicMsg *pDrop = pMsg->rpcMsg.pCont; + SDropTopicReq *pDrop = pMsg->rpcMsg.pCont; mDebug("topic:%s, start to drop", pDrop->name); @@ -309,7 +309,7 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg) { static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + STableInfoReq *pInfo = pMsg->rpcMsg.pCont; mDebug("topic:%s, start to retrieve meta", pInfo->tableFname); @@ -331,9 +331,9 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { taosRLockLatch(&pTopic->lock); int32_t totalCols = pTopic->numOfColumns + pTopic->numOfTags; - int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); - STableMetaMsg *pMeta = rpcMallocCont(contLen); + STableMetaRsp *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { taosRUnLockLatch(&pTopic->lock); mndReleaseDb(pMnode, pDb); @@ -397,7 +397,7 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo return 0; } -static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8c15398215..ad378953eb 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -32,7 +32,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq); static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq); static int32_t mndProcessDropUserReq(SMnodeMsg *pReq); -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); @@ -432,7 +432,7 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index c5c58a075d..4b7b370371 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -27,19 +27,19 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp); -static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { @@ -164,14 +164,14 @@ static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { return 0; } -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { - mTrace("vgId:%d, perform update action, old row:%p new row:%p", pOldVgroup->vgId, pOldVgroup, pNewVgroup); - pOldVgroup->updateTime = pNewVgroup->updateTime; - pOldVgroup->version = pNewVgroup->version; - pOldVgroup->hashBegin = pNewVgroup->hashBegin; - pOldVgroup->hashEnd = pNewVgroup->hashEnd; - pOldVgroup->replica = pNewVgroup->replica; - memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { + mTrace("vgId:%d, perform update action, old row:%p new row:%p", pOld->vgId, pOld, pNew); + pOld->updateTime = pNew->updateTime; + pOld->version = pNew->version; + pOld->hashBegin = pNew->hashBegin; + pOld->hashEnd = pNew->hashEnd; + pOld->replica = pNew->replica; + memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } @@ -427,24 +427,24 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pRsp) { return 0; } static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; @@ -475,8 +475,8 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { @@ -526,8 +526,8 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg return 0; } -static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveVgroups(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; @@ -593,8 +593,8 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -633,8 +633,8 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveVnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 7c740df06d..d70c93e758 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -61,8 +61,8 @@ void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg) { static void mndTransReExecute(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { - STransMsg *pMsg = rpcMallocCont(sizeof(STransMsg)); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pMsg, .contLen = sizeof(STransMsg)}; + STransReq *pMsg = rpcMallocCont(sizeof(STransReq)); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pMsg, .contLen = sizeof(STransReq)}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 1a089e2932..4f0ba9b0e7 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -64,9 +64,9 @@ TEST_F(MndTestDb, 01_ShowDb) { TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { - int32_t contLen = sizeof(SCreateDbMsg); + int32_t contLen = sizeof(SCreateDbReq); - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); + SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -224,9 +224,9 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { - int32_t contLen = sizeof(SCreateDbMsg); + int32_t contLen = sizeof(SCreateDbReq); - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); + SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -290,7 +290,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); EXPECT_EQ(pInfo->inUse, 0); EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + SEpAddr* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); @@ -306,7 +306,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); EXPECT_EQ(pInfo->inUse, 0); EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + SEpAddr* pAddr = &pInfo->epAddr[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index eabcbc7eb4..bfdc0f42b6 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -63,7 +63,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); - STableMetaMsg* pMeta = test.GetShowMeta(); + STableMetaRsp* pMeta = test.GetShowMeta(); EXPECT_STREQ(pMeta->tbFname, "show connections"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->numOfColumns, 7); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 55cc030122..beb52d68c1 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -27,9 +27,9 @@ Testbase MndTestStb::test; TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { - int32_t contLen = sizeof(SCreateDbMsg); + int32_t contLen = sizeof(SCreateDbReq); - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); + SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -59,9 +59,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { int32_t cols = 2; int32_t tags = 3; - int32_t contLen = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); + int32_t contLen = (tags + cols) * sizeof(SSchema) + sizeof(SMCreateStbReq); - SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(contLen); + SMCreateStbReq* pReq = (SMCreateStbReq*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); pReq->numOfTags = htonl(tags); pReq->numOfColumns = htonl(cols); @@ -123,16 +123,16 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // ----- meta ------ { - int32_t contLen = sizeof(STableInfoMsg); + int32_t contLen = sizeof(STableInfoReq); - STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(contLen); + STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen); strcpy(pReq->tableFname, "1.d1.stb"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - STableMetaMsg* pRsp = (STableMetaMsg*)pMsg->pCont; + STableMetaRsp* pRsp = (STableMetaRsp*)pMsg->pCont; pRsp->numOfTags = htonl(pRsp->numOfTags); pRsp->numOfColumns = htonl(pRsp->numOfColumns); pRsp->sversion = htonl(pRsp->sversion); @@ -214,9 +214,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { CheckInt32(3); { - int32_t contLen = sizeof(SDropStbMsg); + int32_t contLen = sizeof(SMDropStbReq); - SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(contLen); + SMDropStbReq* pReq = (SMDropStbReq*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); diff --git a/source/dnode/vnode/impl/inc/vnodeRequest.h b/source/dnode/vnode/impl/inc/vnodeRequest.h index 93b4589bad..52f4281eea 100644 --- a/source/dnode/vnode/impl/inc/vnodeRequest.h +++ b/source/dnode/vnode/impl/inc/vnodeRequest.h @@ -22,9 +22,9 @@ extern "C" { #endif -// SVDropTableReq -// int vnodeBuildDropTableReq(void **buf, const SVDropTableReq *pReq); -// void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq); +// SVDropTbReq +// int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq); +// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index a32ee50df5..29b6984937 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -53,14 +53,14 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - STableInfoMsg * pReq = (STableInfoMsg *)(pMsg->pCont); + STableInfoReq * pReq = (STableInfoReq *)(pMsg->pCont); STbCfg * pTbCfg = NULL; STbCfg * pStbCfg = NULL; tb_uid_t uid; int32_t nCols; int32_t nTagCols; SSchemaWrapper *pSW; - STableMetaMsg * pTbMetaMsg = NULL; + STableMetaRsp * pTbMetaMsg = NULL; SSchema * pTagSchema; SRpcMsg rpcMsg; int msgLen = 0; @@ -94,8 +94,8 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTagSchema = NULL; } - msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols); - pTbMetaMsg = (STableMetaMsg *)rpcMallocCont(msgLen); + msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols); + pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { goto _exit; } @@ -167,7 +167,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp) + payloadLen); - memset(pFetchRsp, 0, sizeof(struct SVShowTablesFetchRsp) + payloadLen); + memset(pFetchRsp, 0, sizeof(SVShowTablesFetchRsp) + payloadLen); char *p = pFetchRsp->data; for (int32_t i = 0; i < numOfTables; ++i) { diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/impl/src/vnodeRequest.c index 4b481bf399..5367c9e091 100644 --- a/source/dnode/vnode/impl/src/vnodeRequest.c +++ b/source/dnode/vnode/impl/src/vnodeRequest.c @@ -108,12 +108,12 @@ static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) { return buf; } -int vnodeBuildDropTableReq(void **buf, const SVDropTableReq *pReq) { +int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) { // TODO return 0; } -void *vnodeParseDropTableReq(void *buf, SVDropTableReq *pReq) { +void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) { // TODO } #endif \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h index 00ae8ab958..d2e3d488d0 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h +++ b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h @@ -54,7 +54,7 @@ typedef struct STsdbMemTable { STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb); void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); +int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRsp *pRsp); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index 4d0f436f49..b16b3581df 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -73,7 +73,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { } } -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { +int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRsp *pRsp) { SSubmitBlk * pBlock = NULL; SSubmitMsgIter msgIter = {0}; int32_t affectedrows = 0, numOfRows = 0; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 49e3ef532f..87a3000d09 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -68,7 +68,7 @@ char *ctgTestSTablename = "stable1"; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(sizeof(SCreateDbReq)); strcpy(pReq->db, "1.db1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -92,7 +92,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.contLen = sizeof(SCreateDbReq); rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; @@ -200,7 +200,7 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1; vgInfo.inUse = i % vgInfo.numOfEps; for (int32_t n = 0; n < vgInfo.numOfEps; ++n) { - SEpAddrMsg *addr = &vgInfo.epAddr[n]; + SEpAddr *addr = &vgInfo.epAddr[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } @@ -234,7 +234,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM vg->numOfEps = i % TSDB_MAX_REPLICA + 1; vg->inUse = i % vg->numOfEps; for (int32_t n = 0; n < vg->numOfEps; ++n) { - SEpAddrMsg *addr = &vg->epAddr[n]; + SEpAddr *addr = &vg->epAddr[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } @@ -249,12 +249,12 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaMsg *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; //todo pRsp->code =0; - pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaMsg *)pRsp->pCont; + rspMsg = (STableMetaRsp *)pRsp->pCont; sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestTablename); rspMsg->numOfTags = 0; rspMsg->numOfColumns = htonl(ctgTestColNum); @@ -285,12 +285,12 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaMsg *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; //todo pRsp->code =0; - pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaMsg *)pRsp->pCont; + rspMsg = (STableMetaRsp *)pRsp->pCont; sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestCTablename); sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); @@ -329,12 +329,12 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaMsg *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; //todo pRsp->code =0; - pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaMsg *)pRsp->pCont; + rspMsg = (STableMetaRsp *)pRsp->pCont; sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); @@ -372,13 +372,13 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc } void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaMsg *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; //todo static int32_t idx = 1; pRsp->code =0; - pRsp->contLen = sizeof(STableMetaMsg) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); + pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaMsg *)pRsp->pCont; + rspMsg = (STableMetaRsp *)pRsp->pCont; sprintf(rspMsg->tbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); sprintf(rspMsg->stbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); rspMsg->numOfTags = htonl(ctgTestTagNum); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a268215d3d..74ee4637c1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -593,17 +593,17 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(STaskParam *param); -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param); +int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlExpr **pExpr, SExprInfo *prevExpr, struct SUdfInfo *pUdfInfo); int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); -SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableReq *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index b98b7fef5c..968380ea01 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -86,7 +86,7 @@ int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTas } if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); + qError("qmsg:%p, SQueryTableReq wrong format", pQueryMsg); code = TSDB_CODE_QRY_INVALID_MSG; goto _over; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1d2740f0e0..b00d37f828 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2458,7 +2458,7 @@ static void doUpdateLastKey(STaskAttr* pQueryAttr) { } } -static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { +static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) { STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // in case of point-interpolation query, use asc order scan @@ -7060,7 +7060,7 @@ bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SCol return j != INT32_MIN; } -static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { +static bool validateQueryMsg(SQueryTableReq *pQueryMsg) { if (pQueryMsg->interval.interval < 0) { //qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); return false; @@ -7126,7 +7126,7 @@ static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pEx return true; } -static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { +static char *createTableIdList(SQueryTableReq *pQueryMsg, char *pMsg, SArray **pTableIdList) { assert(pQueryMsg->numOfTables > 0); *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); @@ -7268,7 +7268,7 @@ int32_t parseTaskInfo(const char* msg, int32_t len) { * @param pExpr * @return */ -int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, STaskParam* param) { +int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param) { int32_t code = TSDB_CODE_SUCCESS; // if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -7843,7 +7843,7 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // todo refactor -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo *pUdfInfo) { // *pExprInfo = NULL; // int32_t code = TSDB_CODE_SUCCESS; @@ -7901,7 +7901,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu return TSDB_CODE_SUCCESS; } -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -8069,7 +8069,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { return ((SQInfo *)qHandle)->qId == qId; } -SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, +SQInfo* createQInfoImpl(SQueryTableReq* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo) { int16_t numOfCols = pQueryMsg->numOfCols; diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 1b7fe5ebc5..83683c2bfc 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -9,9 +9,9 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); -SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); -SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); -SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1ae45556b4..5f45ce824e 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -112,7 +112,7 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, return pShowMsg; } -static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { +static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid number of keep options"; const char* msg2 = "invalid keep value"; const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; @@ -151,7 +151,7 @@ static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, return TSDB_CODE_SUCCESS; } -static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) { +static int32_t setTimePrecision(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) { const char* msg = "invalid time precision"; pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default @@ -178,7 +178,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate return TSDB_CODE_SUCCESS; } -static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { +static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); @@ -196,7 +196,7 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->numOfVgroups = htonl(pCreateDb->numOfVgroups); } -int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { +int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { doSetDbOptions(pCreateDbMsg, pCreateDbSql); if (setKeepOption(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -210,8 +210,8 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { - SCreateDbMsg* pCreateMsg = calloc(1, sizeof(SCreateDbMsg)); +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { + SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { tfree(pCreateMsg); terrno = TSDB_CODE_TSC_INVALID_OPERATION; @@ -230,7 +230,7 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt return pCreateMsg; } -SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SSchema* pSchema; int32_t numOfTags = 0; @@ -239,7 +239,7 @@ SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); } - SCreateStbMsg* pCreateStbMsg = (SCreateStbMsg*)calloc(1, sizeof(SCreateStbMsg) + (numOfCols + numOfTags) * sizeof(SSchema)); + SMCreateStbReq* pCreateStbMsg = (SMCreateStbReq*)calloc(1, sizeof(SMCreateStbReq) + (numOfCols + numOfTags) * sizeof(SSchema)); char* pMsg = NULL; #if 0 @@ -315,7 +315,7 @@ SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, return pCreateStbMsg; } -SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; @@ -325,13 +325,13 @@ SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* p return NULL; } - SDropStbMsg *pDropTableMsg = (SDropStbMsg*) calloc(1, sizeof(SDropStbMsg)); + SMDropStbReq *pDropTableMsg = (SMDropStbReq*) calloc(1, sizeof(SMDropStbReq)); code = tNameExtractFullName(&name, pDropTableMsg->name); assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T); pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - *len = sizeof(SDropStbMsg); + *len = sizeof(SMDropStbReq); return pDropTableMsg; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 947941b2cf..6676e1ebf6 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -117,7 +117,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou } // can only perform the parameters based on the macro definitation -static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { +static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { char msg[512] = {0}; if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { @@ -870,14 +870,14 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c goto _error; } - SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); + SCreateDbReq* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_INVALID_OPERATION; goto _error; } pDcl->pMsg = (char*)pCreateMsg; - pDcl->msgLen = sizeof(SCreateDbMsg); + pDcl->msgLen = sizeof(SCreateDbReq); pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; break; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d67ff956b9..0e9b2dd76f 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -557,7 +557,7 @@ static const char* jkEpAddrFqdn = "Fqdn"; static const char* jkEpAddrPort = "Port"; static bool epAddrToJson(const void* obj, cJSON* json) { - const SEpAddrMsg* ep = (const SEpAddrMsg*)obj; + const SEpAddr* ep = (const SEpAddr*)obj; bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); if (res) { res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); @@ -566,7 +566,7 @@ static bool epAddrToJson(const void* obj, cJSON* json) { } static bool epAddrFromJson(const cJSON* json, void* obj) { - SEpAddrMsg* ep = (SEpAddrMsg*)obj; + SEpAddr* ep = (SEpAddr*)obj; copyString(json, jkEpAddrFqdn, ep->fqdn); ep->port = getNumber(json, jkEpAddrPort); return true; @@ -583,7 +583,7 @@ static bool nodeAddrToJson(const void* obj, cJSON* json) { res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, ep->inUse); } if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, ep->numOfEps, sizeof(SEpAddrMsg)); + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, ep->epAddr, ep->numOfEps, sizeof(SEpAddr)); } return res; } @@ -593,7 +593,7 @@ static bool nodeAddrFromJson(const cJSON* json, void* obj) { ep->nodeId = getNumber(json, jkNodeAddrId); ep->inUse = getNumber(json, jkNodeAddrInUse); int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, nodeAddrFromJson, &ep->epAddr, sizeof(SEpAddrMsg), &numOfEps); + bool res = fromRawArray(json, jkNodeAddrEpAddrs, nodeAddrFromJson, &ep->epAddr, sizeof(SEpAddr), &numOfEps); ep->numOfEps = numOfEps; return res; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 16cabd6483..2cde28baf9 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -29,7 +29,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; - int32_t estimateSize = sizeof(STableInfoMsg); + int32_t estimateSize = sizeof(STableInfoReq); if (NULL == *msg || msgSize < estimateSize) { tfree(*msg); *msg = rpcMallocCont(estimateSize); @@ -38,7 +38,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 } } - STableInfoMsg *bMsg = (STableInfoMsg *)*msg; + STableInfoReq *bMsg = (STableInfoReq *)*msg; bMsg->header.vgId = htonl(bInput->vgId); @@ -146,7 +146,7 @@ _return: return code; } -static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { +static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) { pMetaMsg->numOfTags = ntohl(pMetaMsg->numOfTags); pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns); pMetaMsg->sversion = ntohl(pMetaMsg->sversion); @@ -198,7 +198,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaMsg* pMetaMsg) { return TSDB_CODE_SUCCESS; } -int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STableMeta **pMeta) { +int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta) { int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; @@ -232,7 +232,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaMsg* msg, bool isSuperTable, STabl int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)msg; + STableMetaRsp *pMetaMsg = (STableMetaRsp *)msg; int32_t code = queryConvertTableMetaMsg(pMetaMsg); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 808c1e19f9..15e894fd61 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1080,7 +1080,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ return TSDB_CODE_QRY_INVALID_INPUT; } - SResReadyMsg *msg = pMsg->pCont; + SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task status msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1101,7 +1101,7 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } int32_t code = 0; - SSchTasksStatusMsg *msg = pMsg->pCont; + SSchTasksStatusReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task status msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1125,7 +1125,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SResFetchMsg *msg = pMsg->pCont; + SResFetchReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1157,7 +1157,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } int32_t code = 0; - STaskCancelMsg *msg = pMsg->pCont; + STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task cancel msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -1182,7 +1182,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } int32_t code = 0; - STaskDropMsg *msg = pMsg->pCont; + STaskDropReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task drop msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 7bc1c4ff40..eaa79fd39a 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -113,12 +113,12 @@ void *readyThread(void *param) { uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - SResReadyMsg readyMsg = {0}; + SResReadyReq readyMsg = {0}; readyMsg.sId = htobe64(1); readyMsg.queryId = htobe64(1); readyMsg.taskId = htobe64(1); readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyMsg); + readyRpc.contLen = sizeof(SResReadyReq); while (!testStop) { code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); @@ -137,12 +137,12 @@ void *fetchThread(void *param) { uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - SResFetchMsg fetchMsg = {0}; + SResFetchReq fetchMsg = {0}; fetchMsg.sId = htobe64(1); fetchMsg.queryId = htobe64(1); fetchMsg.taskId = htobe64(1); fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchMsg); + fetchRpc.contLen = sizeof(SResFetchReq); while (!testStop) { code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); @@ -161,12 +161,12 @@ void *dropThread(void *param) { uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - STaskDropMsg dropMsg = {0}; + STaskDropReq dropMsg = {0}; dropMsg.sId = htobe64(1); dropMsg.queryId = htobe64(1); dropMsg.taskId = htobe64(1); dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropMsg); + dropRpc.contLen = sizeof(STaskDropReq); while (!testStop) { code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); @@ -185,10 +185,10 @@ void *statusThread(void *param) { uint32_t n = 0; void *mockPointer = (void *)0x1; void *mgmt = param; - SSchTasksStatusMsg statusMsg = {0}; + SSchTasksStatusReq statusMsg = {0}; statusMsg.sId = htobe64(1); statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.contLen = sizeof(SSchTasksStatusReq); statusRpc.msgType = TDMT_VND_TASKS_STATUS; while (!testStop) { @@ -228,31 +228,31 @@ TEST(seqTest, normalCase) { queryRpc.pCont = queryMsg; queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyMsg readyMsg = {0}; + SResReadyReq readyMsg = {0}; readyMsg.sId = htobe64(1); readyMsg.queryId = htobe64(1); readyMsg.taskId = htobe64(1); readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyMsg); + readyRpc.contLen = sizeof(SResReadyReq); - SResFetchMsg fetchMsg = {0}; + SResFetchReq fetchMsg = {0}; fetchMsg.sId = htobe64(1); fetchMsg.queryId = htobe64(1); fetchMsg.taskId = htobe64(1); fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchMsg); + fetchRpc.contLen = sizeof(SResFetchReq); - STaskDropMsg dropMsg = {0}; + STaskDropReq dropMsg = {0}; dropMsg.sId = htobe64(1); dropMsg.queryId = htobe64(1); dropMsg.taskId = htobe64(1); dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropMsg); + dropRpc.contLen = sizeof(STaskDropReq); - SSchTasksStatusMsg statusMsg = {0}; + SSchTasksStatusReq statusMsg = {0}; statusMsg.sId = htobe64(1); statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.contLen = sizeof(SSchTasksStatusReq); statusRpc.msgType = TDMT_VND_TASKS_STATUS; stubSetStringToPlan(); @@ -312,17 +312,17 @@ TEST(seqTest, cancelFirst) { queryRpc.pCont = queryMsg; queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - STaskDropMsg dropMsg = {0}; + STaskDropReq dropMsg = {0}; dropMsg.sId = htobe64(1); dropMsg.queryId = htobe64(1); dropMsg.taskId = htobe64(1); dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropMsg); + dropRpc.contLen = sizeof(STaskDropReq); - SSchTasksStatusMsg statusMsg = {0}; + SSchTasksStatusReq statusMsg = {0}; statusMsg.sId = htobe64(1); statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.contLen = sizeof(SSchTasksStatusReq); statusRpc.msgType = TDMT_VND_TASKS_STATUS; stubSetStringToPlan(); @@ -370,31 +370,31 @@ TEST(seqTest, randCase) { queryRpc.pCont = queryMsg; queryRpc.contLen = sizeof(SSubQueryMsg) + 100; - SResReadyMsg readyMsg = {0}; + SResReadyReq readyMsg = {0}; readyMsg.sId = htobe64(1); readyMsg.queryId = htobe64(1); readyMsg.taskId = htobe64(1); readyRpc.pCont = &readyMsg; - readyRpc.contLen = sizeof(SResReadyMsg); + readyRpc.contLen = sizeof(SResReadyReq); - SResFetchMsg fetchMsg = {0}; + SResFetchReq fetchMsg = {0}; fetchMsg.sId = htobe64(1); fetchMsg.queryId = htobe64(1); fetchMsg.taskId = htobe64(1); fetchRpc.pCont = &fetchMsg; - fetchRpc.contLen = sizeof(SResFetchMsg); + fetchRpc.contLen = sizeof(SResFetchReq); - STaskDropMsg dropMsg = {0}; + STaskDropReq dropMsg = {0}; dropMsg.sId = htobe64(1); dropMsg.queryId = htobe64(1); dropMsg.taskId = htobe64(1); dropRpc.pCont = &dropMsg; - dropRpc.contLen = sizeof(STaskDropMsg); + dropRpc.contLen = sizeof(STaskDropReq); - SSchTasksStatusMsg statusMsg = {0}; + SSchTasksStatusReq statusMsg = {0}; statusMsg.sId = htobe64(1); statusRpc.pCont = &statusMsg; - statusRpc.contLen = sizeof(SSchTasksStatusMsg); + statusRpc.contLen = sizeof(SSchTasksStatusReq); statusRpc.msgType = TDMT_VND_TASKS_STATUS; stubSetStringToPlan(); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ac08b5c42..a72e97e13f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -595,7 +595,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { - SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; + SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; job->resNumOfRows += rsp->affectedRows; code = schProcessOnTaskSuccess(job, task); @@ -832,14 +832,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { break; } case TDMT_VND_RES_READY: { - msgSize = sizeof(SResReadyMsg); + msgSize = sizeof(SResReadyReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResReadyMsg *pMsg = msg; + SResReadyReq *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); @@ -849,14 +849,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { break; } case TDMT_VND_FETCH: { - msgSize = sizeof(SResFetchMsg); + msgSize = sizeof(SResFetchReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResFetchMsg *pMsg = msg; + SResFetchReq *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); @@ -866,14 +866,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { break; } case TDMT_VND_DROP_TASK:{ - msgSize = sizeof(STaskDropMsg); + msgSize = sizeof(STaskDropReq); msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - STaskDropMsg *pMsg = msg; + STaskDropReq *pMsg = msg; pMsg->header.vgId = htonl(addr->nodeId); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 790778b736..ac37b9585c 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -186,7 +186,7 @@ void *schtSendRsp(void *param) { while (pIter) { SSchTask *task = *(SSchTask **)pIter; - SShellSubmitRspMsg rsp = {0}; + SShellSubmitRsp rsp = {0}; rsp.affectedRows = 10; schProcessRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0); From f9d83c044ddfa316aae92d1f624840e74d29e23f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 10 Jan 2022 23:42:58 +0800 Subject: [PATCH 07/34] profile performance --- source/dnode/vnode/meta/src/metaIdx.c | 10 +-- source/libs/index/src/index.c | 76 ++++++++++--------- source/libs/index/src/index_cache.c | 22 +++++- source/libs/index/src/index_fst.c | 1 + .../index/src/index_fst_counting_writer.c | 14 +++- source/libs/index/src/index_tfile.c | 16 +++- 6 files changed, 85 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/meta/src/metaIdx.c index 828bd12088..3da56fc394 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/meta/src/metaIdx.c @@ -49,9 +49,7 @@ int metaOpenIdx(SMeta *pMeta) { #ifdef USE_INVERTED_INDEX SIndexOpts opts; - if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { - return -1; - } + if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { return -1; } #endif return 0; @@ -67,16 +65,14 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */ #ifdef USE_INVERTED_INDEX SIndexOpts opts; - if (indexClose(pMeta->pIdx->pIdx) != 0) { - return -1; - } + if (indexClose(pMeta->pIdx->pIdx) != 0) { return -1; } #endif } int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { #ifdef USE_INVERTED_INDEX - if (pTbCfgs - type == META_CHILD_TABLE) { + if (pTbCfgs->type == META_CHILD_TABLE) { char buf[8] = {0}; int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; sprintf(buf, "%d", colId); // colname diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 19e9375491..0c222eae1a 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -59,6 +59,10 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); +// merge cache and tfile by opera type +static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, IterateValue* iTfv); +static void indexMergeSameKey(SArray* result, TFileValue* tv); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); @@ -385,6 +389,27 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { taosArrayPush(result, &tv); } } +static void indexMergeCacheAndTFile(SArray* result, IterateValue* cv, IterateValue* tv) { + // opt + char* colVal = (cv != NULL) ? cv->colVal : tv->colVal; + // design merge-algorithm later, too complicated to handle all kind of situation + TFileValue* tfv = tfileValueCreate(colVal); + if (cv != NULL) { + if (cv->type == ADD_VALUE) { + taosArrayAddAll(tfv->tableId, cv->val); + } else if (cv->type == DEL_VALUE) { + } else if (cv->type == UPDATE_VALUE) { + } else { + // do nothing + } + } + if (tv != NULL) { + // opt later + taosArrayAddAll(tfv->tableId, tv->val); + } + + indexMergeSameKey(result, tfv); +} static void indexDestroyTempResult(SArray* result) { int32_t sz = result ? taosArrayGetSize(result) : 0; for (size_t i = 0; i < sz; i++) { @@ -411,51 +436,30 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { bool cn = cacheIter ? cacheIter->next(cacheIter) : false; bool tn = tfileIter ? tfileIter->next(tfileIter) : false; - while (cn == true && tn == true) { - IterateValue* cv = cacheIter->getValue(cacheIter); - IterateValue* tv = tfileIter->getValue(tfileIter); + while (cn == true || tn == true) { + IterateValue* cv = (cn == true) ? cacheIter->getValue(cacheIter) : NULL; + IterateValue* tv = (tn == true) ? tfileIter->getValue(tfileIter) : NULL; - // dump value - int comp = strcmp(cv->colVal, tv->colVal); + int comp = 0; + if (cn == true && tn == true) { + comp = strcmp(cv->colVal, tv->colVal); + } else if (cn == true) { + comp = -1; + } else { + comp = 1; + } if (comp == 0) { - TFileValue* tfv = tfileValueCreate(cv->colVal); - taosArrayAddAll(tfv->tableId, cv->val); - taosArrayAddAll(tfv->tableId, tv->val); - indexMergeSameKey(result, tfv); - + indexMergeCacheAndTFile(result, cv, tv); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); - continue; } else if (comp < 0) { - TFileValue* tfv = tfileValueCreate(cv->colVal); - taosArrayAddAll(tfv->tableId, cv->val); - - indexMergeSameKey(result, tfv); - // copy to final Result; + indexMergeCacheAndTFile(result, cv, NULL); cn = cacheIter->next(cacheIter); } else { - TFileValue* tfv = tfileValueCreate(tv->colVal); - taosArrayAddAll(tfv->tableId, tv->val); - - indexMergeSameKey(result, tfv); - // copy to final result + indexMergeCacheAndTFile(result, NULL, tv); tn = tfileIter->next(tfileIter); } } - while (cn == true) { - IterateValue* cv = cacheIter->getValue(cacheIter); - TFileValue* tfv = tfileValueCreate(cv->colVal); - taosArrayAddAll(tfv->tableId, cv->val); - indexMergeSameKey(result, tfv); - cn = cacheIter->next(cacheIter); - } - while (tn == true) { - IterateValue* tv = tfileIter->getValue(tfileIter); - TFileValue* tfv = tfileValueCreate(tv->colVal); - taosArrayAddAll(tfv->tableId, tv->val); - indexMergeSameKey(result, tfv); - tn = tfileIter->next(tfileIter); - } int ret = indexGenTFile(sIdx, pCache, result); indexDestroyTempResult(result); @@ -503,7 +507,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { tfileWriterClose(tw); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); - if (reader == NULL) { goto END; } + if (reader == NULL) { return -1; } TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 294c8192e8..48566a8674 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -217,9 +217,9 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { // set value ct->uid = uid; ct->operaType = term->operType; - // ugly code, refactor later int64_t estimate = sizeof(ct) + strlen(ct->colVal); + pthread_mutex_lock(&pCache->mtx); pCache->occupiedMem += estimate; indexCacheMakeRoomForWrite(pCache); @@ -331,7 +331,6 @@ static char* indexCacheTermGet(const void* pData) { static int32_t indexCacheTermCompare(const void* l, const void* r) { CacheTerm* lt = (CacheTerm*)l; CacheTerm* rt = (CacheTerm*)r; - // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); if (cmp == 0) { return rt->version - lt->version; } @@ -359,17 +358,32 @@ static bool indexCacheIteratorNext(Iterate* itera) { IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); + // IterateValue* iv = &itera->val; + // IterateValue tIterVal = {.colVal = NULL, .val = taosArrayInit(1, sizeof(uint64_t))}; + bool next = tSkipListIterNext(iter); if (next) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + // equal func + // if (iv->colVal != NULL && ct->colVal != NULL) { + // if (0 == strcmp(iv->colVal, ct->colVal)) { if (iv->type == ADD_VALUE) } + //} else { + // tIterVal.colVal = calloc(1, strlen(ct->colVal) + 1); + // tIterval.colVal = tstrdup(ct->colVal); + //} iv->type = ct->operaType; - iv->colVal = calloc(1, strlen(ct->colVal) + 1); - memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); + iv->colVal = tstrdup(ct->colVal); + // iv->colVal = calloc(1, strlen(ct->colVal) + 1); + // memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } + // IterateValue* iv = &itera->val; + // iterateValueDestroy(iv, true); + //*iv = tIterVal; + return next; } diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 4f782cef26..5299a7dc5f 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -936,6 +936,7 @@ Fst* fstCreate(FstSlice* slice) { len -= sizeof(checkSum); taosDecodeFixedU32(buf + len, &checkSum); if (taosCheckChecksum(buf, len, checkSum)) { + indexError("index file is corrupted"); // verify fst return NULL; } diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 6db5555aa6..b57f639726 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -60,9 +60,10 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off return nRead; } static int writeCtxGetSize(WriterCtx* ctx) { - if (ctx->type == TFile && ctx->file.readOnly) { - // refactor later - return ctx->file.size; + if (ctx->type == TFile) { + struct stat fstat; + stat(ctx->file.buf, &fstat); + return fstat.st_size; } return 0; } @@ -88,7 +89,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int if (readOnly == false) { // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenCreateWriteAppend(path); - + tfFtruncate(ctx->file.fd, 0); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; @@ -138,6 +139,11 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { munmap(ctx->file.ptr, ctx->file.size); #endif } + if (ctx->file.readOnly == false) { + struct stat fstat; + stat(ctx->file.buf, &fstat); + // indexError("write file size: %d", (int)(fstat.st_size)); + } if (remove) { unlink(ctx->file.buf); } } free(ctx); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 4b76402560..98fede4f7b 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -147,21 +147,22 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { reader->ctx = ctx; if (0 != tfileReaderVerify(reader)) { - tfileReaderDestroy(reader); indexError("invalid tfile, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + tfileReaderDestroy(reader); return NULL; } // T_REF_INC(reader); if (0 != tfileReaderLoadHeader(reader)) { - tfileReaderDestroy(reader); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + tfileReaderDestroy(reader); return NULL; } if (0 != tfileReaderLoadFst(reader)) { + indexError("failed to load index fst, suid: %" PRIu64 ", colName: %s, errno: %d", reader->header.suid, + reader->header.colName, errno); tfileReaderDestroy(reader); - indexError("failed to load index fst, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); return NULL; } @@ -303,6 +304,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } else { // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, // (int)taosArrayGetSize(v->tableId)); + + // indexInfo("tfile write data size: %d", tw->ctx->size(tw->ctx)); } } fstBuilderFinish(tw->fb); @@ -485,7 +488,9 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx)); tw->offset += sizeof(fstOffset); return 0; } @@ -495,8 +500,11 @@ static int tfileWriteHeader(TFileWriter* writer) { TFileHeader* header = &writer->header; memcpy(buf, (char*)header, sizeof(buf)); + indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); if (sizeof(buf) != nwrite) { return -1; } + + indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx)); writer->offset = nwrite; return 0; } @@ -521,6 +529,8 @@ static int tfileWriteFooter(TFileWriter* write) { void* pBuf = (void*)buf; taosEncodeFixedU64((void**)(void*)&pBuf, tfileMagicNumber); int nwrite = write->ctx->write(write->ctx, buf, strlen(buf)); + + indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx)); assert(nwrite == sizeof(tfileMagicNumber)); return nwrite; } From 8589d1f3a4bb972b8748c1307b169088d02e2261 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 08:32:33 +0800 Subject: [PATCH 08/34] feature/qnode --- source/libs/scheduler/inc/schedulerInt.h | 6 +++--- source/libs/scheduler/src/scheduler.c | 18 +----------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 8923dc91cd..d127adbcfa 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -130,9 +130,9 @@ typedef struct SSchJob { #define SCH_JOB_ELOG(param, ...) qError("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) #define SCH_JOB_DLOG(param, ...) qDebug("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRIx64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c13aacb2ad..99b4b858d0 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1453,20 +1453,4 @@ void scheduleFreeJob(void *job) { taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); - - tfree(pJob->res); - - tfree(pJob); -} - -void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); //TODO - schMgmt.jobs = NULL; - } -} - - + taosHashCleanup(pJob->suc \ No newline at end of file From 7850f2a9bfa53b2cba0b67127b018aa8983cee61 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 08:40:12 +0800 Subject: [PATCH 09/34] feature/qnode --- source/libs/scheduler/src/scheduler.c | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0e1f294088..d478e80af0 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -697,12 +697,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); - code = qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr); - if (code) { - SCH_UNLOCK(SCH_WRITE, &par->lock); - SCH_TASK_ELOG("qSetSubplanExecutionNode failed, code:%x, templateId:%"PRIx64, code, pTask->plan->id.templateId); - SCH_ERR_JRET(code); - } + qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &pTask->succeedAddr); SCH_UNLOCK(SCH_WRITE, &par->lock); if (SCH_TASK_READY_TO_LUNCH(par)) { @@ -1450,7 +1445,22 @@ void scheduleFreeJob(void *job) { taosArrayDestroy(pLevel->subTasks); } + + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); + + taosArrayDestroy(pJob->levels); + + tfree(pJob->res); + + tfree(pJob); + } + + void schedulerDestroy(void) { + if (schMgmt.jobs) { + taosHashCleanup(schMgmt.jobs); //TODO + schMgmt.jobs = NULL; + } + } - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->suc \ No newline at end of file From 89dcffce889e362831701d46f225f77ac84cee74 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 10:14:06 +0800 Subject: [PATCH 10/34] feature/qnode --- include/libs/scheduler/scheduler.h | 2 +- source/libs/catalog/test/catalogTests.cpp | 1 + source/libs/scheduler/src/scheduler.c | 154 ++++++++++-------- source/libs/scheduler/test/schedulerTests.cpp | 93 +++++++---- source/os/src/osSysinfo.c | 2 +- 5 files changed, 156 insertions(+), 96 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 74b7813465..58ec3de32f 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -89,7 +89,7 @@ int32_t scheduleFetchRows(void *pJob, void **data); * @param pJob * @return */ -int32_t scheduleCancelJob(void *pJob); +//int32_t scheduleCancelJob(void *pJob); /** * Free the query job diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 87a3000d09..436593f9d6 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -1322,3 +1322,4 @@ int main(int argc, char** argv) { + \ No newline at end of file diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d478e80af0..4a0d62a06f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -87,68 +87,72 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; - int8_t oriStatus = SCH_GET_JOB_STATUS(pJob); + int8_t oriStatus = 0; -/* - if (oriStatus == newStatus) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - switch (oriStatus) { - case JOB_TASK_STATUS_NULL: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_NOT_START) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_SUCCEED: - case JOB_TASK_STATUS_FAILED: - case JOB_TASK_STATUS_CANCELLING: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_CANCELLED: - case JOB_TASK_STATUS_DROPPING: + while (true) { + oriStatus = SCH_GET_JOB_STATUS(pJob); + + if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - break; - - default: - qError("invalid task status:%d", oriStatus); - return TSDB_CODE_QRY_APP_ERROR; + } + + switch (oriStatus) { + case JOB_TASK_STATUS_NULL: + if (newStatus != JOB_TASK_STATUS_NOT_START) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_NOT_START: + if (newStatus != JOB_TASK_STATUS_EXECUTING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_EXECUTING: + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_CANCELLING + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_PARTIAL_SUCCEED: + if (newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_SUCCEED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_SUCCEED: + case JOB_TASK_STATUS_FAILED: + case JOB_TASK_STATUS_CANCELLING: + if (newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_CANCELLED: + case JOB_TASK_STATUS_DROPPING: + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + break; + + default: + SCH_JOB_ELOG("invalid job status:%d", oriStatus); + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { + continue; + } + + SCH_JOB_DLOG("job status updated from %d to %d", oriStatus, newStatus); + + break; } -*/ - - SCH_SET_JOB_STATUS(pJob, newStatus); - - SCH_JOB_DLOG("status updated from %d to %d", oriStatus, newStatus); return TSDB_CODE_SUCCESS; @@ -507,6 +511,7 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } +// Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -515,7 +520,13 @@ int32_t schFetchFromRemote(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - if (atomic_load_ptr(&pJob->res)) + void *res = atomic_load_ptr(&pJob->res); + if (res) { + atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + + SCH_JOB_DLOG("res already fetched, res:%p", res); + return TSDB_CODE_SUCCESS; + } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); @@ -525,12 +536,15 @@ _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + schProcessOnJobFailure(pJob, code); + return code; } // Note: no more error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + // if already FAILED, no more processing SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); if (errCode) { @@ -813,7 +827,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { - qError("QID:%"PRIx64" taosHashGet queryId not exist", pParam->queryId); + qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -1147,7 +1161,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); if (size <= 0) { - SCH_TASK_DLOG("empty exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); return; } @@ -1157,6 +1171,8 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); } + + SCH_TASK_DLOG("task has %d exec address", size); } void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { @@ -1331,6 +1347,12 @@ int32_t scheduleFetchRows(void *job, void **data) { SSchJob *pJob = job; int32_t code = 0; + int8_t status = SCH_GET_JOB_STATUS(pJob); + if (status == JOB_TASK_STATUS_DROPPING) { + SCH_JOB_ELOG("job is dropping, status:%d", status); + return TSDB_CODE_SCH_STATUS_ERROR; + } + atomic_add_fetch_32(&pJob->ref, 1); if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { @@ -1345,8 +1367,6 @@ int32_t scheduleFetchRows(void *job, void **data) { SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - int8_t status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_FAILED) { *data = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); @@ -1414,6 +1434,10 @@ void scheduleFreeJob(void *job) { return; } + schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + + SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); + while (true) { int32_t ref = atomic_load_32(&pJob->ref); if (0 == ref) { @@ -1425,6 +1449,8 @@ void scheduleFreeJob(void *job) { } } + SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { schCancelJob(pJob); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 70e49e6b45..f75f0ef263 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,52 +38,73 @@ namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +void schtInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + + void schtBuildQueryDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000001; dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); - SArray *scan = taosArrayInit(1, sizeof(SSubplan)); - SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + SArray *scan = taosArrayInit(1, POINTER_BYTES); + SArray *merge = taosArrayInit(1, POINTER_BYTES); - SSubplan scanPlan = {0}; - SSubplan mergePlan = {0}; + SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); - scanPlan.id.queryId = qId; - scanPlan.id.templateId = 0x0000000000000002; - scanPlan.id.subplanId = 0x0000000000000003; - scanPlan.type = QUERY_TYPE_SCAN; - scanPlan.execNode.numOfEps = 1; - scanPlan.execNode.nodeId = 1; - scanPlan.execNode.inUse = 0; - scanPlan.execNode.epAddr[0].port = 6030; - strcpy(scanPlan.execNode.epAddr[0].fqdn, "ep0"); - scanPlan.pChildren = NULL; - scanPlan.level = 1; - scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); - scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + scanPlan->id.queryId = qId; + scanPlan->id.templateId = 0x0000000000000002; + scanPlan->id.subplanId = 0x0000000000000003; + scanPlan->type = QUERY_TYPE_SCAN; + scanPlan->execNode.numOfEps = 1; + scanPlan->execNode.nodeId = 1; + scanPlan->execNode.inUse = 0; + scanPlan->execNode.epAddr[0].port = 6030; + strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0"); + scanPlan->pChildren = NULL; + scanPlan->level = 1; + scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); + scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); - mergePlan.id.queryId = qId; - mergePlan.id.templateId = 0x4444444444; - mergePlan.id.subplanId = 0x5555555555; - mergePlan.type = QUERY_TYPE_MERGE; - mergePlan.level = 0; - mergePlan.execNode.numOfEps = 0; - mergePlan.pChildren = taosArrayInit(1, POINTER_BYTES); - mergePlan.pParents = NULL; - mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + mergePlan->id.queryId = qId; + mergePlan->id.templateId = 0x4444444444; + mergePlan->id.subplanId = 0x5555555555; + mergePlan->type = QUERY_TYPE_MERGE; + mergePlan->level = 0; + mergePlan->execNode.numOfEps = 0; + mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); + mergePlan->pParents = NULL; + mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); - taosArrayPush(mergePointer->pChildren, &scanPointer); - taosArrayPush(scanPointer->pParents, &mergePointer); + taosArrayPush(mergePlan->pChildren, &scanPlan); + taosArrayPush(scanPlan->pParents, &mergePlan); taosArrayPush(dag->pSubplans, &merge); taosArrayPush(dag->pSubplans, &scan); } +void schtFreeQueryDag(SQueryDag *dag) { + +} + + void schtBuildInsertDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000002; @@ -138,8 +159,8 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { return 0; } -int32_t schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - return 0; +void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { + } @@ -209,6 +230,9 @@ TEST(queryTest, normalCase) { SVgroupInfo vgInfo = {0}; void *pJob = NULL; SQueryDag dag = {0}; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -295,6 +319,8 @@ TEST(queryTest, normalCase) { ASSERT_EQ(data, (void*)NULL); scheduleFreeJob(pJob); + + schtFreeQueryDag(&dag); } @@ -308,6 +334,9 @@ TEST(insertTest, normalCase) { SVgroupInfo vgInfo = {0}; SQueryDag dag = {0}; uint64_t numOfRows = 0; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -336,7 +365,11 @@ TEST(insertTest, normalCase) { scheduleFreeJob(pInsertJob); } +TEST(multiThread, forceFree) { + schtInitLogFile(); + +} int main(int argc, char** argv) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e235b0714e..0be17ca2b9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1087,7 +1087,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return 0; } - return -1; + return 0; } char *taosGetCmdlineByPID(int pid) { From f4bb5d06c82f807a8074b8b1af3003ec524338a7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 17:33:25 -0800 Subject: [PATCH 11/34] minor changes --- source/dnode/mnode/impl/src/mndDb.c | 5 +- source/dnode/mnode/impl/src/mndTrans.c | 4 +- source/dnode/mnode/impl/src/mndVgroup.c | 14 ++--- source/dnode/mnode/impl/test/db/db.cpp | 19 ++---- tests/script/jenkins/basic.txt | 9 +-- tests/script/{general => sim}/db/basic1.sim | 0 tests/script/sim/db/error1.sim | 61 +++++++++++++++++++ tests/script/{unique => sim}/dnode/basic1.sim | 0 .../script/{general => sim}/table/basic1.sim | 0 tests/script/{general => sim}/user/basic1.sim | 0 10 files changed, 82 insertions(+), 30 deletions(-) rename tests/script/{general => sim}/db/basic1.sim (100%) create mode 100644 tests/script/sim/db/error1.sim rename tests/script/{unique => sim}/dnode/basic1.sim (100%) rename tests/script/{general => sim}/table/basic1.sim (100%) rename tests/script/{general => sim}/user/basic1.sim (100%) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2f8931aa1..85b7fbbb42 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -463,7 +463,7 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { pCreate->commitTime = htonl(pCreate->commitTime); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); - mDebug("db:%s, start to create", pCreate->db); + mDebug("db:%s, start to create, vgroups:%d", pCreate->db, pCreate->numOfVgroups); SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); if (pDb != NULL) { @@ -476,6 +476,9 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } + } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; } SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 4a42133ce3..058178260f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -809,7 +809,7 @@ static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, stage from undoLog to rollback", pTrans->id); continueExec = true; } else { - mDebug("trans:%d, stage keep on undoLog since %s", pTrans->id, terrstr()); + mError("trans:%d, stage keep on undoLog since %s", pTrans->id, terrstr()); continueExec = false; } @@ -825,7 +825,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, stage from undoAction to undoLog", pTrans->id); continueExec = true; } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mDebug("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); + mError("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { pTrans->failedTimes++; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 4b7b370371..e0d6d3dd42 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -273,15 +273,10 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 SDnodeObj *pDnode = pObj; SArray *pArray = p1; - pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - int64_t curMs = taosGetTimestampMs(); bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); - if (online && pDnode->numOfSupportVnodes > 0) { - taosArrayPush(pArray, pDnode); - } - - bool isMnode = mndIsMnode(pMnode, pDnode->id); + bool isMnode = mndIsMnode(pMnode, pDnode->id); + pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d", pDnode->id, pDnode->numOfVnodes, pDnode->numOfSupportVnodes, isMnode, online); @@ -290,6 +285,9 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 pDnode->numOfVnodes++; } + if (online && pDnode->numOfSupportVnodes > 0) { + taosArrayPush(pArray, pDnode); + } return true; } @@ -311,7 +309,7 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; - return d1Score > d2Score ? 1 : 0; + return d1Score >= d2Score ? 1 : 0; } static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 4f0ba9b0e7..5d5947b644 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -13,28 +13,17 @@ class MndTestDb : public ::testing::Test { protected: - static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_db", 9030); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9030"; + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); } + static void TearDownTestSuite() { test.Cleanup(); } - server2.Start("/tmp/mnode_test_db2", fqdn, 9031, firstEp); - } - static void TearDownTestSuite() { - server2.Stop(); - test.Cleanup(); - } - - static Testbase test; - static TestServer server2; + static Testbase test; public: void SetUp() override {} void TearDown() override {} }; -Testbase MndTestDb::test; -TestServer MndTestDb::server2; +Testbase MndTestDb::test; TEST_F(MndTestDb, 01_ShowDb) { test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f960abb4e0..bc0c3a4f64 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,15 +2,16 @@ #======================b1-start=============== # ---- user -./test.sh -f general/user/basic1.sim +./test.sh -f sim/user/basic1.sim # ---- db -./test.sh -f general/db/basic1.sim +./test.sh -f sim/db/basic1.sim +./test.sh -f sim/db/error1.sim # ---- table -./test.sh -f general/table/basic1.sim +./test.sh -f sim/table/basic1.sim # ---- dnode -./test.sh -f unique/dnode/basic1.sim +./test.sh -f sim/dnode/basic1.sim #======================b1-end=============== diff --git a/tests/script/general/db/basic1.sim b/tests/script/sim/db/basic1.sim similarity index 100% rename from tests/script/general/db/basic1.sim rename to tests/script/sim/db/basic1.sim diff --git a/tests/script/sim/db/error1.sim b/tests/script/sim/db/error1.sim new file mode 100644 index 0000000000..6698ca6f67 --- /dev/null +++ b/tests/script/sim/db/error1.sim @@ -0,0 +1,61 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode2 -s start +sql connect + +print ========== create dnodes +sql create dnode $hostname port 7200 + +$x = 0 +create1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +if $data4_2 != ready then + goto create1 +endi + +print ========== stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGKILL + +print =============== create database +sql_error create database d1 vgroups 4 + +print ========== start dnode2 +system sh/exec.sh -n dnode2 -s start + +print =============== re-create database +$x = 0 +re-create1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql create database d1 vgroups 2 -x re-create1 + +sql show databases +if $rows != 1 then + return -1 +endi + +if $data00 != d1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/basic1.sim b/tests/script/sim/dnode/basic1.sim similarity index 100% rename from tests/script/unique/dnode/basic1.sim rename to tests/script/sim/dnode/basic1.sim diff --git a/tests/script/general/table/basic1.sim b/tests/script/sim/table/basic1.sim similarity index 100% rename from tests/script/general/table/basic1.sim rename to tests/script/sim/table/basic1.sim diff --git a/tests/script/general/user/basic1.sim b/tests/script/sim/user/basic1.sim similarity index 100% rename from tests/script/general/user/basic1.sim rename to tests/script/sim/user/basic1.sim From d7ddb1589b309ff3c856c5ff9ab3f85c3b639d0d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 10 Jan 2022 21:44:58 -0500 Subject: [PATCH 12/34] TD-12678 datasink interface adjust --- .../libs/executor/inc => include/libs/executor}/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataSinkMgt.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename {source/libs/executor/inc => include/libs/executor}/dataSinkMgt.h (96%) diff --git a/source/libs/executor/inc/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h similarity index 96% rename from source/libs/executor/inc/dataSinkMgt.h rename to include/libs/executor/dataSinkMgt.h index d13423b25d..6470f38ced 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -73,9 +73,9 @@ void dsEndPut(DataSinkHandle handle); /** * Get the length of the data returned by the next call to dsGetDataBlock. * @param handle - * @return data length + * @param pLen data length */ -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus); +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); /** * Get data, the caller needs to allocate data memory. diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 8a96c5d05f..01cc84c4f7 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -41,9 +41,9 @@ void dsEndPut(DataSinkHandle handle) { return pHandleImpl->fEndPut(pHandleImpl); } -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus) { +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetLen(pHandleImpl, pStatus); + *pLen = pHandleImpl->fGetLen(pHandleImpl, pStatus); } int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus) { From 1d5a97cffb1eabc5022b36aac3b16eab077af413 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 18:48:19 -0800 Subject: [PATCH 13/34] minor changes --- source/dnode/mnode/impl/src/mndStb.c | 6 ++-- source/dnode/mnode/impl/src/mndTrans.c | 2 +- tests/script/sim/db/error1.sim | 40 +++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7d77e29d74..ad8c16f826 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -204,7 +204,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { SSdb *pSdb = pMnode->pSdb; SStbObj *pStb = sdbAcquire(pSdb, SDB_STB, stbName); - if (pStb == NULL) { + if (pStb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_STB_NOT_EXIST; } return pStb; @@ -513,9 +513,11 @@ static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg) { return 0; } else { terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; - mError("db:%s, failed to create since %s", pCreate->name, terrstr()); + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } + } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); } // topic should have different name with stb diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 058178260f..2301df65d7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -825,7 +825,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, stage from undoAction to undoLog", pTrans->id); continueExec = true; } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mError("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); + mDebug("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { pTrans->failedTimes++; diff --git a/tests/script/sim/db/error1.sim b/tests/script/sim/db/error1.sim index 6698ca6f67..bf9e04c017 100644 --- a/tests/script/sim/db/error1.sim +++ b/tests/script/sim/db/error1.sim @@ -58,4 +58,42 @@ if $data03 != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ========== stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGKILL + +print =============== create database +sql_error drop database d1 + +print ========== start dnode2 +system sh/exec.sh -n dnode2 -s start + +print =============== re-create database +$x = 0 +re-create2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql create database d1 vgroups 5 -x re-create2 + +sql show databases +if $rows != 1 then + return -1 +endi + +if $data00 != d1 then + return -1 +endi + +if $data02 != 5 then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file From 52457838256af3b3fa4d9f3fabb4a0a83eaefd7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 10:51:23 +0800 Subject: [PATCH 14/34] [td-11818] add data into sink node --- include/libs/executor/executor.h | 12 ++-- source/libs/executor/inc/dataSinkMgt.h | 7 +- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/dataSinkMgt.c | 1 + source/libs/executor/src/executorMain.c | 88 +++++++++++++++---------- source/libs/executor/src/executorimpl.c | 1 + source/libs/qworker/src/qworker.c | 8 +-- src/inc/tsdb.h | 16 ++--- src/query/inc/qExecutor.h | 4 +- src/query/src/qExecutor.c | 66 +++++++++---------- 11 files changed, 120 insertions(+), 91 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2356498bbe..71b014d025 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -21,6 +21,9 @@ extern "C" { #endif typedef void* qTaskInfo_t; +typedef void* DataSinkHandle; +struct SSubplan; + /** * Create the exec task object according to task json @@ -34,13 +37,14 @@ typedef void* qTaskInfo_t; int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main task execution function, including query on both table and multiple tables, + * The main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * - * @param qinfo + * @param tinfo + * @param handle * @return */ -bool qExecTask(qTaskInfo_t qTask, SSDataBlock** pRes); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -62,7 +66,7 @@ int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspCo * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) diff --git a/source/libs/executor/inc/dataSinkMgt.h b/source/libs/executor/inc/dataSinkMgt.h index d0057a213a..35a25f5104 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/source/libs/executor/inc/dataSinkMgt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "os.h" +#include "executor.h" #include "executorimpl.h" #define DS_CAPACITY_ENOUGH 1 @@ -39,8 +40,6 @@ typedef struct SDataSinkMgtCfg { int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); -typedef void* DataSinkHandle; - typedef struct SInputData { const SSDataBlock* pData; SHashObj* pTableRetrieveTsMap; @@ -68,6 +67,10 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH */ int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus); +/** + * + * @param handle + */ void dsEndPut(DataSinkHandle handle); /** diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ef66b3f247..59db66becc 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 33fafb4074..d76f270392 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -20,14 +20,16 @@ #include "ttszip.h" #include "tvariant.h" -#include "thash.h" +#include "dataSinkMgt.h" #include "executil.h" +#include "planner.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" +#include "thash.h" #include "tlockfree.h" #include "tpagedfile.h" -#include "planner.h" +#include "executor.h" struct SColumnFilterElem; @@ -256,6 +258,7 @@ typedef struct SExecTaskInfo { // void* rspContext; // response context char *sql; // query sql string jmp_buf env; // + DataSinkHandle dsHandle; struct SOperatorInfo *pRoot; } SExecTaskInfo; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 8a96c5d05f..d9b122fbdd 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tarray.h" #include "dataSinkMgt.h" #include "dataSinkInt.h" #include "planner.h" diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ab9e7a5211..0c82bd1dac 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ -#include -#include "exception.h" #include "os.h" +#include "tarray.h" +#include "dataSinkMgt.h" +#include "exception.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" @@ -69,8 +70,9 @@ void freeParam(STaskParam *param) { int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { assert(tsdb != NULL && pSubplan != NULL); + SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = doCreateExecTaskInfo(pSubplan, (SExecTaskInfo**) pTaskInfo, tsdb); + int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -81,8 +83,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ goto _error; } - DataSinkHandle pHandle = NULL; - code = dsCreateDataSinker(pSubplan->pDataSink, &pHandle); + code = dsCreateDataSinker(pSubplan->pDataSink, (*pTask)->dsHandle); _error: // if failed to add ref for all tables in this query, abort current query @@ -134,64 +135,79 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes) { - SExecTaskInfo *pTaskInfo = (SExecTaskInfo *) tinfo; - int64_t threadId = taosGetSelfPthreadId(); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*) curOwner); + qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, + (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; + return pTaskInfo->code; } - if(pTaskInfo->cost.start == 0) { + if (pTaskInfo->cost.start == 0) { pTaskInfo->cost.start = taosGetTimestampMs(); } if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", GET_TASKID(pTaskInfo)); -// return doBuildResCheck(pTaskInfo); + qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); + return pTaskInfo->code; } -// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; -// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { -// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); -// setTaskStatus(pTaskInfo, TASK_COMPLETED); -// return doBuildResCheck(pTaskInfo); -// } + // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; + // if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { + // qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); + // setTaskStatus(pTaskInfo, TASK_COMPLETED); + // return doBuildResCheck(pTaskInfo); + // } // error occurs, record the error code and return to client int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); -// return doBuildResCheck(pTaskInfo); + qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + return pTaskInfo->code; } - qDebug("QInfo:0x%"PRIx64" query task is launched", GET_TASKID(pTaskInfo)); + qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + int64_t st = 0; - int64_t st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - // todo put the result into sink node. + handle = &pTaskInfo->dsHandle; - pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); - publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + while(1) { + st = taosGetTimestampUs(); + SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", GET_TASKID(pTaskInfo)); -// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { -// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, -// pRuntimeEnv->resultInfo.total); - } else { -// qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pTaskInfo->qId, -// GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pRes == NULL) { // no results generated yet, abort + return pTaskInfo->code; + } + + int32_t status = 0; + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; + pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &status); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); + // } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + // qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + // pRuntimeEnv->resultInfo.total); + } + + if (status == DS_CAPACITY_FULL) { + qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), + 0, 0L, 0); + return pTaskInfo->code; + } } -// return doBuildResCheck(pTaskInfo); } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db25e384be..9aeb979806 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7181,6 +7181,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { pthread_mutex_init(&pTaskInfo->lock, NULL); pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->id.queryId = queryId; return pTaskInfo; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7351af83c5..ada534de7e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1038,9 +1038,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); queryRsped = true; - - SSDataBlock* pRes = NULL; - code = qExecTask(pTaskInfo, &pRes); + + DataSinkHandle handle = NULL; + code = qExecTask(pTaskInfo, &handle); queryDone = false; //TODO call executer to execute subquery @@ -1048,7 +1048,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (code) { QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, pRes)); +// QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, pRes)); QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 65c8a45d00..130628e799 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -274,7 +274,7 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle); +bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle); /** @@ -308,19 +308,19 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); /** * move to next block if exists * - * @param pQueryHandle + * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); +bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle); /** * Get current data block information * - * @param pQueryHandle + * @param pTsdbReadHandle * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -332,7 +332,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *p * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -340,11 +340,11 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta * the returned data block must be satisfied with the time window condition in any cases, * which means the SData data block is not actually the completed disk data blocks. * - * @param pQueryHandle query handle + * @param pTsdbReadHandle query handle * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList); /** * Get the qualified table id for a super table according to the tag query expression. diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 19ca8e7ed8..0c0e3363c8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -284,7 +284,7 @@ typedef struct SQueryRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -418,7 +418,7 @@ typedef struct SQueryParam { } SQueryParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index be1bfb8143..a6a9115b2f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2326,8 +2326,8 @@ _clean: static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; + tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; SMemRef* pMemRef = &pQueryAttr->memRef; assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -3148,10 +3148,10 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3159,7 +3159,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { { // set previous window @@ -3205,7 +3205,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; } @@ -4494,7 +4494,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4521,15 +4521,15 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { // if (isQueryKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; @@ -4562,7 +4562,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4627,8 +4627,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4674,7 +4674,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4748,7 +4748,7 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4769,11 +4769,11 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } return terrno; @@ -4831,19 +4831,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); break; } case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); + pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); break; } default: { // do nothing @@ -4974,13 +4974,13 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { *newgroup = false; - while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -5037,7 +5037,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)) { + if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) { return NULL; } else { break; @@ -5046,7 +5046,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -5069,7 +5069,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); @@ -5112,8 +5112,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -5142,7 +5142,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5165,7 +5165,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5189,7 +5189,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -5271,7 +5271,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; From 553cc16cbd32de3b33efbe4bb5edcf959dfe8573 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 18:53:25 -0800 Subject: [PATCH 15/34] rename --- tests/script/{general/db/basic.sim => sim/db/basic6.sim} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/script/{general/db/basic.sim => sim/db/basic6.sim} (100%) diff --git a/tests/script/general/db/basic.sim b/tests/script/sim/db/basic6.sim similarity index 100% rename from tests/script/general/db/basic.sim rename to tests/script/sim/db/basic6.sim From 25a48e400643a35d656b389da7098a624221933c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 00:30:11 -0500 Subject: [PATCH 16/34] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataDispatcher.c | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 6470f38ced..b44cad1668 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -24,9 +24,9 @@ extern "C" { #include "executorimpl.h" #define DS_CAPACITY_ENOUGH 1 -#define DS_CAPACITY_FULL 2 +#define DS_DATA_FULL 2 #define DS_NEED_SCHEDULE 3 -#define DS_END 4 +#define DS_QUERY_END 4 #define DS_IN_PROCESS 5 struct SDataSink; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3d8e51d04d..83f180cd1c 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -124,7 +124,8 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { pthread_mutex_lock(&pDispatcher->mutex); - int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_CAPACITY_ENOUGH : DS_CAPACITY_FULL; + int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery + ? DS_CAPACITY_ENOUGH : DS_DATA_FULL; pDispatcher->status = status; pthread_mutex_unlock(&pDispatcher->mutex); return status; @@ -152,14 +153,14 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); - pDispatcher->status = DS_END; + pDispatcher->status = DS_QUERY_END; pthread_mutex_unlock(&pDispatcher->mutex); } static int32_t getDataLength(SDataSinkHandle* pHandle, int32_t* pStatus) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { - *pStatus = getStatus(pDispatcher) ? DS_END : DS_IN_PROCESS; + *pStatus = (DS_QUERY_END == getStatus(pDispatcher) ? DS_QUERY_END : DS_IN_PROCESS); return 0; } SDataDispatchBuf* pBuf = NULL; From 47f1558e7c4c503cb2ccc7b0b22fbb44909e737b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 11 Jan 2022 15:40:10 +0800 Subject: [PATCH 17/34] add wal handle meta corrupt --- include/libs/wal/wal.h | 2 + source/libs/wal/inc/walInt.h | 25 +++- source/libs/wal/src/walMeta.c | 180 +++++++++++++++++++++++---- source/libs/wal/src/walSeek.c | 76 ++++++----- source/libs/wal/src/walWrite.c | 25 ++-- source/libs/wal/test/walMetaTest.cpp | 95 ++++++++++++++ source/util/src/tarray.c | 2 +- source/util/test/arrayTest.cpp | 32 +++++ 8 files changed, 361 insertions(+), 76 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 67d2009d3b..51aaa7d903 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -71,6 +71,7 @@ extern int32_t wDebugFlag; #define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_CUR_FAILED 1 @@ -98,6 +99,7 @@ typedef struct { } SWalCfg; typedef struct { + uint64_t magic; uint32_t cksumHead; uint32_t cksumBody; SWalReadHead head; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 7631593dd8..4624e05f10 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -17,10 +17,10 @@ #define _TD_WAL_INT_H_ #include "compare.h" -#include "tchecksum.h" -#include "wal.h" - #include "taoserror.h" +#include "tchecksum.h" +#include "tcoding.h" +#include "wal.h" #ifdef __cplusplus extern "C" { @@ -40,6 +40,19 @@ typedef struct WalIdxEntry { int64_t offset; } SWalIdxEntry; +static inline int tSerializeWalIdxEntry(void** buf, SWalIdxEntry* pIdxEntry) { + int tlen; + tlen += taosEncodeFixedI64(buf, pIdxEntry->ver); + tlen += taosEncodeFixedI64(buf, pIdxEntry->offset); + return 0; +} + +static inline void* tDeserializeWalIdxEntry(void* buf, SWalIdxEntry* pIdxEntry) { + buf = taosDecodeFixedI64(buf, &pIdxEntry->ver); + buf = taosDecodeFixedI64(buf, &pIdxEntry->offset); + return buf; +} + static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { SWalFileInfo* pInfoLeft = (SWalFileInfo*)pLeft; SWalFileInfo* pInfoRight = (SWalFileInfo*)pRight; @@ -130,12 +143,12 @@ int walMetaDeserialize(SWal* pWal, const char* bytes); // meta section end // seek section -int walChangeFile(SWal* pWal, int64_t ver); -int walChangeFileToLast(SWal* pWal); +int walChangeWrite(SWal* pWal, int64_t ver); +int walSetWrite(SWal* pWal); // seek section end int64_t walGetSeq(); -int walSeekVer(SWal* pWal, int64_t ver); +int walSeekWriteVer(SWal* pWal, int64_t ver); int walRoll(SWal* pWal); #ifdef __cplusplus diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 270a26bf80..ab3aa02f4a 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -17,7 +17,6 @@ #include "cJSON.h" #include "os.h" #include "taoserror.h" -#include "tfile.h" #include "tref.h" #include "walInt.h" @@ -34,13 +33,74 @@ static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } +static inline int64_t walScanLogGetLastVer(SWal* pWal) { + ASSERT(pWal->fileInfoSet != NULL); + int sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz > 0); + for (int i = 0; i < sz; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + + } + SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + + struct stat statbuf; + stat(fnameStr, &statbuf); + int readSize = MIN(WAL_MAX_SIZE, statbuf.st_size); + + FileFd fd = taosOpenFileRead(fnameStr); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + uint64_t magic = WAL_MAGIC; + + char* buf = malloc(readSize + 5); + if (buf == NULL) { + taosCloseFile(fd); + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + return -1; + } + + if (readSize != taosReadFile(fd, buf, readSize)) { + free(buf); + taosCloseFile(fd); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + char* found = strstr(buf, (const char*)&magic); + if (found == NULL) { + ASSERT(false); + // file has to be deleted + free(buf); + taosCloseFile(fd); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + char *another; + while((another = strstr(found + 1, (const char*)&magic)) != NULL) { + // read and validate + SWalHead *logContent = (SWalHead*)another; + if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { + found = another; + } + } + taosCloseFile(fd); + SWalHead *lastEntry = (SWalHead*)found; + + return lastEntry->head.version; +} + int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); + SArray* pLogInfoArray = taosArrayInit(8, sizeof(SWalFileInfo)); regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -51,19 +111,84 @@ int walCheckAndRepairMeta(SWal* pWal) { return -1; } + // scan log files and build new meta struct dirent* ent; while ((ent = readdir(dir)) != NULL) { char* name = basename(ent->d_name); int code = regexec(&logRegPattern, name, 0, NULL, 0); if (code == 0) { - int64_t firstVer; - sscanf(name, "%" PRId64 ".log", &firstVer); - taosArrayPush(pLogArray, &firstVer); + SWalFileInfo fileInfo; + memset(&fileInfo, -1, sizeof(SWalFileInfo)); + sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); + FileFd fd = taosOpenFileRead(ent->d_name); + //get lastVer + //get size + taosArrayPush(pLogInfoArray, &fileInfo); } } - // load meta - // if not match, or meta missing + regfree(&logRegPattern); + regfree(&idxRegPattern); + + taosArraySort(pLogInfoArray, compareWalFileInfo); + int oldSz = 0; + if (pWal->fileInfoSet) { + oldSz = taosArrayGetSize(pWal->fileInfoSet); + } + int newSz = taosArrayGetSize(pLogInfoArray); + // case 1. meta file not exist / cannot be parsed + if (pWal->fileInfoSet == NULL && newSz != 0) { + // recover fileInfo set + pWal->fileInfoSet = pLogInfoArray; + if (newSz != 0) { + // recover meta version + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; + pWal->writeCur = newSz - 1; + } + // recover file size + } else if (oldSz < newSz) { + for (int i = oldSz; i < newSz; i++) { + SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); + taosArrayPush(pWal->fileInfoSet, pFileInfo); + } + pWal->writeCur = newSz - 1; + } + + if (pWal->fileInfoSet && taosArrayGetSize(pWal->fileInfoSet) != 0) { + pWal->vers.lastVer = walScanLogGetLastVer(pWal); + ASSERT(pWal->vers.lastVer != -1); + } + + // case 2. versions in meta not match log + // or some log not included in meta + // (e.g. program killed) + // + // case 3. other corrupt cases + // +#if 0 + int sz = taosArrayGetSize(pLogInfoArray); + for (int i = 0; i < sz; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i); + if (i == 0 && pFileInfo->firstVer != walGetFirstVer(pWal)) { + //repair + } + + if (i > 0) { + SWalFileInfo* pLastFileInfo = taosArrayGet(pLogInfoArray, i-1); + if (pLastFileInfo->lastVer != pFileInfo->firstVer) { + + } + } + } +#endif + + int code = walSaveMeta(pWal); + if (code < 0) { + return -1; + } + + // get last version of this file + // // rebuild meta return 0; } @@ -87,6 +212,7 @@ int walRollFileInfo(SWal* pWal) { // TODO: change to emplace back SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -94,7 +220,7 @@ int walRollFileInfo(SWal* pWal) { pNewInfo->createTs = ts; pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; - taosArrayPush(pWal->fileInfoSet, pNewInfo); + taosArrayPush(pArray, pNewInfo); free(pNewInfo); return 0; } @@ -108,7 +234,16 @@ char* walMetaSerialize(SWal* pWal) { cJSON* pFiles = cJSON_CreateArray(); cJSON* pField; if (pRoot == NULL || pMeta == NULL || pFiles == NULL) { - // TODO + if(pRoot) { + cJSON_Delete(pRoot); + } + if(pMeta) { + cJSON_Delete(pMeta); + } + if(pFiles) { + cJSON_Delete(pFiles); + } + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -221,18 +356,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - int metaTfd = tfOpenCreateWrite(fnameStr); - if (metaTfd < 0) { + FileFd metaFd = taosOpenFileCreateWrite(fnameStr); + if (metaFd < 0) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != tfWrite(metaTfd, serialized, len)) { + if (len != taosWriteFile(metaFd, serialized, len)) { // TODO:clean file return -1; } - tfClose(metaTfd); + taosCloseFile(metaFd); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -247,7 +382,7 @@ int walLoadMeta(SWal* pWal) { // find existing meta file int metaVer = walFindCurMetaVer(pWal); if (metaVer == -1) { - return 0; + return -1; } char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); @@ -257,23 +392,20 @@ int walLoadMeta(SWal* pWal) { int size = statbuf.st_size; char* buf = malloc(size + 5); if (buf == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } memset(buf, 0, size + 5); - int tfd = tfOpenRead(fnameStr); - if (tfRead(tfd, buf, size) != size) { - tfClose(tfd); + FileFd fd = taosOpenFileRead(fnameStr); + if (taosReadFile(fd, buf, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(fd); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - if (code != 0) { - tfClose(tfd); - free(buf); - return -1; - } - tfClose(tfd); + taosCloseFile(fd); free(buf); - return 0; + return code; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1d9f7bdf4d..91b172444b 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -20,7 +20,7 @@ #include "tref.h" #include "walInt.h" -static int walSeekFilePos(SWal* pWal, int64_t ver) { +static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; int64_t idxTfd = pWal->writeIdxTfd; @@ -41,7 +41,7 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_CUR); + code = tfLseek(logTfd, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -49,7 +49,7 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { return code; } -int walChangeFileToLast(SWal* pWal) { +int walSetWrite(SWal* pWal) { int64_t idxTfd, logTfd; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); @@ -57,13 +57,13 @@ int walChangeFileToLast(SWal* pWal) { char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenReadWrite(fnameStr); + idxTfd = tfOpenCreateWriteAppend(fnameStr); if (idxTfd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenReadWrite(fnameStr); + logTfd = tfOpenCreateWriteAppend(fnameStr); if (logTfd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -74,46 +74,57 @@ int walChangeFileToLast(SWal* pWal) { return 0; } -int walChangeFile(SWal* pWal, int64_t ver) { +int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; int64_t idxTfd, logTfd; char fnameStr[WAL_FILE_LEN]; - code = tfClose(pWal->writeLogTfd); - if (code != 0) { - // TODO - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + if (pWal->writeLogTfd != -1) { + code = tfClose(pWal->writeLogTfd); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } - code = tfClose(pWal->writeIdxTfd); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + if (pWal->writeIdxTfd != -1) { + code = tfClose(pWal->writeIdxTfd); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } + SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; // bsearch in fileSet - SWalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; - // closed - if (taosArrayGetLast(pWal->fileInfoSet) != pRet) { - walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenRead(fnameStr); - walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenRead(fnameStr); - } else { - walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenReadWrite(fnameStr); - walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenReadWrite(fnameStr); + int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + ASSERT(idx != -1); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx); + /*ASSERT(pFileInfo != NULL);*/ + + int64_t fileFirstVer = pFileInfo->firstVer; + walBuildIdxName(pWal, fileFirstVer, fnameStr); + idxTfd = tfOpenCreateWriteAppend(fnameStr); + if (idxTfd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + pWal->writeIdxTfd = -1; + return -1; + } + walBuildLogName(pWal, fileFirstVer, fnameStr); + logTfd = tfOpenCreateWriteAppend(fnameStr); + if (logTfd < 0) { + tfClose(idxTfd); + terrno = TAOS_SYSTEM_ERROR(errno); + pWal->writeLogTfd = -1; + return -1; } pWal->writeLogTfd = logTfd; pWal->writeIdxTfd = idxTfd; + pWal->writeCur = idx; return fileFirstVer; } -int walSeekVer(SWal* pWal, int64_t ver) { +int walSeekWriteVer(SWal* pWal, int64_t ver) { int code; if (ver == pWal->vers.lastVer) { return 0; @@ -123,14 +134,15 @@ int walSeekVer(SWal* pWal, int64_t ver) { return -1; } if (ver < pWal->vers.snapshotVer) { + } if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { - code = walChangeFile(pWal, ver); + code = walChangeWrite(pWal, ver); if (code != 0) { return -1; } } - code = walSeekFilePos(pWal, ver); + code = walSeekWritePos(pWal, ver); if (code != 0) { return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 975f232e3d..2bc328b4e2 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -46,12 +46,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // find correct file if (ver < walGetLastFileFirstVer(pWal)) { - // close current files - tfClose(pWal->writeIdxTfd); - tfClose(pWal->writeLogTfd); - // open old files - code = walChangeFile(pWal, ver); - if (code != 0) { + // change current files + code = walChangeWrite(pWal, ver); + if (code < 0) { return -1; } @@ -166,7 +163,8 @@ int32_t walEndSnapshot(SWal *pWal) { } // iterate files, until the searched result for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) { + if ((pWal->cfg.retentionSize != -1 && pWal->totSize > pWal->cfg.retentionSize) + || (pWal->cfg.retentionPeriod != -1 && iter->closeTs + pWal->cfg.retentionPeriod > ts)) { // delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; @@ -191,13 +189,12 @@ int32_t walEndSnapshot(SWal *pWal) { pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; - ; pWal->totSize = newTotSize; pWal->vers.verInSnapshotting = -1; // save snapshot ver, commit ver int code = walSaveMeta(pWal); - if (code != 0) { + if (code < 0) { return -1; } @@ -225,18 +222,17 @@ int walRoll(SWal *pWal) { walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWriteAppend(fnameStr); if (idxTfd < 0) { - ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); logTfd = tfOpenCreateWriteAppend(fnameStr); if (logTfd < 0) { - ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } code = walRollFileInfo(pWal); if (code != 0) { - ASSERT(0); return -1; } @@ -291,8 +287,11 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); + if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { - walChangeFileToLast(pWal); + walSetWrite(pWal); + tfLseek(pWal->writeLogTfd, 0, SEEK_END); + tfLseek(pWal->writeIdxTfd, 0, SEEK_END); } pWal->writeHead.head.version = index; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index cd082a3a43..a95c75b11d 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -107,6 +107,43 @@ class WalKeepEnv : public ::testing::Test { const char* pathName = "/tmp/wal_test"; }; +class WalRetentionEnv : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1, + cfg.segSize = -1, + cfg.retentionPeriod = -1, + cfg.retentionSize = 0, + cfg.rollPeriod = 0, + cfg.vgId = 0, + cfg.level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; +}; + TEST_F(WalCleanEnv, createNew) { walRollFileInfo(pWal); ASSERT(pWal->fileInfoSet != NULL); @@ -283,3 +320,61 @@ TEST_F(WalKeepEnv, readHandleRead) { } } } + +TEST_F(WalRetentionEnv, repairMeta1) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + + TearDown(); + + //getchar(); + char buf[100]; + sprintf(buf, "%s/meta-ver%d", pathName, 0); + remove(buf); + SetUp(); + + ASSERT_EQ(pWal->vers.lastVer, 99); + + SWalReadHandle* pRead = walOpenReadHandle(pWal); + ASSERT(pRead != NULL); + + for (int i = 0; i < 1000; i++) { + int ver = rand() % 100; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + + for (i = 100; i < 200; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index cc8d6646b6..09aeff5ff3 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -342,7 +342,7 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { void* item = taosArraySearch(pArray, key, comparFn, flags); - return (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; + return item == NULL ? -1 : (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; } void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { diff --git a/source/util/test/arrayTest.cpp b/source/util/test/arrayTest.cpp index 94b08ca6d7..939d4a701d 100644 --- a/source/util/test/arrayTest.cpp +++ b/source/util/test/arrayTest.cpp @@ -4,6 +4,7 @@ #include #include "tarray.h" +#include "tcompare.h" namespace { @@ -48,3 +49,34 @@ static void remove_batch_test() { TEST(arrayTest, array_list_test) { remove_batch_test(); } + +TEST(arrayTest, array_search_test) { + SArray *pa = (SArray*) taosArrayInit(4, sizeof(int32_t)); + + for(int32_t i = 10; i < 20; ++i) { + int32_t a = i; + taosArrayPush(pa, &a); + } + + for(int i = 0; i < 30; i++) { + int32_t k = i; + int32_t* pRet = (int32_t*)taosArraySearch(pa, &k, compareInt32Val, TD_GE); + int32_t idx = taosArraySearchIdx(pa, &k, compareInt32Val, TD_GE); + + if(pRet == NULL) { + ASSERT_EQ(idx, -1); + } else { + ASSERT_EQ(taosArrayGet(pa, idx), pRet); + } + + pRet = (int32_t*)taosArraySearch(pa, &k, compareInt32Val, TD_LE); + idx = taosArraySearchIdx(pa, &k, compareInt32Val, TD_LE); + + if(pRet == NULL) { + ASSERT_EQ(idx, -1); + } else { + ASSERT_EQ(taosArrayGet(pa, idx), pRet); + } + + } +} From 3e40fa34277f2e609206c5870cb112bab962decb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 15:42:27 +0800 Subject: [PATCH 18/34] [td-11818] fix error in fetch data. --- include/libs/planner/planner.h | 2 +- include/libs/scheduler/scheduler.h | 10 ++-- source/client/inc/clientInt.h | 6 ++- source/client/src/clientImpl.c | 58 +++++++++++++++-------- source/client/test/clientTests.cpp | 2 +- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/planner/src/planner.c | 11 +---- source/libs/planner/test/phyPlanTests.cpp | 2 +- source/libs/scheduler/inc/schedulerInt.h | 1 - source/libs/scheduler/src/scheduler.c | 50 ++++++++++--------- 10 files changed, 76 insertions(+), 70 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 91421a9174..da70f21498 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -150,7 +150,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 74b7813465..ceb54eabef 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,8 +20,10 @@ extern "C" { #endif -#include "planner.h" #include "catalog.h" +#include "planner.h" + +struct SSchJob; typedef struct SSchedulerCfg { uint32_t maxJobNum; @@ -65,7 +67,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes); +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -73,7 +75,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob); +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor @@ -81,7 +83,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, * @param data * @return */ -int32_t scheduleFetchRows(void *pJob, void **data); +int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 26afe237c9..7292375e7f 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -113,6 +113,7 @@ typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now void* fp; SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + struct SSchJob *pQueryJob; // query job, created according to sql query DAG. SDataBuf requestMsg; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -129,7 +130,7 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; + uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -161,12 +162,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str); void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + void initMsgHandleFp(); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); -TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void *doFetchRow(SRequestObj* pRequest); + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 26f97cc6ef..14cb35355f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,6 +1,10 @@ +#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" @@ -8,9 +12,6 @@ #include "tnote.h" #include "tpagedfile.h" #include "tref.h" -#include "parser.h" -#include "planner.h" -#include "scheduler.h" #define CHECK_CODE_GOTO(expr, lable) \ do { \ @@ -57,6 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i } static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { if (taos_init() != TSDB_CODE_SUCCESS) { @@ -198,36 +200,48 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - SSchema *pSchema = NULL; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - - int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &pResInfo->numOfCols, pRequest->requestId); + int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); if (code != 0) { return code; } if (pQueryNode->type == TSDB_SQL_SELECT) { - pResInfo->fields = calloc(1, sizeof(TAOS_FIELD)); - for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - pResInfo->fields[i].bytes = pSchema[i].bytes; - pResInfo->fields[i].type = pSchema[i].type; - tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); - } + SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0); + + SSubplan* pPlan = taosArrayGetP(pa, 0); + SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); + setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; } return code; } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { +void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) { + assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0); + + pResInfo->numOfCols = pDataBlockSchema->numOfCols; + pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0])); + + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + SSchema* pSchema = &pDataBlockSchema->pSchema[i]; + pResInfo->fields[i].bytes = pSchema->bytes; + pResInfo->fields[i].type = pSchema->type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } +} + +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res); + int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { - if (*pJob != NULL) { - scheduleFreeJob(*pJob); + if (pRequest->body.pQueryJob != NULL) { + scheduleFreeJob(pRequest->body.pQueryJob); } } @@ -235,7 +249,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { return res.code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { @@ -312,7 +326,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQuery = NULL; SQueryDag *pDag = NULL; - void *pJob = NULL; terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -322,9 +335,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return); pRequest->code = terrno; - return pRequest; } _return: @@ -333,6 +345,7 @@ _return: if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; } + return pRequest; } @@ -531,7 +544,10 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - if (pRequest->type == TDMT_MND_SHOW) { + if (pRequest->type == TDMT_VND_QUERY) { + pRequest->type = TDMT_VND_FETCH; + scheduleFetchRows(pRequest->body.pQueryJob, &pRequest->body.resInfo.pData); + } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index af5ad874f8..d507c565df 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -568,7 +568,7 @@ TEST(testCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); // pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); -// taos_free_result(pRes); + taos_free_result(pRes); // // pRes = taos_query(pConn, "create table tu using m1 tags(1)"); // taos_free_result(pRes); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 7ef434ccf5..01fb3c2513 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -89,7 +89,7 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfExpr; - dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfExpr); if (NULL == dataBlockSchema->pSchema) { return false; } @@ -306,8 +306,6 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; - case QNODE_PROJECT: -// node = create case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index de3454ddbb..3047ef4f5a 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -56,7 +56,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { SQueryPlanNode* pLogicPlan; int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { @@ -70,15 +70,6 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, printf("%s\n", str); } - int32_t numOfOutput = pLogicPlan->numOfExpr; - *pSchema = calloc(numOfOutput, sizeof(SSchema)); - *numOfResCols = numOfOutput; - - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pLogicPlan->pExpr, i); - memcpy(&(*pSchema)[i], pExprInfo->pExpr->pSchema, sizeof(SSchema)); - } - code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(pLogicPlan); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 0d3820cc77..2733a73a3f 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -65,7 +65,7 @@ protected: SSchema *schema = NULL; uint32_t numOfOutput = 0; - code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, requestId); + code = qCreateQueryDag(query, &dag, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index ada9b247ce..b2115b5c6d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -58,7 +58,6 @@ typedef struct SSchLevel { SArray *subTasks; // Element is SQueryTask } SSchLevel; - typedef struct SSchTask { uint64_t taskId; // task id SSchLevel *level; // level diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f9406a04b6..2ce555525f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -606,8 +606,8 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + if (rspCode != TSDB_CODE_SUCCESS || rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); if (code) { @@ -986,7 +986,7 @@ void schDropJobAllTasks(SSchJob *job) { } } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** job, bool syncSchedule) { +int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { if (nodeList && taosArrayGetSize(nodeList) <= 0) { qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); } @@ -1092,7 +1092,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) { +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1107,7 +1107,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1116,53 +1116,51 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, } -int32_t scheduleFetchRows(void *pJob, void **data) { - if (NULL == pJob || NULL == data) { +int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { + if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = pJob; int32_t code = 0; - - if (!SCH_JOB_NEED_FETCH(&job->attr)) { + if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { qError("no need to fetch data"); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (job->status == JOB_TASK_STATUS_FAILED) { - job->res = NULL; - SCH_RET(atomic_load_32(&job->errCode)); + if (pJob->status == JOB_TASK_STATUS_FAILED) { + pJob->res = NULL; + SCH_RET(atomic_load_32(&pJob->errCode)); } - if (job->status == JOB_TASK_STATUS_SUCCEED) { - job->res = NULL; + if (pJob->status == JOB_TASK_STATUS_SUCCEED) { + pJob->res = NULL; return TSDB_CODE_SUCCESS; } - if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { + if (atomic_val_compare_exchange_32(&pJob->userFetch, 0, 1) != 0) { qError("prior fetching not finished"); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (job->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(job)); + if (pJob->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); } - tsem_wait(&job->rspSem); + tsem_wait(&pJob->rspSem); - if (job->status == JOB_TASK_STATUS_FAILED) { - code = atomic_load_32(&job->errCode); + if (pJob->status == JOB_TASK_STATUS_FAILED) { + code = atomic_load_32(&pJob->errCode); } - if (job->res && ((SRetrieveTableRsp *)job->res)->completed) { - job->status = JOB_TASK_STATUS_SUCCEED; + if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { + pJob->status = JOB_TASK_STATUS_SUCCEED; } - *data = job->res; - job->res = NULL; + *pData = pJob->res; + pJob->res = NULL; _return: - atomic_val_compare_exchange_32(&job->userFetch, 1, 0); + atomic_val_compare_exchange_32(&pJob->userFetch, 1, 0); SCH_RET(code); } From e3ee88244e8fd08a0f90bf499613112ff425d3f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 23:42:25 -0800 Subject: [PATCH 19/34] TD-12868 --- source/common/src/tglobal.c | 2 +- tests/script/sh/deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a20fadbfb..463b107aa7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -37,7 +37,7 @@ uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second int8_t tsEnableTelemetryReporting = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 16; +int32_t tsNumOfSupportVnodes = 128; // common int32_t tsRpcTimer = 300; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index fcc11ca213..e00363b28f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -120,7 +120,7 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG -echo "supportVnodes 16" >> $TAOS_CFG +echo "supportVnodes 128" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG From 8a0da7bad296fbd3756c0de77edcec9c2d06aa02 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 02:56:24 -0500 Subject: [PATCH 20/34] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 25 ++++++++------- source/libs/executor/inc/dataSinkInt.h | 8 ++--- source/libs/executor/src/dataDispatcher.c | 38 +++++++++++++++-------- source/libs/executor/src/dataSinkMgt.c | 16 +++++----- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index b44cad1668..733e6d665d 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -23,11 +23,9 @@ extern "C" { #include "os.h" #include "executorimpl.h" -#define DS_CAPACITY_ENOUGH 1 -#define DS_DATA_FULL 2 -#define DS_NEED_SCHEDULE 3 -#define DS_QUERY_END 4 -#define DS_IN_PROCESS 5 +#define DS_BUF_LOW 1 +#define DS_BUF_FULL 2 +#define DS_BUF_EMPTY 3 struct SDataSink; struct SSDataBlock; @@ -46,11 +44,16 @@ typedef struct SInputData { SHashObj* pTableRetrieveTsMap; } SInputData; -typedef struct SOutPutData { +typedef struct SOutputData { int32_t numOfRows; int8_t compressed; char* pData; -} SOutPutData; + bool queryEnd; + bool needSchedule; + int32_t bufStatus; + int64_t useconds; + int8_t precision; +} SOutputData; /** * Create a subplan's datasinker handle for all later operations. @@ -66,16 +69,16 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH * @param pRes * @return error code */ -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus); +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue); -void dsEndPut(DataSinkHandle handle); +void dsEndPut(DataSinkHandle handle, int64_t useconds); /** * Get the length of the data returned by the next call to dsGetDataBlock. * @param handle * @param pLen data length */ -void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd); /** * Get data, the caller needs to allocate data memory. @@ -84,7 +87,7 @@ void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); * @param pStatus output * @return error code */ -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus); +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput); /** * After dsGetStatus returns DS_NEED_SCHEDULE, the caller need to put this into the work queue. diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 1bbf5494dd..69727626af 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -31,10 +31,10 @@ typedef struct SDataSinkManager { pthread_mutex_t mutex; } SDataSinkManager; -typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus); -typedef void (*FEndPut)(struct SDataSinkHandle* pHandle); -typedef int32_t (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pStatus); -typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus); +typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); +typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, int64_t useconds); +typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); +typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef struct SDataSinkHandle { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 83f180cd1c..954ed6daf1 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -42,6 +42,8 @@ typedef struct SDataDispatchHandle { STaosQueue* pDataBlocks; SDataDispatchBuf nextOutput; int32_t status; + bool queryEnd; + int64_t useconds; pthread_mutex_t mutex; } SDataDispatchHandle; @@ -124,8 +126,9 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { pthread_mutex_lock(&pDispatcher->mutex); - int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery - ? DS_CAPACITY_ENOUGH : DS_DATA_FULL; + int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); + int32_t status = (0 == blockNums ? DS_BUF_EMPTY : + (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; pthread_mutex_unlock(&pDispatcher->mutex); return status; @@ -138,7 +141,7 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { return status; } -static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus) { +static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf)); if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { @@ -146,38 +149,46 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, } toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - *pStatus = updateStatus(pDispatcher); + *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); return TSDB_CODE_SUCCESS; } -static void endPut(struct SDataSinkHandle* pHandle) { +static void endPut(struct SDataSinkHandle* pHandle, int64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); - pDispatcher->status = DS_QUERY_END; + pDispatcher->queryEnd = true; + pDispatcher->useconds = useconds; pthread_mutex_unlock(&pDispatcher->mutex); } -static int32_t getDataLength(SDataSinkHandle* pHandle, int32_t* pStatus) { +static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { - *pStatus = (DS_QUERY_END == getStatus(pDispatcher) ? DS_QUERY_END : DS_IN_PROCESS); - return 0; + *pQueryEnd = pDispatcher->queryEnd; + *pLen = 0; + return; } SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); - return ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; } -static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus) { +static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; tfree(pDispatcher->nextOutput.pData); // todo persistent - *pStatus = updateStatus(pDispatcher); + pOutput->bufStatus = updateStatus(pDispatcher); + pthread_mutex_lock(&pDispatcher->mutex); + pOutput->queryEnd = pDispatcher->queryEnd; + pOutput->needSchedule = false; + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + pthread_mutex_unlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } @@ -206,7 +217,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->pManager = pManager; dispatcher->schema = pDataSink->schema; - dispatcher->status = DS_CAPACITY_ENOUGH; + dispatcher->status = DS_BUF_EMPTY; + dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); pthread_mutex_init(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 01cc84c4f7..e3f0cd7eaa 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -31,24 +31,24 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH return TSDB_CODE_FAILED; } -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus) { +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fPut(pHandleImpl, pInput, pStatus); + return pHandleImpl->fPut(pHandleImpl, pInput, pContinue); } -void dsEndPut(DataSinkHandle handle) { +void dsEndPut(DataSinkHandle handle, int64_t useconds) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fEndPut(pHandleImpl); + return pHandleImpl->fEndPut(pHandleImpl, useconds); } -void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus) { +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - *pLen = pHandleImpl->fGetLen(pHandleImpl, pStatus); + pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); } -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus) { +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetData(pHandleImpl, pOutput, pStatus); + return pHandleImpl->fGetData(pHandleImpl, pOutput); } void dsScheduleProcess(void* ahandle, void* pItem) { From 4d8cc7aedd682015b11fc19197ba530760f3eac1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 03:20:06 -0500 Subject: [PATCH 21/34] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataDispatcher.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 733e6d665d..a0819fcf85 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "executorimpl.h" +#include "thash.h" #define DS_BUF_LOW 1 #define DS_BUF_FULL 2 @@ -40,7 +40,7 @@ int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); typedef void* DataSinkHandle; typedef struct SInputData { - const SSDataBlock* pData; + const struct SSDataBlock* pData; SHashObj* pTableRetrieveTsMap; } SInputData; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 954ed6daf1..e4b0557bff 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -19,6 +19,7 @@ #include "tcompression.h" #include "tglobal.h" #include "tqueue.h" +#include "executorimpl.h" #define DATA_META_LENGTH(tables) (sizeof(int32_t) + sizeof(STableIdInfo) * taosHashGetSize(tables) + sizeof(SRetrieveTableRsp)) From 93be707cdb0858e5db748a7cca57d08922bd244e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 16:23:23 +0800 Subject: [PATCH 22/34] feature/qnode --- source/client/src/clientImpl.c | 2 +- source/libs/executor/src/executorMain.c | 6 +- source/libs/qworker/src/qworker.c | 9 +-- source/libs/scheduler/src/scheduler.c | 84 +++++++++++++------------ 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 14cb35355f..616397f8e8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -546,7 +546,7 @@ void* doFetchRow(SRequestObj* pRequest) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; - scheduleFetchRows(pRequest->body.pQueryJob, &pRequest->body.resInfo.pData); + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index e8799542b2..3f77e15382 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -191,9 +191,9 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { return pTaskInfo->code; } - int32_t status = 0; + bool qcontinue = false; SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; - pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &status); + pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &qcontinue); if (isTaskKilled(pTaskInfo)) { qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); @@ -202,7 +202,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { // pRuntimeEnv->resultInfo.total); } - if (status == DS_DATA_FULL) { + if (!qcontinue) { qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), 0, 0L, 0); return pTaskInfo->code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 91a27a831f..9401d38fc7 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -6,6 +6,7 @@ #include "qworkerInt.h" #include "tmsg.h" #include "tname.h" +#include "dataSinkMgt.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -891,14 +892,10 @@ int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uin QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - code = dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd); - if (TSDB_CODE_SUCCESS != code) { - qError("dsGetDataLength failed, code:%x", code); - QW_ERR_JRET(code); - } + dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd); if (dataLength > 0) { - SOutPutData output = {0}; + SOutputData output = {0}; QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp)); output.pData = rsp->data; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 9ce2e80951..37e061126b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -509,6 +509,27 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } + +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + // if already FAILED, no more processing + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); + + if (errCode) { + atomic_store_32(&pJob->errCode, errCode); + } + + if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + tsem_post(&pJob->rspSem); + } + + SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + + assert(0); +} + + + // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -540,25 +561,6 @@ _return: } -// Note: no more error processing, handled in function internal -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { - // if already FAILED, no more processing - SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); - - if (errCode) { - atomic_store_32(&pJob->errCode, errCode); - } - - if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { - tsem_post(&pJob->rspSem); - } - - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); - - assert(0); -} - - // Note: no more error processing, handled in function internal int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; @@ -760,8 +762,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } - break; - } case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; @@ -784,8 +784,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } - break; - } case TDMT_VND_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -1316,7 +1314,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, true)); *pJob = job; @@ -1333,7 +1331,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, false)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); *pJob = job; @@ -1368,11 +1366,11 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { } if (status == JOB_TASK_STATUS_FAILED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } else if (status == JOB_TASK_STATUS_SUCCEED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { @@ -1392,9 +1390,9 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { } while (true) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); - if (*data != atomic_val_compare_exchange_ptr(&pJob->res, *data, NULL)) { + if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { continue; } @@ -1472,17 +1470,21 @@ void scheduleFreeJob(void *job) { taosArrayDestroy(pLevel->subTasks); } - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); - tfree(pJob->res); - - tfree(pJob); + taosArrayDestroy(pJob->levels); + + tfree(pJob->res); + + tfree(pJob); +} + +void schedulerDestroy(void) { + if (schMgmt.jobs) { + taosHashCleanup(schMgmt.jobs); //TODO + schMgmt.jobs = NULL; } - - void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(sch +} + From c22de779ac3956c267bcd9e913c015f072665359 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 11 Jan 2022 16:55:40 +0800 Subject: [PATCH 23/34] add tmemmem --- source/libs/wal/src/walMeta.c | 52 ++++++++++++++++++++-------- source/libs/wal/src/walMgmt.c | 1 + source/libs/wal/test/walMetaTest.cpp | 3 ++ 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index ab3aa02f4a..1c08f88dc6 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "cJSON.h" #include "os.h" #include "taoserror.h" @@ -33,6 +32,24 @@ static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } +void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { + char* limit; + + if (nlen == 0 || hlen < nlen) { + return false; + } + + limit = haystack + hlen - nlen + 1; + while ((haystack = (char*)memchr( + haystack, needle[0], limit - haystack)) != NULL) { + if (memcmp(haystack, needle, nlen) == 0) { + return haystack; + } + haystack++; + } + return NULL; +} + static inline int64_t walScanLogGetLastVer(SWal* pWal) { ASSERT(pWal->fileInfoSet != NULL); int sz = taosArrayGetSize(pWal->fileInfoSet); @@ -47,7 +64,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { struct stat statbuf; stat(fnameStr, &statbuf); - int readSize = MIN(WAL_MAX_SIZE, statbuf.st_size); + int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); FileFd fd = taosOpenFileRead(fnameStr); if (fd < 0) { @@ -64,6 +81,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { return -1; } + taosLSeekFile(fd, -readSize, SEEK_END); if (readSize != taosReadFile(fd, buf, readSize)) { free(buf); taosCloseFile(fd); @@ -71,21 +89,25 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { return -1; } - char* found = strstr(buf, (const char*)&magic); - if (found == NULL) { - ASSERT(false); - // file has to be deleted - free(buf); - taosCloseFile(fd); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } - char *another; - while((another = strstr(found + 1, (const char*)&magic)) != NULL) { + char* haystack = buf; + char* found = NULL; + char *candidate = NULL; + while((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate - SWalHead *logContent = (SWalHead*)another; + SWalHead *logContent = (SWalHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = another; + found = candidate; + } + haystack = candidate + 1; + } + if (found == buf) { + SWalHead *logContent = (SWalHead*)found; + if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { + // file has to be deleted + free(buf); + taosCloseFile(fd); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; } } taosCloseFile(fd); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d12acb52c6..93ec4693a3 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -106,6 +106,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // init write buffer memset(&pWal->writeHead, 0, sizeof(SWalHead)); pWal->writeHead.head.headVer = WAL_HEAD_VER; + pWal->writeHead.magic = WAL_MAGIC; if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index a95c75b11d..5774eea8c0 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -340,7 +340,10 @@ TEST_F(WalRetentionEnv, repairMeta1) { char buf[100]; sprintf(buf, "%s/meta-ver%d", pathName, 0); remove(buf); + sprintf(buf, "%s/meta-ver%d", pathName, 1); + remove(buf); SetUp(); + //getchar(); ASSERT_EQ(pWal->vers.lastVer, 99); From a6303b07d38ca8cdbd6507ea6119310281eaa6e8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 11 Jan 2022 17:55:53 +0800 Subject: [PATCH 24/34] force repair --- source/libs/wal/src/walMeta.c | 40 +++++++++++++--------------- source/libs/wal/src/walMgmt.c | 5 +++- source/libs/wal/test/walMetaTest.cpp | 22 +++++++++++++++ tests/script/sim/db/basic1.sim | 3 +-- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 1c08f88dc6..cac80c0a5f 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -65,6 +65,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { struct stat statbuf; stat(fnameStr, &statbuf); int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); + pLastFileInfo->fileSize = statbuf.st_size; FileFd fd = taosOpenFileRead(fnameStr); if (fd < 0) { @@ -91,7 +92,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* haystack = buf; char* found = NULL; - char *candidate = NULL; + char *candidate; while((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate SWalHead *logContent = (SWalHead*)candidate; @@ -142,7 +143,6 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; memset(&fileInfo, -1, sizeof(SWalFileInfo)); sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - FileFd fd = taosOpenFileRead(ent->d_name); //get lastVer //get size taosArrayPush(pLogInfoArray, &fileInfo); @@ -159,28 +159,25 @@ int walCheckAndRepairMeta(SWal* pWal) { } int newSz = taosArrayGetSize(pLogInfoArray); // case 1. meta file not exist / cannot be parsed - if (pWal->fileInfoSet == NULL && newSz != 0) { - // recover fileInfo set - pWal->fileInfoSet = pLogInfoArray; - if (newSz != 0) { - // recover meta version - pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; - pWal->writeCur = newSz - 1; - } - // recover file size - } else if (oldSz < newSz) { + if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } + pWal->writeCur = newSz - 1; + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; + pWal->vers.lastVer = walScanLogGetLastVer(pWal); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; + ASSERT(pWal->vers.lastVer != -1); + + int code = walSaveMeta(pWal); + if (code < 0) { + taosArrayDestroy(pLogInfoArray); + return -1; + } } - if (pWal->fileInfoSet && taosArrayGetSize(pWal->fileInfoSet) != 0) { - pWal->vers.lastVer = walScanLogGetLastVer(pWal); - ASSERT(pWal->vers.lastVer != -1); - } - // case 2. versions in meta not match log // or some log not included in meta // (e.g. program killed) @@ -204,14 +201,11 @@ int walCheckAndRepairMeta(SWal* pWal) { } #endif - int code = walSaveMeta(pWal); - if (code < 0) { - return -1; - } // get last version of this file // // rebuild meta + taosArrayDestroy(pLogInfoArray); return 0; } @@ -419,6 +413,10 @@ int walLoadMeta(SWal* pWal) { } memset(buf, 0, size + 5); FileFd fd = taosOpenFileRead(fnameStr); + if (fd < 0) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } if (taosReadFile(fd, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); taosCloseFile(fd); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 93ec4693a3..d5c28d9d9b 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -122,7 +122,9 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { + walLoadMeta(pWal); + + if (walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -131,6 +133,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } if (walCheckAndRepairIdx(pWal) < 0) { + } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 5774eea8c0..b65a200ca1 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -380,4 +380,26 @@ TEST_F(WalRetentionEnv, repairMeta1) { ASSERT_EQ(code, 0); } + for (int i = 0; i < 1000; i++) { + int ver = rand() % 200; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); + //} + // printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver + 1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for (int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } + } diff --git a/tests/script/sim/db/basic1.sim b/tests/script/sim/db/basic1.sim index 52af7d93ea..33af1c5b59 100644 --- a/tests/script/sim/db/basic1.sim +++ b/tests/script/sim/db/basic1.sim @@ -85,7 +85,6 @@ if $data02 != 2 then return -1 endi -return system sh/exec.sh -n dnode1 -s stop -x SIGKILL system sh/exec.sh -n dnode1 -s start @@ -104,4 +103,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From e105d952bcbd2bdab369ba6ae22093b302e0d1fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 18:15:37 +0800 Subject: [PATCH 25/34] feature/qnode --- include/libs/executor/dataSinkMgt.h | 1 + source/client/src/clientImpl.c | 9 ++ source/libs/executor/src/dataDispatcher.c | 7 +- source/libs/executor/src/executorMain.c | 7 +- source/libs/qworker/src/qworker.c | 126 +++++++++--------- source/libs/scheduler/src/scheduler.c | 42 +++++- source/libs/scheduler/test/schedulerTests.cpp | 4 +- 7 files changed, 127 insertions(+), 69 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 3ac16e9b29..f13ba5f87e 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "thash.h" +#include "executor.h" #define DS_BUF_LOW 1 #define DS_BUF_FULL 2 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 616397f8e8..727b668f3a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -547,6 +547,13 @@ void* doFetchRow(SRequestObj* pRequest) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; + } + + goto _return; } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { @@ -590,6 +597,8 @@ void* doFetchRow(SRequestObj* pRequest) { } } +_return: + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e4b0557bff..a69084f3db 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -109,10 +109,14 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema)); pEntry->numOfRows = pInput->pData->info.rows; + pEntry->dataLen = 0; pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap); copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen); - pBuf->useSize += (pEntry->compressed ? pEntry->dataLen : pHandle->schema.resultRowSize * pInput->pData->info.rows); + if (0 == pEntry->compressed) { + pEntry->dataLen = pHandle->schema.resultRowSize * pInput->pData->info.rows; + } + pBuf->useSize += pEntry->dataLen; // todo completed } @@ -213,6 +217,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS return TSDB_CODE_QRY_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; + dispatcher->sink.fEndPut = endPut; dispatcher->sink.fGetLen = getDataLength; dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 3f77e15382..50f69fb567 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -77,13 +77,13 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ goto _error; } - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = dsCreateDataSinker(pSubplan->pDataSink, (*pTask)->dsHandle); + code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle); _error: // if failed to add ref for all tables in this query, abort current query @@ -178,7 +178,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = 0; - handle = &pTaskInfo->dsHandle; + *handle = pTaskInfo->dsHandle; while(1) { st = taosGetTimestampUs(); @@ -188,6 +188,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pRes == NULL) { // no results generated yet, abort + dsEndPut(pTaskInfo->dsHandle, pTaskInfo->cost.elapsedTime); return pTaskInfo->code; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9401d38fc7..ad3e8073ee 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -617,6 +617,8 @@ int32_t qwInitFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + *rsp = pRsp; + return TSDB_CODE_SUCCESS; } @@ -866,7 +868,61 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI return TSDB_CODE_SUCCESS; } -int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + +int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; + int32_t code = 0; + int8_t newStatus = JOB_TASK_STATUS_CANCELLED; + + code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); + if (code) { + qError("sId:%"PRIx64" not in cache", sId); + QW_ERR_RET(code); + } + + code = qwAcquireTask(QW_READ, sch, qId, tId, &task); + if (code) { + qwReleaseScheduler(QW_READ, mgmt); + + if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { + qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); + QW_ERR_RET(code); + } + + QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); + } + + if (task->cancel) { + QW_LOCK(QW_WRITE, &task->lock); + qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); + QW_UNLOCK(QW_WRITE, &task->lock); + } + + if (task->drop) { + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + qwDropTask(mgmt, sId, qId, tId); + + return TSDB_CODE_SUCCESS; + } + + if (!(task->cancel || task->drop)) { + QW_LOCK(QW_WRITE, &task->lock); + qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + task->code = errCode; + QW_UNLOCK(QW_WRITE, &task->lock); + } + + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -876,6 +932,9 @@ int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uin int32_t dataLength = 0; SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; + SQWorkerTaskHandlesCache *handles = NULL; + + QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles)); QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -955,59 +1014,11 @@ _return: qwBuildAndSendFetchRsp(pMsg, rsp, dataLength, code); } - QW_RET(code); -} - -int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { - SQWSchStatus *sch = NULL; - SQWTaskStatus *task = NULL; - int32_t code = 0; - int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - - code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); - if (code) { - qError("sId:%"PRIx64" not in cache", sId); - QW_ERR_RET(code); - } - - code = qwAcquireTask(QW_READ, sch, qId, tId, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - - if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { - qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); - QW_ERR_RET(code); - } - - QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); - } - - if (task->cancel) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); - QW_UNLOCK(QW_WRITE, &task->lock); - } - - if (task->drop) { - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - qwDropTask(mgmt, sId, qId, tId); - - return TSDB_CODE_SUCCESS; - } - - if (!(task->cancel || task->drop)) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); - task->code = errCode; - QW_UNLOCK(QW_WRITE, &task->lock); + if (handles) { + qwReleaseTaskResCache(QW_READ, mgmt); } - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - return TSDB_CODE_SUCCESS; + QW_RET(code); } int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { @@ -1081,6 +1092,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { + qError("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); @@ -1094,6 +1106,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { code = qExecTask(pTaskInfo, &sinkHandle); if (code) { + qError("qExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); @@ -1225,17 +1238,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId)); void *data = NULL; - SQWorkerTaskHandlesCache *handles = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireTaskHandles(QW_READ, qWorkerMgmt, msg->queryId, msg->taskId, &handles)); + QW_ERR_RET(qwHandleFetch(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - QW_ERR_JRET(qwHandleFetch(handles, qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - -_return: - - qwReleaseTaskResCache(QW_READ, qWorkerMgmt); - QW_RET(code); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 37e061126b..ed12408844 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -68,7 +68,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -497,6 +497,33 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { return TSDB_CODE_SUCCESS; } + +int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { + if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { + SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + } + + int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + *moved = true; + + SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + *moved = true; + + SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); + + return TSDB_CODE_SUCCESS; +} + + int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -657,7 +684,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(code); } - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); @@ -689,6 +716,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } pJob->fetchTask = pTask; + + code = schMoveTaskToExecList(pJob, pTask, &moved); + if (code && moved) { + SCH_ERR_RET(code); + } SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); @@ -836,7 +868,10 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in atomic_add_fetch_32(&pJob->ref, 1); int32_t s = taosHashGetSize(pJob->execTasks); - assert(s != 0); + if (s <= 0) { + qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { @@ -1383,6 +1418,7 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (status == JOB_TASK_STATUS_FAILED) { code = atomic_load_32(&pJob->errCode); + SCH_ERR_JRET(code); } if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f75f0ef263..b438521234 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -217,7 +217,7 @@ void *schtSendRsp(void *param) { return NULL; } -void *pInsertJob = NULL; +struct SSchJob *pInsertJob = NULL; } @@ -228,7 +228,7 @@ TEST(queryTest, normalCase) { char *dbname = "1.db1"; char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - void *pJob = NULL; + SSchJob *pJob = NULL; SQueryDag dag = {0}; schtInitLogFile(); From 59d23065fd469447717b2bc7da46ce2ec012c01e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 02:54:11 -0800 Subject: [PATCH 26/34] fix invalid write in mnode --- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++ tests/script/sh/exec.sh | 4 +- tests/script/sim/db/basic6.sim | 56 ++++++++++--------------- 5 files changed, 33 insertions(+), 39 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 85b7fbbb42..bc4d890257 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -740,7 +740,7 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SCreateVnodeReq); + action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4bc570c11d..2a67e6a0f9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -235,7 +235,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = ABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * pMnode->cfg.statusInterval) { + if (interval > 10000 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e0d6d3dd42..93d6d104ff 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -319,6 +319,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + int32_t size = taosArrayGetSize(pArray); + if (size < pVgroup->replica) { + mError("db:%s, vgId:%d, no enough online dnodes:%d to alloc %d replica", pVgroup->dbName, pVgroup->vgId, size, + pVgroup->replica); + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + for (int32_t v = 0; v < pVgroup->replica; ++v) { SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pDnode = taosArrayGet(pArray, v); diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 2e95a740d0..05f756ebb6 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -30,7 +30,7 @@ do CLEAR_OPTION="clear" ;; v) - SHELL_OPTION="true" + VALGRIND_OPTION="true" ;; u) USERS=$OPTARG @@ -99,7 +99,7 @@ fi if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - if [ "$SHELL_OPTION" = "true" ]; then + if [ "$VALGRIND_OPTION" = "true" ]; then TT=`date +%s` mkdir ${LOG_DIR}/${TT} nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 684ce825fe..8e7d4f5ec1 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -1,29 +1,24 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 - system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect + print ============================ dnode1 start $i = 0 -$dbPrefix = ob_db_db -$tbPrefix = ob_db_tb +$dbPrefix = db +$tbPrefix = tb $db = $dbPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 cache 16 +sql create database $db replica 1 days 20 keep 2000 cache 16 vgroups 4 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then return -1 endi -if $data02 != 0 then +if $data02 != 4 then return -1 endi if $data03 != 0 then @@ -63,9 +58,6 @@ print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then return -1 endi -if $data02 != 0 then - return -1 -endi if $data03 != 0 then return -1 endi @@ -77,30 +69,14 @@ if $data06 != 15 then endi print =============== step6 -sql use $db -sql create table $tb (ts timestamp, speed int) -$i = 1 -while $i < 4 +$i = $i + 1 +while $i < 5 $db = $dbPrefix . $i - $tb = $tbPrefix . $i sql create database $db sql use $db - sql create table $tb (ts timestamp, speed int) $i = $i + 1 endw -sql show databases -if $rows != 4 then - return -1 -endi - -$i = 4 -$db = $dbPrefix . $i -$tb = $tbPrefix . $i -sql create database $db -sql use $db -sql create table $tb (ts timestamp, speed int) - print =============== step7 $i = 0 while $i < 5 @@ -115,7 +91,8 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i sql create database $db sql use $db -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 @@ -133,7 +110,8 @@ if $rows != 0 then endi print =============== step11 -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 @@ -149,16 +127,23 @@ sql show tables if $rows != 0 then return -1 endi -sql create table $tb (ts timestamp, speed int) + +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 endi + sql insert into $tb values (now+1a, 0) sql insert into $tb values (now+2a, 1) sql insert into $tb values (now+3a, 2) sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) + + +return + sql select * from $tb if $rows != 5 then return -1 @@ -176,7 +161,8 @@ if $rows != 0 then endi print =============== step16 -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 From 0920bb05763b6f8d1e6b0cc508b6190d332779a5 Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 11 Jan 2022 18:55:09 +0800 Subject: [PATCH 27/34] [modify] --- tests/test/c/create_table.c | 117 ++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index f2db9d0a0c..a5a55bbc91 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -40,6 +40,8 @@ typedef struct { float createTableSpeed; float insertDataSpeed; int64_t startMs; + int64_t maxDelay; + int64_t minDelay; pthread_t thread; } SThreadInfo; @@ -58,12 +60,30 @@ int32_t main(int32_t argc, char *argv[]) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - int64_t numOfTablesPerThread = numOfTables / numOfThreads; - numOfTables = numOfTablesPerThread * numOfThreads; + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = i * numOfTablesPerThread; - pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); @@ -74,9 +94,15 @@ int32_t main(int32_t argc, char *argv[]) { pthread_join(pInfo[i].thread, NULL); } + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + float createTableSpeed = 0; for (int32_t i = 0; i < numOfThreads; ++i) { createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; } float insertDataSpeed = 0; @@ -84,10 +110,19 @@ int32_t main(int32_t argc, char *argv[]) { insertDataSpeed += pInfo[i].insertDataSpeed; } - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d %s", GREEN, numOfTables, createTableSpeed, - numOfThreads, NC); - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, numOfThreads, NC); + } pthread_attr_destroy(&thattr); free(pInfo); @@ -99,36 +134,36 @@ void createDbAndStb() { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("failed to connect to DB, reason:%s", taos_errstr(con)); + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); exit(1); } sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); - TAOS_RES *pSql = taos_query(con, qstr); - int32_t code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + int32_t code = taos_errno(pRes); if (code != 0) { - pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "use %s", dbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); taos_close(con); } @@ -160,16 +195,20 @@ void *threadFunc(void *param) { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); exit(1); } + //printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); - TAOS_RES *pSql = taos_query(con, qstr); - taos_free_result(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + taos_free_result(pRes); + if (createTable) { - pInfo->startMs = taosGetTimestampMs(); + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); batch = MIN(batch, batchNum); @@ -179,14 +218,22 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + //printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; - if (t % 100000 == 0) { + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; printCreateProgress(pInfo, t); } t += (batch - 1); @@ -195,6 +242,9 @@ void *threadFunc(void *param) { } if (insertData) { + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs();; + pInfo->startMs = taosGetTimestampMs(); for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); @@ -205,14 +255,15 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); + taos_free_result(pRes); - if (t % 100000 == 0) { + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { printInsertProgress(pInfo, t); } t += (batch - 1); @@ -266,7 +317,7 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { + } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { createTable = atoi(argv[++i]); From 6677dfba943e176ddde55ca373fb6dfe26541fef Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 18:57:19 +0800 Subject: [PATCH 28/34] feature/qnode --- source/libs/qworker/src/qworker.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ad3e8073ee..2a395fcfe1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -964,6 +964,13 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 qError("dsGetDataBlock failed, code:%x", code); QW_ERR_JRET(code); } + + rsp->useconds = htobe64(output.useconds); + rsp->completed = 0; + rsp->precision = output.precision; + rsp->compressed = output.compressed; + rsp->compLen = htonl(dataLength); + rsp->numOfRows = htonl(output.numOfRows); if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { rsp->completed = 1; From f94f8aadc0b00e15a0869eb7291ae53b8bec8d49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:13:26 -0800 Subject: [PATCH 29/34] dnode may offline while create vnode --- source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/src/dndMgmt.c | 13 ++++++++++++- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index afdd678213..e637b38815 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -94,6 +94,7 @@ typedef struct { pthread_t *threadId; SRWLatch latch; SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; } SDnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index f252bffbbf..d9edf39b73 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -536,6 +536,11 @@ int32_t dndInitMgmt(SDnode *pDnode) { return -1; } + if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode); if (pMgmt->threadId == NULL) { dError("failed to init dnode thread"); @@ -550,6 +555,7 @@ int32_t dndInitMgmt(SDnode *pDnode) { void dndStopMgmt(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; dndCleanupWorker(&pMgmt->mgmtWorker); + dndCleanupWorker(&pMgmt->statusWorker); if (pMgmt->threadId != NULL) { taosDestoryThread(pMgmt->threadId); @@ -587,7 +593,12 @@ void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { dndUpdateMnodeEpSet(pDnode, pEpSet); } - if (dndWriteMsgToWorker(&pMgmt->mgmtWorker, pMsg, sizeof(SRpcMsg)) != 0) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + if (pMsg->msgType == TDMT_MND_STATUS_RSP) { + pWorker = &pMgmt->statusWorker; + } + + if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) { if (pMsg->msgType & 1u) { SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; rpcSendResponse(&rsp); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2a67e6a0f9..4bc570c11d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -235,7 +235,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = ABS(pDnode->lastAccessTime - curMs); - if (interval > 10000 * pMnode->cfg.statusInterval) { + if (interval > 3500 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } From 07a1b4acf21d25321819781d1ec9a2214d98e175 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:41:38 -0800 Subject: [PATCH 30/34] minor changes --- source/dnode/mnode/impl/test/profile/profile.cpp | 10 +++++----- tests/script/jenkins/basic.txt | 5 +++-- tests/script/sim/db/basic6.sim | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bdffb6c72a..bf047517d3 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -13,7 +13,7 @@ class MndTestProfile : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9022); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9031); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -53,7 +53,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -127,7 +127,7 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } @@ -185,7 +185,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -249,7 +249,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bc0c3a4f64..a84e473d9f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,8 +5,9 @@ ./test.sh -f sim/user/basic1.sim # ---- db -./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/error1.sim +./test.sh -f sim/db/basic1.sim +./test.sh -f sim/db/error6.sim +./test.sh -f sim/db/error1.sim # ---- table ./test.sh -f sim/table/basic1.sim diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 8e7d4f5ec1..a688b4c2f3 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -93,6 +93,10 @@ sql create database $db sql use $db sql create table st (ts timestamp, i int) tags (j int) sql create table $tb using st tags(1) + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT + sql show tables if $rows != 1 then return -1 From 530ffb6cbc6fc4b14a07c96388bc6e7fcf22c1fc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:46:05 -0800 Subject: [PATCH 31/34] minor changes --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a84e473d9f..bb6569deb0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -6,7 +6,7 @@ # ---- db ./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/error6.sim +./test.sh -f sim/db/basic6.sim ./test.sh -f sim/db/error1.sim # ---- table From f1f666e780238cbd39373ddb98fc6a07df6d6b3a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 23:22:08 +0800 Subject: [PATCH 32/34] add message queue for vnode --- include/dnode/vnode/vnode.h | 17 ++++++++++++++--- source/dnode/mgmt/impl/src/dnode.c | 15 +++++++++++++-- source/dnode/vnode/impl/inc/vnodeDef.h | 4 ++++ source/dnode/vnode/impl/src/vnodeMgr.c | 18 +++++++++++++----- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 2212a8c29a..b83ca53859 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -66,15 +66,26 @@ typedef struct SVnodeCfg { SWalCfg walCfg; } SVnodeCfg; +typedef struct SDnode SDnode; +typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef struct { + int32_t sver; + SDnode *pDnode; + char *timezone; + char *locale; + char *charset; + PutReqToVQueryQFp putReqToVQueryQFp; + uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) +} SVnodeOpt; + /* ------------------------ SVnode ------------------------ */ /** * @brief Initialize the vnode module * - * @param nthreads number of commit threads. 0 for no threads and - * a schedule queue should be given (TODO) + * @param pOption Option of the vnode mnodule * @return int 0 for success and -1 for failure */ -int vnodeInit(uint16_t nthreads); +int vnodeInit(const SVnodeOpt *pOption); /** * @brief clear a vnode diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index cd27781df3..f3016feda5 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -22,8 +22,8 @@ #include "dndTransport.h" #include "dndVnodes.h" #include "sync.h" -#include "wal.h" #include "tfs.h" +#include "wal.h" EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -153,6 +153,8 @@ static void dndCleanupEnv(SDnode *pDnode) { taosStopCacheRefreshWorker(); } +static void dndPutMsgToVQueryQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { dndProcessVnodeQueryMsg(pDnode, pRpcMsg, NULL); } + SDnode *dndInit(SDnodeOpt *pOption) { taosIgnSIGPIPE(); taosBlockSIGPIPE(); @@ -196,7 +198,16 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } - if (vnodeInit(pDnode->opt.numOfCommitThreads) != 0) { + SVnodeOpt vnodeOpt = { + .sver = pDnode->opt.sver, + .pDnode = pDnode, + .timezone = pDnode->opt.timezone, + .locale = pDnode->opt.locale, + .charset = pDnode->opt.charset, + .putReqToVQueryQFp = dndPutMsgToVQueryQ, + .nthreads = pDnode->opt.numOfCommitThreads, + }; + if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); dndCleanup(pDnode); return NULL; diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index e70e891794..361fdd10e0 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -57,6 +57,8 @@ typedef struct SVnodeMgr { pthread_cond_t hasTask; TD_DLIST(SVnodeTask) queue; // For vnode Mgmt + SDnode* pDnode; + PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -79,6 +81,8 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); +void vnodePutReqToVQueryQ(struct SRpcMsg *pReq); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 784d1abb60..43527c13fe 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -19,17 +19,19 @@ SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED}; static void* loop(void* arg); -int vnodeInit(uint16_t nthreads) { +int vnodeInit(const SVnodeOpt *pOption) { if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { return 0; } vnodeMgr.stop = false; + vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; + vnodeMgr.pDnode = pOption->pDnode; // Start commit handers - if (nthreads > 0) { - vnodeMgr.nthreads = nthreads; - vnodeMgr.threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + if (pOption->nthreads > 0) { + vnodeMgr.nthreads = pOption->nthreads; + vnodeMgr.threads = (pthread_t*)calloc(pOption->nthreads, sizeof(pthread_t)); if (vnodeMgr.threads == NULL) { return -1; } @@ -38,7 +40,7 @@ int vnodeInit(uint16_t nthreads) { pthread_cond_init(&(vnodeMgr.hasTask), NULL); TD_DLIST_INIT(&(vnodeMgr.queue)); - for (uint16_t i = 0; i < nthreads; i++) { + for (uint16_t i = 0; i < pOption->nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } @@ -89,6 +91,12 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } +void vnodePutReqToVQueryQ(struct SRpcMsg* pReq) { + if (vnodeMgr.putReqToVQueryQFp) { + (*vnodeMgr.putReqToVQueryQFp)(vnodeMgr.pDnode, pReq); + } +} + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { SVnodeTask* pTask; From ab3378e090100a98451f4fa606ca85c2a8fc9ec8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 16:44:11 -0800 Subject: [PATCH 33/34] add msg queue --- include/dnode/vnode/vnode.h | 11 ++++++----- source/dnode/mgmt/impl/src/dndVnodes.c | 6 ++++-- source/dnode/mgmt/impl/src/dnode.c | 3 +-- source/dnode/vnode/impl/inc/vnodeDef.h | 3 ++- source/dnode/vnode/impl/src/vnodeMain.c | 21 ++++++++++++--------- source/dnode/vnode/impl/src/vnodeMgr.c | 9 ++++----- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index b83ca53859..03a59b01e6 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -31,8 +31,12 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; +typedef struct SDnode SDnode; +typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); + typedef struct SVnodeCfg { int32_t vgId; + SDnode *pDnode; /** vnode buffer pool options */ struct { @@ -66,16 +70,13 @@ typedef struct SVnodeCfg { SWalCfg walCfg; } SVnodeCfg; -typedef struct SDnode SDnode; -typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct { int32_t sver; - SDnode *pDnode; char *timezone; char *locale; char *charset; - PutReqToVQueryQFp putReqToVQueryQFp; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) + PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; /* ------------------------ SVnode ------------------------ */ @@ -100,7 +101,7 @@ void vnodeClear(); * @param pVnodeCfg options of the vnode * @return SVnode* The vnode object */ -SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid); +SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg); /** * @brief Close a VNODE diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index bf27a542ae..9f585859a8 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -381,7 +381,8 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnode *pImpl = vnodeOpen(pCfg->path, NULL, pCfg->vgId); + SVnodeCfg cfg = {.pDnode = pDnode, .vgId = pCfg->vgId}; + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -581,7 +582,8 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { return -1; } - SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/, pCreate->vgId); + vnodeCfg.pDnode = pDnode; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index f3016feda5..ea42db96ab 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -200,12 +200,11 @@ SDnode *dndInit(SDnodeOpt *pOption) { SVnodeOpt vnodeOpt = { .sver = pDnode->opt.sver, - .pDnode = pDnode, .timezone = pDnode->opt.timezone, .locale = pDnode->opt.locale, .charset = pDnode->opt.charset, - .putReqToVQueryQFp = dndPutMsgToVQueryQ, .nthreads = pDnode->opt.numOfCommitThreads, + .putReqToVQueryQFp = dndPutMsgToVQueryQ, }; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 361fdd10e0..4f53dcd899 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -77,11 +77,12 @@ struct SVnode { SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; + SDnode* pDnode; }; int vnodeScheduleTask(SVnodeTask* task); -void vnodePutReqToVQueryQ(struct SRpcMsg *pReq); +void vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index eb4b45bc20..85ccc9879e 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -15,27 +15,29 @@ #include "vnodeDef.h" -static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid); +static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); -SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid) { +SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; // Set default options - //if (pVnodeCfg == NULL) { - pVnodeCfg = &defaultVnodeOptions; - //} + SVnodeCfg cfg = defaultVnodeOptions; + if (pVnodeCfg != NULL) { + cfg.vgId = pVnodeCfg->vgId; + cfg.pDnode = pVnodeCfg->pDnode; + } // Validate options - if (vnodeValidateOptions(pVnodeCfg) < 0) { + if (vnodeValidateOptions(&cfg) < 0) { // TODO return NULL; } // Create the handle - pVnode = vnodeNew(path, pVnodeCfg, vid); + pVnode = vnodeNew(path, &cfg); if (pVnode == NULL) { // TODO: handle error return NULL; @@ -62,7 +64,7 @@ void vnodeClose(SVnode *pVnode) { void vnodeDestroy(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid) { +static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; pVnode = (SVnode *)calloc(1, sizeof(*pVnode)); @@ -71,7 +73,8 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vi return NULL; } - pVnode->vgId = vid; + pVnode->vgId = pVnodeCfg->vgId; + pVnode->pDnode = pVnodeCfg->pDnode; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 43527c13fe..51f33031ac 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -26,7 +26,6 @@ int vnodeInit(const SVnodeOpt *pOption) { vnodeMgr.stop = false; vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; - vnodeMgr.pDnode = pOption->pDnode; // Start commit handers if (pOption->nthreads > 0) { @@ -91,10 +90,10 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } -void vnodePutReqToVQueryQ(struct SRpcMsg* pReq) { - if (vnodeMgr.putReqToVQueryQFp) { - (*vnodeMgr.putReqToVQueryQFp)(vnodeMgr.pDnode, pReq); - } +void vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + assert(vnodeMgr.putReqToVQueryQFp); + assert(pVnode->pDnode); + (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } /* ------------------------ STATIC METHODS ------------------------ */ From 53b4c06d9b1442c0063dc6520fa8982d913f8da3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 17:07:34 -0800 Subject: [PATCH 34/34] add msg queue --- include/dnode/vnode/vnode.h | 2 +- source/dnode/mgmt/impl/inc/dndVnodes.h | 2 ++ source/dnode/mgmt/impl/src/dndVnodes.c | 26 ++++++++++++++++++++------ source/dnode/mgmt/impl/src/dnode.c | 4 +--- source/dnode/vnode/impl/inc/vnodeDef.h | 2 +- source/dnode/vnode/impl/src/vnodeMgr.c | 10 ++++++---- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 03a59b01e6..4332fc8e58 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -32,7 +32,7 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; typedef struct SDnode SDnode; -typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index b5fae62959..a78db602fe 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -36,6 +36,8 @@ int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 9f585859a8..2d7999fe5a 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -802,7 +802,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t } } -static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) { +static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg, bool sendRsp) { int32_t code = 0; if (pQueue == NULL) { @@ -819,13 +819,15 @@ static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) } } - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS && sendRsp) { if (pRpcMsg->msgType & 1u) { SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; rpcSendResponse(&rsp); } rpcFreeCont(pRpcMsg->pCont); } + + return code; } static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { @@ -848,7 +850,7 @@ static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -856,7 +858,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -864,7 +866,7 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -872,11 +874,23 @@ void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } +int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + // pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); + if (pVnode == NULL) return -1; + + int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, false); + dndReleaseVnode(pDnode, pVnode); + return code; +} + static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) return -1; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index ea42db96ab..f3d5e09564 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -153,8 +153,6 @@ static void dndCleanupEnv(SDnode *pDnode) { taosStopCacheRefreshWorker(); } -static void dndPutMsgToVQueryQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { dndProcessVnodeQueryMsg(pDnode, pRpcMsg, NULL); } - SDnode *dndInit(SDnodeOpt *pOption) { taosIgnSIGPIPE(); taosBlockSIGPIPE(); @@ -204,7 +202,7 @@ SDnode *dndInit(SDnodeOpt *pOption) { .locale = pDnode->opt.locale, .charset = pDnode->opt.charset, .nthreads = pDnode->opt.numOfCommitThreads, - .putReqToVQueryQFp = dndPutMsgToVQueryQ, + .putReqToVQueryQFp = dndPutReqToVQueryQ, }; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 4f53dcd899..f9172dd351 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -82,7 +82,7 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); -void vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); +int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 51f33031ac..cc369a0d15 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -90,10 +90,12 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } -void vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - assert(vnodeMgr.putReqToVQueryQFp); - assert(pVnode->pDnode); - (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); +int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + if (pVnode == NULL || pVnode->pDnode == NULL || vnodeMgr.putReqToVQueryQFp == NULL) { + terrno = TSDB_CODE_VND_APP_ERROR; + return -1; + } + return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } /* ------------------------ STATIC METHODS ------------------------ */