diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 9e89e5a2f8..33c3ef55c9 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -430,8 +430,6 @@ pipeline { rm -rf ${WKC}/debug cd ${WKC}/tests/parallel_test time ./container_build.sh -w ${WKDIR} -t 10 -e - rm -f /tmp/cases.task - ./collect_cases.sh -e ''' def extra_param = "" def log_server_file = "/home/log_server.json" @@ -462,7 +460,7 @@ pipeline { cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' ''' } } diff --git a/cmake/cmake.version b/cmake/cmake.version index 92f7f8b895..03598519ed 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,13 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) -MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) +MESSAGE(STATUS "build date: " ${TD_VER_DATE}) +MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type: " ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fdbb2c5123..502ba10d33 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -244,7 +244,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index db7d0640f6..b16b5a2d4b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -190,6 +190,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SBuildUseDBInput; typedef struct SField { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 7717d3bccc..6154882756 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -56,6 +56,7 @@ typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; int64_t dbId; + int64_t stateTs; } SDbInfo; typedef struct STablesReq { @@ -153,7 +154,7 @@ int32_t catalogInit(SCatalogCfg* cfg); */ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* stateTs); /** * Get a DB's all vgroup info. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 74c1a8c07c..96ccef7164 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -127,6 +127,7 @@ typedef struct SDBVgInfo { int16_t hashSuffix; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; @@ -203,6 +204,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +void destroyAhandle(void* ahandle); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 53edcd4fee..8a0cccc71f 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -72,6 +72,7 @@ typedef struct SRpcMsg { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -97,6 +98,9 @@ typedef struct SRpcInit { // set up timeout for particular msg RpcTfp tfp; + // destroy client ahandle; + RpcDfp dfp; + void *parent; } SRpcInit; diff --git a/include/util/tdef.h b/include/util/tdef.h index ff1b26c21a..556ad6b680 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -290,7 +290,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB -#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 +#define TSDB_DEFAULT_BUFFER_PER_VNODE 256 #define TSDB_MIN_PAGES_PER_VNODE 64 #define TSDB_MAX_PAGES_PER_VNODE (INT32_MAX - 1) #define TSDB_DEFAULT_PAGES_PER_VNODE 256 diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb25a595af..bb93e4d934 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -140,12 +140,13 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; - // rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index fa0e3212a3..bf34d3e2df 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -61,7 +61,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); @@ -72,6 +72,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } vgInfo->vgVersion = rsp->vgVersion; + vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; vgInfo->hashPrefix = rsp->hashPrefix; vgInfo->hashSuffix = rsp->hashSuffix; @@ -486,6 +487,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); db->numOfTable = htonl(db->numOfTable); + db->stateTs = htobe64(db->stateTs); } SKv kv = { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e17e7e79c6..8437c93bae 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -181,7 +181,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); struct SCatalog* pCatalog = NULL; - if (usedbRsp.vgVersion >= 0) { + if (usedbRsp.vgVersion >= 0) { // cached in local uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { @@ -442,8 +442,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d0121aa15..536cbed33e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2197,7 +2197,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return rname.ctbShortName; } -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { + int32_t dataLen = 0; + // todo extract method int32_t* version = (int32_t*)data; *version = 1; @@ -2238,7 +2240,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = blockDataGetSerialMetaSize(numOfCols); + dataLen = blockDataGetSerialMetaSize(numOfCols); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -2255,26 +2257,23 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ } data += metaSize; - (*dataLen) += metaSize; + dataLen += metaSize; - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); } - *actualLen = *dataLen; + *actualLen = dataLen; *groupId = pBlock->info.groupId; - ASSERT(*dataLen > 0); - uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); + ASSERT(dataLen > 0); + + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols); + + return dataLen; } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 85a09b79fd..8a8561161b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -307,8 +307,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f2a4462bf5..78d84c9c25 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -606,7 +606,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } else { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and response after trans finished", createReq.db); + mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); code = TSDB_CODE_ACTION_IN_PROGRESS; goto _OVER; } else { @@ -1175,7 +1175,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { + if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); } @@ -1225,6 +1225,14 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { usedbRsp.vgVersion = usedbReq.vgVersion; usedbRsp.errCode = terrno; + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { + mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } + } + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { @@ -1255,7 +1263,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; _OVER: - if (code != 0) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1275,12 +1283,31 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, for (int32_t i = 0; i < numOfDbs; ++i) { SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); + pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); SUseDbRsp usedbRsp = {0}; + if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); + if (pDbVgVersion->vgVersion < vgVersion) { + usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); + usedbRsp.vgVersion = vgVersion++; + } else { + usedbRsp.vgVersion = pDbVgVersion->vgVersion; + } + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + + continue; + } + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { mTrace("db:%s, no exist", pDbVgVersion->dbFName); @@ -1293,8 +1320,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable /* && - pDbVgVersion->stateTs == pDb->stateTs */) { + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && + pDbVgVersion->stateTs == pDb->stateTs) { mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 637a3818e6..b0af98b933 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -303,8 +303,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ac05598bdc..db878d72b9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -938,11 +938,15 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - mInfo("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pInfo->ahandle); if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } + if (i != 0 && code == 0) { + code = TSDB_CODE_RPC_REDIRECT; + } + mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a5257b32c0..ba9d68ee7f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -110,7 +110,6 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { #define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // SRowIter void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); @@ -210,11 +209,10 @@ void tsdbRefMemTable(SMemTable *pMemTable); void tsdbUnrefMemTable(SMemTable *pMemTable); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter -int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); -void *tsdbTbDataIterDestroy(STbDataIter *pIter); -void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter); -bool tsdbTbDataIterNext(STbDataIter *pIter); +int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); +void *tsdbTbDataIterDestroy(STbDataIter *pIter); +void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); +bool tsdbTbDataIterNext(STbDataIter *pIter); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== @@ -772,6 +770,40 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +static FORCE_INLINE int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = tGetI64(p, &pRow->version); + pRow->pTSRow = (STSRow *)(p + n); + n += pRow->pTSRow->len; + return n; +} + +static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { + if (pIter == NULL) return NULL; + + if (pIter->pRow) { + return pIter->pRow; + } + + if (pIter->backward) { + if (pIter->pNode == pIter->pTbData->sl.pHead) { + return NULL; + } + } else { + if (pIter->pNode == pIter->pTbData->sl.pTail) { + return NULL; + } + } + + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + pIter->pRow = &pIter->row; + + return pIter->pRow; +} + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a6e8767a4d..48c14bc758 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -27,9 +27,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->completed = 1; pRetrieve->numOfRows = htonl(pBlock->info.rows); - // TODO enable compress - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 76236e5078..291a5ab1eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -475,7 +475,7 @@ _err: } return code; } - +/* static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { int32_t code = 0; SArray *pDelIdxArray = NULL; @@ -499,7 +499,7 @@ _err: } return code; } - +*/ typedef enum { SFSLASTNEXTROW_FS, SFSLASTNEXTROW_FILESET, @@ -997,8 +997,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); - SDelIdx delIdx; - SDelFile *pDelFile = pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader *pDelFReader; @@ -1006,18 +1004,20 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; - code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); + SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); + + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); + if (code) goto _err; + + SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + + code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); if (code) { tsdbDelFReaderClose(&pDelFReader); goto _err; } + taosArrayDestroy(pDelIdxArray); tsdbDelFReaderClose(&pDelFReader); } else { code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index c663e2b526..52c7a07c49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -294,31 +294,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { - // we add here for commit usage - if (pIter == NULL) return NULL; - - if (pIter->pRow) { - goto _exit; - } - - if (pIter->backward) { - if (pIter->pNode == pIter->pTbData->sl.pHead) { - goto _exit; - } - } else { - if (pIter->pNode == pIter->pTbData->sl.pTail) { - goto _exit; - } - } - - tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); - pIter->pRow = &pIter->row; - -_exit: - return pIter->pRow; -} - static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9dab1e1d33..c157faecb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -973,7 +973,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int8_t* pts = (int8_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int8_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } @@ -998,7 +998,7 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo int32_t mid = dumpedRows >> 1u; int32_t* pts = (int32_t*)pColData->pData; for (int32_t j = 0; j < mid; ++j) { - int64_t t = pts[j]; + int32_t t = pts[j]; pts[j] = pts[dumpedRows - j - 1]; pts[dumpedRows - j - 1] = t; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 37367bd838..a40b1667c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -575,16 +575,6 @@ int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { return n; } -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { - int32_t n = 0; - - n += tGetI64(p, &pRow->version); - pRow->pTSRow = (STSRow *)(p + n); - n += pRow->pTSRow->len; - - return n; -} - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } @@ -1053,7 +1043,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS tRowIterInit(&rIter, pRow, pTSchema); pColVal = tRowIterNext(&rIter); for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; while (pColVal && pColVal->cid < pColData->cid) { pColVal = tRowIterNext(&rIter); diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 9fe37505ad..782ffd788d 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,6 +37,12 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); +static tsem_t canCommit = {0}; + +static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; +void vnode_wait_commit() { tsem_wait(&canCommit); } +void vnode_done_commit() { tsem_wait(&canCommit); } + int vnodeInit(int nthreads) { int8_t init; int ret; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1f87066c82..2dcd681205 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -715,10 +715,10 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* pStateTs) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId || NULL == tableNum || NULL == pStateTs) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b601865306..ff3bd4d33c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1998,6 +1998,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + pInfo->stateTs = dbCache->vgCache.vgInfo->stateTs; ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 51807a145a..e99beca69a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1231,14 +1231,16 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + } return TSDB_CODE_SUCCESS; @@ -1563,7 +1565,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1579,15 +1581,15 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SDBVgInfo *vgInfo = vgCache->vgInfo; if (dbInfo->vgVersion < vgInfo->vgVersion) { - ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgDebug("db updateVgroup is ignored, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); goto _return; } - if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { - ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, - dbInfo->numOfTable); + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable && dbInfo->stateTs == vgInfo->stateTs) { + ctgDebug("no new db vgroup update info, dbFName:%s, vgVer:%d, numOfTable:%d, stateTs:%" PRId64, dbFName, dbInfo->vgVersion, + dbInfo->numOfTable, dbInfo->stateTs); ctgWUnlockVgInfo(dbCache); goto _return; @@ -1599,15 +1601,17 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + } _return: @@ -2170,7 +2174,6 @@ void *ctgUpdateThreadFunc(void *param) { CTG_RT_STAT_INC(numOfOpDequeue, 1); ctgdShowCacheInfo(); - ctgdShowClusterCache(pCtg); } qInfo("catalog update thread stopped"); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 26b5903bb0..b6ff1c8b98 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -78,7 +78,7 @@ void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { num = taosArrayGetSize(pResult->pDbInfo); for (int32_t i = 0; i < num; ++i) { SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i); - qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); + qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, stateTs:%" PRId64 " dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->stateTs, pDb->dbId); } } else { qDebug("empty db info"); @@ -462,6 +462,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { int32_t hashMethod = -1; int16_t hashPrefix = 0; int16_t hashSuffix = 0; + int64_t stateTs = 0; int32_t vgNum = 0; if (dbCache->vgCache.vgInfo) { @@ -469,16 +470,35 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { hashMethod = dbCache->vgCache.vgInfo->hashMethod; hashPrefix = dbCache->vgCache.vgInfo->hashPrefix; hashSuffix = dbCache->vgCache.vgInfo->hashSuffix; + stateTs = dbCache->vgCache.vgInfo->stateTs; if (dbCache->vgCache.vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } ctgDebug("[%d] db [%.*s][0x%" PRIx64 - "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, + "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs, hashMethod, hashPrefix, hashSuffix, vgNum); + if (dbCache->vgCache.vgInfo) { + int32_t i = 0; + void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL); + while (pVgIter) { + SVgroupInfo * pVg = (SVgroupInfo *)pVgIter; + + ctgDebug("The %04dth VG [id:%d, hashBegin:%u, hashEnd:%u, numOfTable:%d, epNum:%d, inUse:%d]", + i++, pVg->vgId, pVg->hashBegin, pVg->hashEnd, pVg->numOfTable, pVg->epSet.numOfEps, pVg->epSet.inUse); + + for (int32_t n = 0; n < pVg->epSet.numOfEps; ++n) { + SEp *pEp = &pVg->epSet.eps[n]; + ctgDebug("\tEp %d [fqdn:%s, port:%d]", n, pEp->fqdn, pEp->port); + } + + pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter); + } + } + pIter = taosHashIterate(dbHash, pIter); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebf7c7baeb..5e543384ac 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2486,7 +2486,8 @@ TEST(dbVgroup, getSetDbVgroupCase) { int32_t dbVer = 0; int64_t dbId = 0; int32_t tbNum = 0; - code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum); + int64_t stateTs = 0; + code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum, &stateTs); ASSERT_EQ(code, 0); ASSERT_EQ(dbVer, ctgTestVgVersion); ASSERT_EQ(dbId, ctgTestDbId); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 76c84b3be9..1c2d7e1f66 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -39,8 +39,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 80a524496c..915dc08c14 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1610,8 +1610,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - int32_t len = 0; - blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ced668cf37..62146b6048 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -842,10 +842,8 @@ typedef struct SJoinOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -void doDestroyExchangeOperatorInfo(void* param); - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain); +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -881,7 +879,11 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); -void doSetOperatorCompleted(SOperatorInfo* pOperator); +void doDestroyExchangeOperatorInfo(void* param); + +void setOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, + SExecTaskInfo* pTaskInfo); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo * pCache); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 76866eedb7..6f9084ce52 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -93,16 +93,11 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - pOperator->name = "LastrowScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, NULL, destroyLastrowScanOperator, NULL); + createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL); pOperator->cost.openCost = 0; return pOperator; @@ -126,7 +121,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); if (size == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -182,7 +177,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes += 1; return pRes; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } else { @@ -234,7 +229,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index bc4ab9c468..3d9757c96f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c new file mode 100644 index 0000000000..049de727df --- /dev/null +++ b/source/libs/executor/src/exchangeoperator.c @@ -0,0 +1,638 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" +#include "tref.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "tsort.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef struct SFetchRspHandleWrapper { + uint32_t exchangeId; + int32_t sourceIndex; +} SFetchRspHandleWrapper; + +static void destroyExchangeOperatorInfo(void* param); +static void freeBlock(void* pParam); +static void freeSourceDataInfo(void* param); +static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs); + +static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); +static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); +static int32_t getCompletedSources(const SArray* pArray); +static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); +static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); + +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { + int32_t code = 0; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); + int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (completed == totalSources) { + setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); + return; + } + + while (1) { + tsem_wait(&pExchangeInfo->ready); + + for (int32_t i = 0; i < totalSources; ++i) { + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { + continue; + } + + if (pDataInfo->status != EX_SOURCE_DATA_READY) { + continue; + } + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); + + // todo + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, + pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + taosMemoryFreeClear(pDataInfo->pRsp); + break; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); + + if (pRsp->completed == 1) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + ", total:%.2f Kb, try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + i + 1, totalSources); + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + } + + taosMemoryFreeClear(pDataInfo->pRsp); + + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { + pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + } + return; + } // end loop + + int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (complete1 == totalSources) { + qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); + return; + } + } + + _error: + pTaskInfo->code = code; +} + +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pOperator->status == OP_EXEC_DONE) { + qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + return NULL; + } + + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } + } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); +} + +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pBlock->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + setOperatorCompleted(pOperator); + return NULL; + } else { + return pBlock; + } + } + } else { + return pBlock; + } + } +} + +static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = EX_SOURCE_DATA_NOT_READY; + dataInfo.taskId = id; + dataInfo.index = i; + SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + if (pDs == NULL) { + taosArrayDestroy(pInfo->pSourceDataInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + + if (numOfSources == 0) { + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); + return TSDB_CODE_INVALID_PARA; + } + + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); + taosArrayPush(pInfo->pSources, pNode); + } + + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); + pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); + + return initDataSource(numOfSources, pInfo, id); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); + + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + + setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + doDestroyExchangeOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + taosRemoveRef(exchangeObjRefPool, pExInfo->self); +} + +void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + +void freeSourceDataInfo(void* p) { + SSourceDataInfo* pInfo = (SSourceDataInfo*)p; + taosMemoryFreeClear(pInfo->pRsp); +} + +void doDestroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + + taosArrayDestroy(pExInfo->pSources); + taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; + } + + blockDataDestroy(pExInfo->pDummyBlock); + + tsem_destroy(&pExInfo->ready); + taosMemoryFreeClear(param); +} + +int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { + SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; + + SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); + if (pExchangeInfo == NULL) { + qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); + return TSDB_CODE_SUCCESS; + } + + int32_t index = pWrapper->sourceIndex; + SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + + if (code == TSDB_CODE_SUCCESS) { + pSourceDataInfo->pRsp = pMsg->pData; + + SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->compLen = htonl(pRsp->compLen); + pRsp->numOfCols = htonl(pRsp->numOfCols); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); + + ASSERT(pRsp != NULL); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); + } else { + taosMemoryFree(pMsg->pData); + pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + } + + pSourceDataInfo->status = EX_SOURCE_DATA_READY; + + tsem_post(&pExchangeInfo->ready); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + + return TSDB_CODE_SUCCESS; +} + +int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + + SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); + pWrapper->exchangeId = pExchangeInfo->self; + pWrapper->sourceIndex = sourceIndex; + + if (pSource->localExec) { + SDataBuf pBuf = {0}; + int32_t code = + (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + loadRemoteDataCallback(pWrapper, &pBuf, code); + taosMemoryFree(pWrapper); + } else { + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, + pSource->execId, sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pMsg->execId = htonl(pSource->execId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = pSource->fetchMsgType; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { + if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + *pNextStart = (char*)blockDecode(pRes, pData); + } else { // extract data according to pColList + char* pStart = pData; + + int32_t numOfCols = htonl(*(int32_t*)pStart); + pStart += sizeof(int32_t); + + // todo refactor:extract method + SSysTableSchema* pSchema = (SSysTableSchema*)pStart; + for (int32_t i = 0; i < numOfCols; ++i) { + SSysTableSchema* p = (SSysTableSchema*)pStart; + + p->colId = htons(p->colId); + p->bytes = htonl(p->bytes); + pStart += sizeof(SSysTableSchema); + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); + } + + blockDecode(pBlock, pStart); + blockDataEnsureCapacity(pRes, pBlock->info.rows); + + // data from mnode + pRes->info.rows = pBlock->info.rows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + blockDataDestroy(pBlock); + } + + // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. + blockDataUpdateTsWindow(pRes, 0); + return TSDB_CODE_SUCCESS; +} + +void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int64_t el = taosGetTimestampUs() - startTs; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + + pLoadInfo->totalElapsed += el; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + + setOperatorCompleted(pOperator); + return NULL; +} + +int32_t getCompletedSources(const SArray* pArray) { + size_t total = taosArrayGetSize(pArray); + + int32_t completed = 0; + for (int32_t k = 0; k < total; ++k) { + SSourceDataInfo* p = taosArrayGet(pArray, k); + if (p->status == EX_SOURCE_DATA_EXHAUSTED) { + completed += 1; + } + } + + return completed; +} + +int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for (int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return code; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), + totalSources, (endTs - startTs) / 1000.0); + + pOperator->status = OP_RES_TO_RETURN; + pOperator->cost.openCost = taosGetTimestampUs() - startTs; + + tsem_wait(&pExchangeInfo->ready); + tsem_post(&pExchangeInfo->ready); + return TSDB_CODE_SUCCESS; +} + +int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + while (1) { + if (pExchangeInfo->current >= totalSources) { + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; + } + + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); + tsem_wait(&pExchangeInfo->ready); + + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), + pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + pOperator->pTaskInfo->code = pDataInfo->code; + return pOperator->pTaskInfo->code; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, + pDataInfo->totalRows, pLoadInfo->totalRows); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + taosMemoryFreeClear(pDataInfo->pRsp); + continue; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); + + if (pRsp->completed == 1) { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, + totalSources); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pLoadInfo->totalRows, pLoadInfo->totalSize); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + + taosMemoryFreeClear(pDataInfo->pRsp); + return TSDB_CODE_SUCCESS; + } +} + +int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + int64_t st = taosGetTimestampUs(); + + SExchangeInfo* pExchangeInfo = pOperator->info; + if (!pExchangeInfo->seqLoadData) { + int32_t code = prepareConcurrentlyLoad(pOperator); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pExchangeInfo->openedTs = taosGetTimestampUs(); + } + + OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1cf01c8661..1aa9a3c613 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1106,3 +1106,24 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } return 0; } + +void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; + assert(pMsg->info.ahandle != NULL); + + SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; + + if (pMsg->contLen > 0) { + buf.pData = taosMemoryCalloc(1, pMsg->contLen); + if (buf.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(buf.pData, pMsg->pCont, pMsg->contLen); + } + } + + pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); + rpcFreeCont(pMsg->pCont); + destroySendMsgInfo(pSendInfo); +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7d662f8784..709e981a1f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -32,7 +32,6 @@ #include "index.h" #include "query.h" #include "tcompare.h" -#include "tcompression.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" @@ -91,11 +90,11 @@ static void destroySortOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param); static void destroyIntervalOperatorInfo(void* param); -static void destroyExchangeOperatorInfo(void* param); + static void destroyOperatorInfo(SOperatorInfo* pOperator); -void doSetOperatorCompleted(SOperatorInfo* pOperator) { +void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); @@ -103,14 +102,24 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo) { + pOperator->name = (char*)name; + pOperator->operatorType = type; + pOperator->blocking = blocking; + pOperator->status = status; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; +} + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, @@ -1652,596 +1661,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t return TSDB_CODE_SUCCESS; } -typedef struct SFetchRspHandleWrapper { - uint32_t exchangeId; - int32_t sourceIndex; -} SFetchRspHandleWrapper; - -int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { - SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; - - SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); - if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; - } - - int32_t index = pWrapper->sourceIndex; - SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - - if (code == TSDB_CODE_SUCCESS) { - pSourceDataInfo->pRsp = pMsg->pData; - - SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->compLen = htonl(pRsp->compLen); - pRsp->numOfCols = htonl(pRsp->numOfCols); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - - ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); - } else { - taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); - } - - pSourceDataInfo->status = EX_SOURCE_DATA_READY; - - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - - return TSDB_CODE_SUCCESS; -} - -void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); - - SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; - - if (pMsg->contLen > 0) { - buf.pData = taosMemoryCalloc(1, pMsg->contLen); - if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - memcpy(buf.pData, pMsg->pCont, pMsg->contLen); - } - } - - pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); - rpcFreeCont(pMsg->pCont); - destroySendMsgInfo(pSendInfo); -} - -static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); - - ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - - SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); - pWrapper->exchangeId = pExchangeInfo->self; - pWrapper->sourceIndex = sourceIndex; - - if (pSource->localExec) { - SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); - loadRemoteDataCallback(pWrapper, &pBuf, code); - taosMemoryFree(pWrapper); - } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - taosMemoryFree(pWrapper); - return pTaskInfo->code; - } - - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); - taosMemoryFree(pWrapper); - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - - pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = pSource->fetchMsgType; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator) { - pInfo->totalRows += numOfRows; - pInfo->totalSize += dataLen; - pInfo->totalElapsed += (taosGetTimestampUs() - startTs); - pOperator->resultInfo.totalRows += numOfRows; -} - -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { - if (pColList == NULL) { // data from other sources - blockDataCleanup(pRes); - *pNextStart = (char*)blockDecode(pRes, pData); - } else { // extract data according to pColList - char* pStart = pData; - - int32_t numOfCols = htonl(*(int32_t*)pStart); - pStart += sizeof(int32_t); - - // todo refactor:extract method - SSysTableSchema* pSchema = (SSysTableSchema*)pStart; - for (int32_t i = 0; i < numOfCols; ++i) { - SSysTableSchema* p = (SSysTableSchema*)pStart; - - p->colId = htons(p->colId); - p->bytes = htonl(p->bytes); - pStart += sizeof(SSysTableSchema); - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); - blockDataAppendColInfo(pBlock, &idata); - } - - blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, pBlock->info.rows); - - // data from mnode - pRes->info.rows = pBlock->info.rows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - blockDataDestroy(pBlock); - } - - // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. - blockDataUpdateTsWindow(pRes, 0); - return TSDB_CODE_SUCCESS; -} - -static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int64_t el = taosGetTimestampUs() - startTs; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - - pLoadInfo->totalElapsed += el; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - - doSetOperatorCompleted(pOperator); - return NULL; -} - - -static int32_t getCompletedSources(const SArray* pArray) { - size_t total = taosArrayGetSize(pArray); - - int32_t completed = 0; - for (int32_t k = 0; k < total; ++k) { - SSourceDataInfo* p = taosArrayGet(pArray, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - return completed; -} - -static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { - int32_t code = 0; - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); - int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (completed == totalSources) { - setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); - return; - } - - while (1) { - tsem_wait(&pExchangeInfo->ready); - - for (int32_t i = 0; i < totalSources; ++i) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { - continue; - } - - if (pDataInfo->status != EX_SOURCE_DATA_READY) { - continue; - } - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - code = pDataInfo->code; - goto _error; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - - // todo - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); - taosMemoryFreeClear(pDataInfo->pRsp); - break; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); - - if (pRsp->completed == 1) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - i + 1, totalSources); - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); - } - - taosMemoryFreeClear(pDataInfo->pRsp); - - if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { - pDataInfo->status = EX_SOURCE_DATA_NOT_READY; - code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - } - return; - } // end loop - - int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); - if (complete1 == totalSources) { - qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); - return; - } - } - -_error: - pTaskInfo->code = code; -} - -static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return code; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), - totalSources, (endTs - startTs) / 1000.0); - - pOperator->status = OP_RES_TO_RETURN; - pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); - tsem_post(&pExchangeInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - while (1) { - if (pExchangeInfo->current >= totalSources) { - setAllSourcesCompleted(pOperator, startTs); - return TSDB_CODE_SUCCESS; - } - - doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); - tsem_wait(&pExchangeInfo->ready); - - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); - pOperator->pTaskInfo->code = pDataInfo->code; - return pOperator->pTaskInfo->code; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - taosMemoryFreeClear(pDataInfo->pRsp); - continue; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - - if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); - pDataInfo->totalRows += pRetrieveRsp->numOfRows; - - taosMemoryFreeClear(pDataInfo->pRsp); - return TSDB_CODE_SUCCESS; - } -} - -static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - - SExchangeInfo* pExchangeInfo = pOperator->info; - if (!pExchangeInfo->seqLoadData) { - int32_t code = prepareConcurrentlyLoad(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - pExchangeInfo->openedTs = taosGetTimestampUs(); - } - - OPTR_SET_OPENED(pOperator); - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return TSDB_CODE_SUCCESS; -} - -static void freeBlock(void* pParam) { - SSDataBlock* pBlock = *(SSDataBlock**)pParam; - blockDataDestroy(pBlock); -} - -static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pOperator->status == OP_EXEC_DONE) { - qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - return NULL; - } - - size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); - if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { - pExchangeInfo->rspBlockIndex = 0; - taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); - if (pExchangeInfo->seqLoadData) { - seqLoadRemoteData(pOperator); - } else { - concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } - - if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { - return NULL; - } - } - - // we have buffered retrieved datablock, return it directly - return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); -} - -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); - if (pBlock == NULL) { - return NULL; - } - - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; - if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); - if (status == PROJECT_RETRIEVE_CONTINUE) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pBlock->info.rows; - pExchangeInfo->limitInfo.numOfOutputRows += rows; - - if (rows == 0) { - doSetOperatorCompleted(pOperator); - return NULL; - } else { - return pBlock; - } - } - } else { - return pBlock; - } - } -} - -static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SSourceDataInfo dataInfo = {0}; - dataInfo.status = EX_SOURCE_DATA_NOT_READY; - dataInfo.taskId = id; - dataInfo.index = i; - SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); - if (pDs == NULL) { - taosArrayDestroy(pInfo->pSourceDataInfo); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { - size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); - - if (numOfSources == 0) { - qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); - return TSDB_CODE_INVALID_PARA; - } - - pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - if (pInfo->pSources == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); - taosArrayPush(pInfo->pSources, pNode); - } - - initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); - pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); - - return initDataSource(numOfSources, pInfo, id); -} - -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tsem_init(&pInfo->ready, 0, 0); - pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); - - pInfo->seqLoadData = false; - pInfo->pTransporter = pTransporter; - - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - doDestroyExchangeOperatorInfo(pInfo); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); @@ -2386,7 +1805,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2396,7 +1815,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2645,7 +2064,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2722,7 +2141,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { while (true) { fillResult = doFillImpl(pOperator); if (fillResult == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2952,15 +2371,10 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupId = UINT64_MAX; - pOperator->name = "TableAggregate"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_AGG; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, NULL); + createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -3024,33 +2438,6 @@ void destroyFillOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void destroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - taosRemoveRef(exchangeObjRefPool, pExInfo->self); -} - -void freeSourceDataInfo(void* p) { - SSourceDataInfo* pInfo = (SSourceDataInfo*)p; - taosMemoryFreeClear(pInfo->pRsp); -} - -void doDestroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - - taosArrayDestroy(pExInfo->pSources); - taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); - - if (pExInfo->pResultBlockList != NULL) { - taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); - pExInfo->pResultBlockList = NULL; - } - - blockDataDestroy(pExInfo->pDummyBlock); - - tsem_destroy(&pExInfo->ready); - taosMemoryFreeClear(param); -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { @@ -3182,15 +2569,9 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pOperator->name = "FillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; + setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 2a7d1b089b..47ee663abe 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -316,7 +316,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -438,15 +438,10 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -654,7 +649,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { // try next group data ++pInfo->groupIndex; if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); clearPartitionOperator(pInfo); return NULL; } @@ -821,17 +816,12 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; + setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -909,7 +899,10 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { void* pData = colDataGetVarData(pCol, 0); // TODO check tbname validity if (pData != (void*)-1) { - memcpy(pDest->info.parTbName, varDataVal(pData), varDataLen(pData)); + memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + memcpy(pDest->info.parTbName, varDataVal(pData), len); + /*pDest->info.parTbName[len + 1] = 0;*/ } else { pDest->info.parTbName[0] = 0; } @@ -965,7 +958,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->pInputDataBlock = NULL; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } printDataBlock(pBlock, "stream partitionby recv"); @@ -1106,15 +1099,10 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; + setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 45d76dce74..4e1daac643 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -73,14 +73,10 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + + setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); @@ -121,8 +117,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,13 +367,13 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) if (leftTs == rightTs) { mergeJoinJoinDownstreamTsRanges(pOperator, leftTs, pRes, &nrows); - } else if (asc && leftTs < rightTs || !asc && leftTs > rightTs) { + } else if ((asc && leftTs < rightTs) || (!asc && leftTs > rightTs)) { pJoinInfo->leftPos += 1; if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { continue; } - } else if (asc && leftTs > rightTs || !asc && leftTs < rightTs) { + } else if ((asc && leftTs > rightTs) || (!asc && leftTs < rightTs)) { pJoinInfo->rightPos += 1; if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { continue; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index b2865d15f0..ce1d13775c 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -98,13 +98,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -153,7 +149,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return PROJECT_RETRIEVE_DONE; } @@ -187,7 +183,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS // TODO: optimize it later when partition by + limit if ((pLimitInfo->slimit.limit == -1 && pLimitInfo->currentGroupId == 0) || (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -252,7 +248,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { @@ -400,14 +396,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - pOperator->name = "IndefinitOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, - destroyIndefinitOperatorInfo, NULL); + setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -499,7 +489,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -628,7 +618,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7faf75d5cd..b83fac9fa6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,7 +284,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* } static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsAgg, int32_t numOfCols, - int32_t numOfRows) { + int32_t numOfRows) { if (pColsAgg == NULL || pFilterInfo == NULL) { return true; } @@ -345,7 +345,7 @@ static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlo // todo handle the slimit info void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator) { - SLimit* pLimit = &pLimitInfo->limit; + SLimit* pLimit = &pLimitInfo->limit; const char* id = GET_TASKID(pTaskInfo); if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) { @@ -499,7 +499,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction typedef struct STableCachedVal { const char* pName; - STag* pTags; + STag* pTags; } STableCachedVal; static void freeTableCachedVal(void* param) { @@ -513,13 +513,11 @@ static void freeTableCachedVal(void* param) { taosMemoryFree(pVal); } -//const void *key, size_t keyLen, void *value -static void freeCachedMetaItem(const void *key, size_t keyLen, void *value) { - freeTableCachedVal(value); -} +// const void *key, size_t keyLen, void *value +static void freeCachedMetaItem(const void* key, size_t keyLen, void* value) { freeTableCachedVal(value); } -int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) { +int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, + int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) { // currently only the tbname pseudo column if (numOfExpr <= 0) { return TSDB_CODE_SUCCESS; @@ -531,11 +529,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; - bool freeReader = false; + bool freeReader = false; STableCachedVal val = {0}; SMetaReader mr = {0}; - LRUHandle* h = NULL; + LRUHandle* h = NULL; // 1. check if it is existed in meta cache if (pCache == NULL) { @@ -582,7 +580,8 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int val = *pVal; freeReader = true; - int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); + int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, + sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); if (ret != TAOS_LRU_STATUS_OK) { qError("failed to put meta into lru cache, code:%d, %s", ret, idStr); freeTableCachedVal(pVal); @@ -594,13 +593,13 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false); } - qDebug("retrieve table meta from cache:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, - (pCache->metaFetch - pCache->cacheHit), idStr); + qDebug("retrieve table meta from cache:%" PRIu64 ", hit:%" PRIu64 " miss:%" PRIu64 ", %s", pCache->metaFetch, + pCache->cacheHit, (pCache->metaFetch - pCache->cacheHit), idStr); } for (int32_t j = 0; j < numOfExpr; ++j) { const SExprInfo* pExpr1 = &pExpr[j]; - int32_t dstSlotId = pExpr1->base.resSchema.slotId; + int32_t dstSlotId = pExpr1->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); colInfoDataCleanup(pColInfoData, pBlock->info.rows); @@ -652,7 +651,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, fmGetScalarFuncExecFuncs(functionId, &fpSet); size_t len = TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE; - char buf[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char buf[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(buf, name) SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, len, 1); @@ -820,7 +819,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -843,7 +842,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -865,7 +864,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -904,12 +903,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } - SScanPhysiNode* pScanNode = &pTableScanNode->scan; + SScanPhysiNode* pScanNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, - &pInfo->matchInfo); + int32_t code = + extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -947,22 +946,17 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; - pOperator->name = "TableScanOperator"; // for debug purpose - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024*128, -1, .5); + pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->metaCache.pTableMetaEntryCache == NULL) { code = terrno; goto _error; } taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 @@ -986,14 +980,8 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL); + setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1049,8 +1037,8 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, - GET_TASKID(pTaskInfo)); + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -1148,15 +1136,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - pOperator->name = "DataBlockDistScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL); + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; _error: @@ -1599,8 +1580,10 @@ static void calBlockTbName(SExprSupp* pTbNameCalSup, SSDataBlock* pBlock) { void* pData = colDataGetData(pCol, 0); // TODO check tbname validation if (pData != (void*)-1 && pData != NULL) { - memcpy(pBlock->info.parTbName, varDataVal(pData), TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN)); - pBlock->info.parTbName[TSDB_TABLE_NAME_LEN - 1] = 0; + memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + memcpy(pBlock->info.parTbName, varDataVal(pData), len); + /*pBlock->info.parTbName[len + 1] = 0;*/ } else { pBlock->info.parTbName[0] = 0; } @@ -2368,11 +2351,9 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; - pOperator->name = "RawScanOperator"; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; _end: @@ -2556,16 +2537,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->partitionSup.needCalc = false; - pOperator->name = "StreamScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); return pOperator; @@ -2900,7 +2876,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2953,7 +2929,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3743,7 +3719,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } if (i >= taosArrayGetSize(pIdx->uids)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } else { pIdx->lastIdx = i + 1; } @@ -3925,7 +3901,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3947,7 +3923,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { if (pInfo->showRewrite == false) { @@ -4199,15 +4175,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; _error: @@ -4283,7 +4253,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { count += 1; if (++pInfo->curPos >= size) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -4319,7 +4289,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi int32_t numOfExprs = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4335,18 +4305,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); return pOperator; @@ -4455,7 +4418,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc T_LONG_JMP(pTaskInfo->env, code); } - if (pOperator->exprSupp.pFilterInfo!= NULL) { + if (pOperator->exprSupp.pFilterInfo != NULL) { int64_t st = taosGetTimestampMs(); doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo); @@ -4713,7 +4676,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; if (tableListSize == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->tableStartIndex = 0; @@ -4732,7 +4695,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } else { stopGroupTableMergeScan(pOperator); if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; @@ -4831,7 +4794,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->sample.seed = taosGetTimestampSec(); pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; - code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4853,15 +4815,10 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - pOperator->name = "TableMergeScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index af87c4b2eb..fc53623d44 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -53,11 +53,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); - pOperator->name = "SortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; @@ -67,7 +63,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -214,7 +210,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -428,7 +424,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); if (pInfo->prefetchedSortInput == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; @@ -453,7 +449,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { beginSortGroup(pOperator); } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { finishSortGroup(pOperator); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -509,15 +505,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort } pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - - pOperator->name = "GroupSortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); @@ -705,7 +694,7 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } return pBlock; @@ -774,14 +763,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->bufPageSize = getProperSortPageSize(rowSize); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. - pOperator->name = "MultiwayMerge"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 9f06e639b3..7c9d73ad13 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1443,7 +1443,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRes, "stream fill"); return pInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1512,7 +1512,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } if (pInfo->pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1690,15 +1690,9 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->srcRowIndex = 0; - - pOperator->name = "StreamFillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index e5b49acf0b..f0310a45d8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1221,7 +1221,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1232,7 +1232,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1269,7 +1269,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1739,7 +1739,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pOperator->pTaskInfo = pTaskInfo; pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -1777,15 +1776,10 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1890,7 +1884,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1933,7 +1927,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2281,7 +2275,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2330,7 +2324,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2342,7 +2336,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2365,7 +2359,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2386,7 +2380,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } else { @@ -2448,7 +2442,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2463,7 +2457,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2557,15 +2551,9 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pScanInfo->cond.twindows = pInfo->win; pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; - pOperator->name = "TimeSliceOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -2633,15 +2621,10 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2711,14 +2694,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW goto _error; } - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3134,7 +3112,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -3403,7 +3381,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -4027,7 +4005,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4133,7 +4111,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4200,13 +4178,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); @@ -4257,7 +4233,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -4336,7 +4312,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4347,20 +4323,21 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pOperator == NULL) { goto _error; } + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pInfo->isFinal = true; - pOperator->name = "StreamSessionFinalAggOperator"; - } else { - pInfo->isFinal = false; + pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); + char* name = (pInfo->isFinal)? "StreamSessionFinalAggOperator":"StreamSessionSemiAggOperator"; + + if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->name = "StreamSessionSemiAggOperator"; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } + setOperatorInfo(pOperator, name, pPhyNode->type , false, OP_NOT_OPENED, pInfo, pTaskInfo); + pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); @@ -4590,7 +4567,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4656,7 +4633,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4721,14 +4698,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamStateAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); @@ -4876,7 +4848,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -5001,16 +4973,10 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5254,7 +5220,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } size_t rows = pRes->info.rows; @@ -5313,16 +5279,9 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge } initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - - pOperator->name = "TimeMergeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; - + setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5366,7 +5325,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, &pInfo->delKey); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); return NULL; } @@ -5550,12 +5509,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamIntervalOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, + setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6eb57c1a18..079e553b07 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3096,27 +3096,86 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } #else int64_t* pts = (int64_t*)pInput->pPTS->pData; - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; - } - numOfElems++; +#if 0 + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } - char* data = colDataGetData(pInputCol, i); - TSKEY cts = pts[i]; - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; + numOfElems++; + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } } - } +#else + if (!pInputCol->hasNull) { + numOfElems = 1; + + int32_t round = pInput->numOfRows >> 2; + int32_t reminder = pInput->numOfRows & 0x03; + + int32_t tick = 0; + for (int32_t i = pInput->startRowIndex; tick < round; i += 4, tick += 1) { + int64_t cts = pts[i]; + int32_t chosen = i; + + if (cts < pts[i + 1]) { + cts = pts[i + 1]; + chosen = i + 1; + } + + if (cts < pts[i + 2]) { + cts = pts[i + 2]; + chosen = i + 2; + } + + if (cts < pts[i + 3]) { + cts = pts[i + 3]; + chosen = i + 3; + } + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetData(pInputCol, chosen); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + + for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } + } +#endif + #endif // save selectivity value for column consisted of all null values if (numOfElems == 0) { firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); } - SET_VAL(pResInfo, numOfElems, 1); + +// SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3266,8 +3325,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. - // this is NOT always works if project operator exists in downstream. + // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. + // this assumption is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 10a86866d5..c53d3f9320 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -108,7 +108,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum); + int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 82170a27d8..05fc651618 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -466,14 +466,14 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SParseContext* pParCxt = pCxt->pParseCxt; int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { - code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum); + code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } else { - code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); + code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } } if (TSDB_CODE_SUCCESS != code) { @@ -1252,6 +1252,19 @@ static bool dataTypeEqual(const SDataType* l, const SDataType* r) { return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); @@ -1876,10 +1889,15 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW } pWhenThen->pWhen = pIsTrue; } - if (first) { - first = false; + if (first || dataTypeComp(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType) < 0) { pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; - } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { + } + first = false; + } + + FOREACH(pNode, pCaseWhen->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pWhenThen->pThen, pCaseWhen->node.resType, &pCastFunc); if (TSDB_CODE_SUCCESS != pCxt->errCode) { @@ -1889,6 +1907,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->node.resType = pCaseWhen->node.resType; } } + if (NULL != pCaseWhen->pElse && !dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pCaseWhen->pElse)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pCaseWhen->pElse, pCaseWhen->node.resType, &pCastFunc); @@ -3431,19 +3450,6 @@ static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { return (SNode*)pCol; } -// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail -static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { - if (l->type != r->type) { - return 1; - } - - if (l->bytes != r->bytes) { - return l->bytes > r->bytes ? 1 : -1; - } - - return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; -} - static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); @@ -4969,7 +4975,8 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); + int32_t code = + getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 466c6edd24..00a72a1946 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -876,13 +876,14 @@ int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCac } int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SDbInfo* pDbInfo = NULL; int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo); if (TSDB_CODE_SUCCESS == code) { *pVersion = pDbInfo->vgVer; *pDbId = pDbInfo->dbId; *pTableNum = pDbInfo->tbNum; + *pStateTs = pDbInfo->stateTs; } return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 4f5ddd9a51..8f051c67a0 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -249,7 +249,7 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con } int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, - int32_t* tableNum) { + int32_t* tableNum, int64_t* stateTs) { return 0; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cf064881c2..6eadf80e3d 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -146,6 +146,12 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { } taosMemoryFreeClear(pMsgBody); } +void destroyAhandle(void *ahandle) { + SMsgSendInfo *pSendInfo = ahandle; + if (pSendInfo == NULL) return; + + destroySendMsgInfo(pSendInfo); +} int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 181abc2247..fadc39f21d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -41,8 +41,9 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix; pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix; + pOut->dbVgroup->stateTs = usedbRsp->stateTs; - qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); + qDebug("Got %d vgroup for db %s, vgVersion:%d, stateTs:%" PRId64, usedbRsp->vgNum, usedbRsp->db, usedbRsp->vgVersion, usedbRsp->stateTs); if (usedbRsp->vgNum <= 0) { return TSDB_CODE_SUCCESS; @@ -103,6 +104,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms usedbReq.vgVersion = pInput->vgVersion; usedbReq.dbId = pInput->dbId; usedbReq.numOfTable = pInput->numOfTable; + usedbReq.stateTs = pInput->stateTs; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 801f38bd1a..22fb66d92f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -425,6 +425,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -438,6 +439,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { code); if (pMsg) { taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); } return TSDB_CODE_SUCCESS; } @@ -492,6 +494,7 @@ _return: tFreeSSchedulerHbRsp(&rsp); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); SCH_RET(code); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index dbdccff302..7e5b3faedb 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -439,6 +439,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(code); @@ -446,6 +448,8 @@ _return: taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -942,7 +946,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1115,7 +1119,7 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index ad342edfa0..573a1ea31f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -118,8 +118,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -200,8 +199,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index dc40e2fc72..0a67939d35 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 0227d832fc..d2dff92d43 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index 7458ce28ab..d3ba556f82 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 9ccd9dd28f..662c17094e 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // RequestVote(i, j) == diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index e8f17537b4..bd88f5cdce 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 19079d518d..936081c7b2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 15ca82664a..6b6ca21399 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define CONFIG_FILE_LEN 2048 diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 8b8ab41b53..2a73b95e4d 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" -#include "tref.h" #include "tskiplist.h" typedef struct SSyncRaftEntry { diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index c25d4ae34e..17ea36c640 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncRaftEntry.h" -#include "taosdef.h" #include "wal.h" typedef struct SSyncLogStoreData { diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index ac298c5375..077e6a7658 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 4f15a45cec..e08430327b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // AppendEntries(i, j) == diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 73b4a0efae..c51ea4eea6 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 6bef18405c..1adab30aaa 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index 9026ecb66e..1d41c0a85f 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" typedef struct SRespStub { SRpcMsg rpcMsg; diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 5594fe46ed..963fedce31 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -20,13 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "syncMessage.h" -#include "taosdef.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 @@ -36,7 +30,6 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 -//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; @@ -65,12 +58,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finis int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); - -int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); - -//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; @@ -83,7 +70,6 @@ typedef struct SSyncSnapshotReceiver { // init when create SSyncNode *pSyncNode; - } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); @@ -93,13 +79,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); - -//--------------------------------------------------- // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode *ths, SyncSnapshotRsp *pMsg); +int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 112a3d8610..3139707d54 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 6a760ecd87..3fb4a5ba0c 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -728,7 +728,6 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); -int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7f241e827d..076101ef43 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -24,17 +24,15 @@ extern "C" { uint64_t syncUtilAddr2U64(const char* host, uint16_t port); void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilEmptyId(const SRaftId* pId); int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); -char* syncUtilRaftId2Str(const SRaftId* p); const char* syncStr(ESyncState state); char* syncUtilPrintBin(char* ptr, uint32_t len); char* syncUtilPrintBin2(char* ptr, uint32_t len); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index d894e91600..066a4dd76f 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -39,8 +39,6 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncN bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted); -char *voteGranted2Str(SVotesGranted *pVotesGranted); typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; @@ -56,8 +54,6 @@ void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSync bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond); -char *votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 252d5d0219..25f6c5e6f1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -757,7 +757,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); goto _error; } @@ -772,7 +772,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { sError("vgId:%d, failed to determine raft member id, peer:%d", pSyncNode->vgId, i); goto _error; } @@ -781,7 +781,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; } @@ -1213,7 +1213,7 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); + syncUtilRaftId2EpSet(destRaftId, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1230,7 +1230,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + syncUtilNodeInfo2EpSet(nodeInfo, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1344,7 +1344,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; @@ -1356,13 +1356,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // update quorum first diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index c5b1399c8c..262a7b0879 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" +#include "tref.h" SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 85c94884f2..d4a1d35c74 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -294,6 +294,12 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return 0; } + // need not truncate + SyncIndex walCommitVer = walGetCommittedVer(pWal); + if (fromIndex <= walCommitVer) { + return 0; + } + int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 78413bbeff..3273718f7d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -13,33 +13,25 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncSnapshot.h" #include "syncIndexMgr.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "wal.h" -//---------------------------------- -static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg); -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); -static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); - -//---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); SSyncSnapshotSender *pSender = NULL; if (condition) { - pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pSender, 0, sizeof(*pSender)); pSender->start = false; pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; @@ -249,64 +241,6 @@ static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnaps ++(pSender->seq); } -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pSender != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pSender->start); - cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); - cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); - cJSON_AddStringToObject(pRoot, "pReader", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); - cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); - - if (pSender->pCurrentBlock != NULL) { - char *s; - s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); - taosMemoryFree(s); - } - - cJSON *pSnapshot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); - cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); - cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); - // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); - return pJson; -} - -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { - cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { sNTrace(pSyncNode, "starting snapshot ..."); @@ -335,16 +269,17 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } -// ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); SSyncSnapshotReceiver *pReceiver = NULL; if (condition) { - pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); - ASSERT(pReceiver != NULL); - memset(pReceiver, 0, sizeof(*pReceiver)); + pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); + if (pReceiver == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pReceiver->start = false; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; @@ -530,63 +465,6 @@ static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapsh } } -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pReceiver != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); - cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); - cJSON_AddStringToObject(pRoot, "pWriter", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - cJSON *pFromId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); - cJSON_AddStringToObject(pFromId, "addr", u64buf); - { - uint64_t u64 = pReceiver->fromId.addr; - cJSON *pTmp = pFromId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); - cJSON_AddItemToObject(pRoot, "fromId", pFromId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); - return pJson; -} - -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { - cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { SyncIndex snapStart = SYNC_INDEX_INVALID; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 30e0617f43..91d807319b 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -13,28 +13,23 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncTimeout.h" #include "syncElection.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncReplication.h" -#include "syncRespMgr.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { int32_t newArrIndex = 0; - SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT]; - memset(newConfigIndexArr, 0, sizeof(newConfigIndexArr)); - + SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0}; SSnapshot snapshot = {0}; - if (ths->pFsm != NULL && ths->pFsm->FpGetSnapshotInfo != NULL) { - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - } - + + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) { - for (int i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { + for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) { // pass - ; } else { // save newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i]; @@ -47,13 +42,15 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); int32_t code = raftCfgPersist(ths->pRaftCfg); - ASSERT(code == 0); - - sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + if (code != 0) { + sNFatal(ths, "failed to persist cfg"); + } else { + sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + } } } -int32_t syncNodeTimerRoutine(SSyncNode* ths) { +static int32_t syncNodeTimerRoutine(SSyncNode* ths) { sNTrace(ths, "timer routines"); // timer replicate @@ -71,7 +68,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); + sNError(ths, "timer wal snapshot end error since:%s", terrstr()); return -1; } else { sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1a00b0f5a4..894aa05aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -41,15 +41,15 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort); } -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { char host[TSDB_FQDN_LEN] = {0}; - uint16_t port; + uint16_t port = 0; syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); pEpSet->inUse = 0; @@ -57,7 +57,7 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { addEpIntoEpSet(pEpSet, host, port); } -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { sError("failed to resolve ipv4 addr, fqdn: %s", pInfo->nodeFqdn); @@ -73,8 +73,7 @@ bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - bool ret = pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; - return ret; + return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; } bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } @@ -90,18 +89,6 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); - cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); - return pJson; -} - cJSON* syncUtilRaftId2Json(const SRaftId* p) { char u64buf[128] = {0}; cJSON* pRoot = cJSON_CreateObject(); @@ -120,13 +107,6 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { return pJson; } -char* syncUtilRaftId2Str(const SRaftId* p) { - cJSON* pJson = syncUtilRaftId2Json(p); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - static inline bool syncUtilCanPrint(char c) { if (c >= 32 && c <= 126) { return true; @@ -165,14 +145,12 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) { } void syncUtilMsgHtoN(void* msg) { - // htonl SMsgHead* pHead = msg; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); } void syncUtilMsgNtoH(void* msg) { - // ntohl SMsgHead* pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index ee1f83ee6a..4ca4e26bec 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,12 +23,11 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryCalloc(1, sizeof(SVotesGranted)); if (pVotesGranted == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); pVotesGranted->replicaNum = pSyncNode->replicaNum; @@ -59,20 +58,24 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode) { pVotesGranted->pSyncNode = pSyncNode; } -bool voteGrantedMajority(SVotesGranted *pVotesGranted) { - bool ret = pVotesGranted->votes >= pVotesGranted->quorum; - return ret; -} +bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->votes >= pVotesGranted->quorum; } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { - ASSERT(pMsg->voteGranted == true); - - if (pMsg->term != pVotesGranted->term) { - sNTrace(pVotesGranted->pSyncNode, "vote grant vnode error"); + if (!pMsg->voteGranted) { + sNFatal(pVotesGranted->pSyncNode, "vote granted should be true"); return; } - ASSERT(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + if (pMsg->term != pVotesGranted->term) { + sNTrace(pVotesGranted->pSyncNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, + pVotesGranted->term, pMsg->term); + return; + } + + if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { + sNFatal(pVotesGranted->pSyncNode, "vote granted raftId not matched with msg"); + return; + } int32_t j = -1; for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -81,14 +84,21 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - ASSERT(j != -1); - ASSERT(j >= 0 && j < pVotesGranted->replicaNum); + if ((j == -1) || !(j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, "invalid msg srcId, index:%d", j); + return; + } if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); pVotesGranted->isGranted[j] = true; } - ASSERT(pVotesGranted->votes <= pVotesGranted->replicaNum); + + if (pVotesGranted->votes > pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + pVotesGranted->replicaNum); + return; + } } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { @@ -97,53 +107,12 @@ void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->toLeader = false; } -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesGranted != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); - } - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - arr[i] = pVotesGranted->isGranted[i]; - } - cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); - - cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); - cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - bool majority = voteGrantedMajority(pVotesGranted); - cJSON_AddNumberToObject(pRoot, "majority", majority); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); - return pJson; -} - -char *voteGranted2Str(SVotesGranted *pVotesGranted) { - cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); - ASSERT(pVotesRespond != NULL); - memset(pVotesRespond, 0, sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryCalloc(1, sizeof(SVotesRespond)); + if (pVotesRespond == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pVotesRespond->replicas = &(pSyncNode->replicasId); pVotesRespond->replicaNum = pSyncNode->replicaNum; @@ -185,62 +154,15 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { - // ASSERT(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } } - ASSERT(0); + + sNFatal(pVotesRespond->pSyncNode, "votes respond not found"); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); - /* - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; - } - */ -} - -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesRespond != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); - } - int32_t respondNum = 0; - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - arr[i] = pVotesRespond->isRespond[i]; - if (pVotesRespond->isRespond[i]) { - respondNum++; - } - } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); - cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); - return pJson; -} - -char *votesRespond2Str(SVotesRespond *pVotesRespond) { - cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; } diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index 98b8654734..947a2cb05c 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -1,11 +1,4 @@ //#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncUtil.h" -#include "trpc.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a5e96233c2..1eb803e846 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 256c13e267..7b9e04a814 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp index ce129015c6..a8a1931d1d 100644 --- a/source/libs/sync/test/syncApplyMsgTest.cpp +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index dcf7249d30..a6d33aa674 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c5548d25d4..cf498933e1 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 5cdbb2cc88..59d731c823 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 216291c1d8..528cc1614d 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" #if 0 void logTest() { diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index bb5f968dc6..54bfb1b387 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 56b3d6da2e..b86422b0b1 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -1,12 +1,3 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" -#include "tskiplist.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index e94755195b..369306b857 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index a220c2aed4..3cd6048428 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -1,9 +1,4 @@ -#include "syncEnv.h" -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "ttime.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 2f5bb07f9a..14a29c9a1e 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -1,12 +1,4 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "tskiplist.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp index 1fac03652b..6f7f4d18f3 100644 --- a/source/libs/sync/test/syncHeartbeatReplyTest.cpp +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp index b0c0554355..c8c38c0c38 100644 --- a/source/libs/sync/test/syncHeartbeatTest.cpp +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index bd0221114a..f66a854657 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" -#include "tdatablock.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index f88e4f240d..a082f9373a 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -101,7 +95,7 @@ int main(int argc, char** argv) { syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); SEpSet epSet; - syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + syncUtilNodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); rpcMsg.info.noResp = 1; pSyncNode->syncSendMSg(&epSet, &rpcMsg); diff --git a/source/libs/sync/test/syncIOServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp index 1d7402e461..add7d6df60 100644 --- a/source/libs/sync/test/syncIOServerTest.cpp +++ b/source/libs/sync/test/syncIOServerTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 9c2342828e..71274f5ca3 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 64b65f25c8..9ef8a7ad40 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 23b8693a0b..f84119045f 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -1,12 +1,4 @@ -#include "syncIndexMgr.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp index 763117c0c9..a44d9c4e31 100644 --- a/source/libs/sync/test/syncIndexTest.cpp +++ b/source/libs/sync/test/syncIndexTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void print(SHashObj *pNextIndex) { printf("----------------\n"); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 2fe5dd7e18..b15a767db5 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLeaderTransferTest.cpp b/source/libs/sync/test/syncLeaderTransferTest.cpp index bd1f22edff..fd8cdfaac0 100644 --- a/source/libs/sync/test/syncLeaderTransferTest.cpp +++ b/source/libs/sync/test/syncLeaderTransferTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index b42626df29..fa0d91d45f 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 0161160a75..e02c484729 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 29ad0610e7..fce0a21a47 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -1,12 +1,4 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 832b42bf80..1b898803ef 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index e5158a7932..661509e9a2 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 781ffc31db..19f377e037 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 1777b9b9e2..54cc506b03 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 5a0b5e9953..ded7e22eac 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 09f56815cd..4f1bb64798 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp index a30dcc2c54..e5175f1101 100644 --- a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp @@ -1,10 +1,6 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotTest.cpp b/source/libs/sync/test/syncPreSnapshotTest.cpp index 03894dfa84..2ce95abea3 100644 --- a/source/libs/sync/test/syncPreSnapshotTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgIndexTest.cpp b/source/libs/sync/test/syncRaftCfgIndexTest.cpp index 522da159b5..e6d3f23b58 100644 --- a/source/libs/sync/test/syncRaftCfgIndexTest.cpp +++ b/source/libs/sync/test/syncRaftCfgIndexTest.cpp @@ -1,10 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index b1f820c78e..c841a68fde 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -1,10 +1,4 @@ -#include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index e7ef69da20..08716c876b 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void usage(char* exe) { printf("Usage: %s host port \n", exe); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp index 278113919a..e309a2e432 100644 --- a/source/libs/sync/test/syncRaftLogTest.cpp +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -1,12 +1,5 @@ #include "syncRaftLog.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 3d50b63ff9..a7752dcb8b 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -1,13 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 070f8c0f07..87798a7d80 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,9 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index c5132018a2..47bc290abe 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "tref.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 7506ae19bd..2419ec0974 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 3d9db17725..973ccf040c 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 94e1504add..275f7804bf 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp index 470dd678b0..44444a7f5e 100644 --- a/source/libs/sync/test/syncRestoreFromSnapshot.cpp +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 127d8e1c41..3f66cd9d1c 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 6eb8548740..49b06a7d1b 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 63905c2182..0a06b512ea 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 83f1dfebb3..46f33e0c56 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 7b2c13463c..9a234d412e 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index e2264fd08c..40eac0b55f 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -1,14 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 97de81572a..55480802da 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" /* typedef enum { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index e6f6006410..b0fe612b7f 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -1,12 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index bae5e0ea88..ed0742f499 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 411915fe22..bbd4fa7d18 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -1,9 +1,4 @@ -#include "syncUtil.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 6a8404308b..2d2a12c656 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -1,11 +1,3 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" void logTest() { diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index d5e7d71030..712e97edc9 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -1,14 +1,6 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" #include "syncTest.h" - void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 2e5c26719b..56dd0a5845 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -1,15 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" #include "syncTest.h" -#include "syncUtil.h" -#include "wal.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h index 3a49074272..a1881bf1b3 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncTest.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -22,8 +22,10 @@ extern "C" { #include "syncInt.h" +#include "tref.h" #include "wal.h" +#include "tref.h" #include "syncEnv.h" #include "syncIO.h" #include "syncIndexMgr.h" @@ -38,6 +40,8 @@ extern "C" { #include "syncUtil.h" #include "syncVoteMgr.h" +extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); + cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntryPrint(const SSyncRaftEntry* pObj); @@ -82,6 +86,18 @@ void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); +cJSON* voteGranted2Json(SVotesGranted* pVotesGranted); +char* voteGranted2Str(SVotesGranted* pVotesGranted); +cJSON* votesRespond2Json(SVotesRespond* pVotesRespond); +char* votesRespond2Str(SVotesRespond* pVotesRespond); + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +char* syncUtilRaftId2Str(const SRaftId* p); + +cJSON* snapshotSender2Json(SSyncSnapshotSender* pSender); +char* snapshotSender2Str(SSyncSnapshotSender* pSender); +cJSON* snapshotReceiver2Json(SSyncSnapshotReceiver* pReceiver); +char* snapshotReceiver2Str(SSyncSnapshotReceiver* pReceiver); #ifdef __cplusplus } diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c new file mode 100644 index 0000000000..fdaf4b8b34 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pFromId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); + cJSON_AddStringToObject(pFromId, "addr", u64buf); + { + uint64_t u64 = pReceiver->fromId.addr; + cJSON *pTmp = pFromId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); + cJSON_AddItemToObject(pRoot, "fromId", pFromId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c new file mode 100644 index 0000000000..b12f4e76a5 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c new file mode 100644 index 0000000000..f1c26a97aa --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesGranted != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesRespond != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); + } + int32_t respondNum = 0; + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index b9167501e2..3b7182f983 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d3db4879d1..feb55985de 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -53,6 +53,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c3abfa971e..457d3e5cb1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -143,6 +143,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); +static void cliReleaseUnfinishedMsg(SCliConn* conn); // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); @@ -163,17 +164,6 @@ static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); -static void cliReleaseUnfinishedMsg(SCliConn* conn) { - for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { - SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); - if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { - if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { - conn->ctx.freeFunc(msg->ctx->ahandle); - } - } - destroyCmsg(msg); - } -} #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -266,6 +256,23 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { static void* cliWorkThread(void* arg); +static void cliReleaseUnfinishedMsg(SCliConn* conn) { + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); + if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { + if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { + conn->ctx.freeFunc(msg->ctx->ahandle); + } else if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); + pTransInst->destroyFp(msg->ctx->ahandle); + } + } + destroyCmsg(msg); + } +} bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1bd2c6f0a0..f7949048ca 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -586,7 +586,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; + char tmpBuf[128] = {0}; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; diff --git a/tests/develop-test/fulltest.sh b/tests/develop-test/fulltest.sh deleted file mode 100644 index e986ed6966..0000000000 --- a/tests/develop-test/fulltest.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e -set -x - -python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/commandline.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/limit_offset_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_interlace.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_telnet_alltypes.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/taosadapter_json.py -#python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task new file mode 100644 index 0000000000..ad8444f795 --- /dev/null +++ b/tests/parallel_test/cases.task @@ -0,0 +1,1014 @@ +#Coulumn Define +#caseID,rerunTimes,Run with Sanitizer,casePath,caseCommand +#NA,NA,y or n,script,./test.sh -f tsim/user/basic.sim + +#unit-test +,,y,unit-test,bash test.sh + +#tsim test +,,y,script,./test.sh -f tsim/user/basic.sim +,,y,script,./test.sh -f tsim/user/password.sim +,,,script,./test.sh -f tsim/user/privilege_db.sim +,,,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,,script,./test.sh -f tsim/db/alter_option.sim +,,,script,./test.sh -f tsim/db/alter_replica_13.sim +,,,script,./test.sh -f tsim/db/alter_replica_31.sim +,,,script,./test.sh -f tsim/db/basic1.sim +,,,script,./test.sh -f tsim/db/basic2.sim +,,,script,./test.sh -f tsim/db/basic3.sim +,,,script,./test.sh -f tsim/db/basic4.sim +,,,script,./test.sh -f tsim/db/basic5.sim +,,,script,./test.sh -f tsim/db/basic6.sim +,,,script,./test.sh -f tsim/db/commit.sim +,,,script,./test.sh -f tsim/db/create_all_options.sim +,,,script,./test.sh -f tsim/db/delete_reuse1.sim +,,,script,./test.sh -f tsim/db/delete_reuse2.sim +,,,script,./test.sh -f tsim/db/delete_reusevnode.sim +,,,script,./test.sh -f tsim/db/delete_reusevnode2.sim +,,,script,./test.sh -f tsim/db/delete_writing1.sim +,,,script,./test.sh -f tsim/db/delete_writing2.sim +,,,script,./test.sh -f tsim/db/error1.sim +,,,script,./test.sh -f tsim/db/keep.sim +,,,script,./test.sh -f tsim/db/len.sim +,,,script,./test.sh -f tsim/db/repeat.sim +,,,script,./test.sh -f tsim/db/show_create_db.sim +,,,script,./test.sh -f tsim/db/show_create_table.sim +,,,script,./test.sh -f tsim/db/tables.sim +,,,script,./test.sh -f tsim/db/taosdlog.sim +,,,script,./test.sh -f tsim/dnode/balance_replica1.sim +,,,script,./test.sh -f tsim/dnode/balance_replica3.sim +,,,script,./test.sh -f tsim/dnode/balance1.sim +,,,script,./test.sh -f tsim/dnode/balance2.sim +,,,script,./test.sh -f tsim/dnode/balance3.sim +,,,script,./test.sh -f tsim/dnode/balancex.sim +,,,script,./test.sh -f tsim/dnode/create_dnode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +,,,script,./test.sh -f tsim/dnode/drop_dnode_force.sim +,,,script,./test.sh -f tsim/dnode/offline_reason.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +,,,script,./test.sh -f tsim/dnode/vnode_clean.sim +,,,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim +,,,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim +,,,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim +,,,script,./test.sh -f tsim/import/basic.sim +,,,script,./test.sh -f tsim/import/commit.sim +,,,script,./test.sh -f tsim/import/large.sim +,,,script,./test.sh -f tsim/import/replica1.sim +,,,script,./test.sh -f tsim/insert/backquote.sim +,,,script,./test.sh -f tsim/insert/basic.sim +,,,script,./test.sh -f tsim/insert/basic0.sim +,,,script,./test.sh -f tsim/insert/basic1.sim +,,,script,./test.sh -f tsim/insert/basic2.sim +,,,script,./test.sh -f tsim/insert/commit-merge0.sim +,,,script,./test.sh -f tsim/insert/insert_drop.sim +,,,script,./test.sh -f tsim/insert/insert_select.sim +,,,script,./test.sh -f tsim/insert/null.sim +,,,script,./test.sh -f tsim/insert/query_block1_file.sim +,,,script,./test.sh -f tsim/insert/query_block1_memory.sim +,,,script,./test.sh -f tsim/insert/query_block2_file.sim +,,,script,./test.sh -f tsim/insert/query_block2_memory.sim +,,,script,./test.sh -f tsim/insert/query_file_memory.sim +,,,script,./test.sh -f tsim/insert/query_multi_file.sim +,,,script,./test.sh -f tsim/insert/tcp.sim +,,,script,./test.sh -f tsim/insert/update0.sim +,,,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,,script,./test.sh -f tsim/parser/alter__for_community_version.sim +,,,script,./test.sh -f tsim/parser/alter_column.sim +,,,script,./test.sh -f tsim/parser/alter_stable.sim +,,,script,./test.sh -f tsim/parser/alter.sim +,,,script,./test.sh -f tsim/parser/alter1.sim +,,,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +,,,script,./test.sh -f tsim/parser/auto_create_tb.sim +,,,script,./test.sh -f tsim/parser/between_and.sim +,,,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim +,,,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim +,,,script,./test.sh -f tsim/parser/columnValue_bigint.sim +,,,script,./test.sh -f tsim/parser/columnValue_bool.sim +,,,script,./test.sh -f tsim/parser/columnValue_double.sim +,,,script,./test.sh -f tsim/parser/columnValue_float.sim +,,,script,./test.sh -f tsim/parser/columnValue_int.sim +,,,script,./test.sh -f tsim/parser/columnValue_smallint.sim +,,,script,./test.sh -f tsim/parser/columnValue_tinyint.sim +,,,script,./test.sh -f tsim/parser/columnValue_unsign.sim +,,,script,./test.sh -f tsim/parser/commit.sim +,,,script,./test.sh -f tsim/parser/condition.sim +,,,script,./test.sh -f tsim/parser/constCol.sim +,,,script,./test.sh -f tsim/parser/create_db.sim +,,,script,./test.sh -f tsim/parser/create_mt.sim +,,,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim +,,,script,./test.sh -f tsim/parser/create_tb.sim +,,,script,./test.sh -f tsim/parser/dbtbnameValidate.sim +,,,script,./test.sh -f tsim/parser/distinct.sim +,,,script,./test.sh -f tsim/parser/fill_us.sim +,,,script,./test.sh -f tsim/parser/fill.sim +,,,script,./test.sh -f tsim/parser/first_last.sim +,,,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim +,,,script,./test.sh -f tsim/parser/function.sim +,,,script,./test.sh -f tsim/parser/groupby-basic.sim +,,,script,./test.sh -f tsim/parser/groupby.sim +,,,script,./test.sh -f tsim/parser/having_child.sim +,,,script,./test.sh -f tsim/parser/having.sim +,,,script,./test.sh -f tsim/parser/import_commit1.sim +,,,script,./test.sh -f tsim/parser/import_commit2.sim +,,,script,./test.sh -f tsim/parser/import_commit3.sim +,,,script,./test.sh -f tsim/parser/import_file.sim +,,,script,./test.sh -f tsim/parser/import.sim +,,,script,./test.sh -f tsim/parser/insert_multiTbl.sim +,,,script,./test.sh -f tsim/parser/insert_tb.sim +,,,script,./test.sh -f tsim/parser/join_manyblocks.sim +,,,script,./test.sh -f tsim/parser/join_multitables.sim +,,,script,./test.sh -f tsim/parser/join_multivnode.sim +,,,script,./test.sh -f tsim/parser/join.sim +,,,script,./test.sh -f tsim/parser/last_cache.sim +,,,script,./test.sh -f tsim/parser/last_groupby.sim +,,,script,./test.sh -f tsim/parser/lastrow.sim +,,,script,./test.sh -f tsim/parser/lastrow2.sim +,,,script,./test.sh -f tsim/parser/like.sim +,,,script,./test.sh -f tsim/parser/limit.sim +,,,script,./test.sh -f tsim/parser/limit1.sim +,,,script,./test.sh -f tsim/parser/mixed_blocks.sim +,,,script,./test.sh -f tsim/parser/nchar.sim +,,,script,./test.sh -f tsim/parser/nestquery.sim +,,,script,./test.sh -f tsim/parser/null_char.sim +,,,script,./test.sh -f tsim/parser/precision_ns.sim +,,,script,./test.sh -f tsim/parser/projection_limit_offset.sim +,,,script,./test.sh -f tsim/parser/regex.sim +,,,script,./test.sh -f tsim/parser/select_across_vnodes.sim +,,,script,./test.sh -f tsim/parser/select_distinct_tag.sim +,,,script,./test.sh -f tsim/parser/select_from_cache_disk.sim +,,,script,./test.sh -f tsim/parser/select_with_tags.sim +,,,script,./test.sh -f tsim/parser/selectResNum.sim +,,,script,./test.sh -f tsim/parser/set_tag_vals.sim +,,,script,./test.sh -f tsim/parser/single_row_in_tb.sim +,,,script,./test.sh -f tsim/parser/sliding.sim +,,,script,./test.sh -f tsim/parser/slimit_alter_tags.sim +,,,script,./test.sh -f tsim/parser/slimit.sim +,,,script,./test.sh -f tsim/parser/slimit1.sim +,,,script,./test.sh -f tsim/parser/stableOp.sim +,,,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +,,,script,./test.sh -f tsim/parser/tags_filter.sim +,,,script,./test.sh -f tsim/parser/tbnameIn.sim +,,,script,./test.sh -f tsim/parser/timestamp.sim +,,,script,./test.sh -f tsim/parser/top_groupby.sim +,,,script,./test.sh -f tsim/parser/topbot.sim +,,,script,./test.sh -f tsim/parser/union.sim +,,,script,./test.sh -f tsim/parser/union_sysinfo.sim +,,,script,./test.sh -f tsim/parser/where.sim +,,,script,./test.sh -f tsim/query/charScalarFunction.sim +,,,script,./test.sh -f tsim/query/explain.sim +,,,script,./test.sh -f tsim/query/interval-offset.sim +,,,script,./test.sh -f tsim/query/interval.sim +,,,script,./test.sh -f tsim/query/scalarFunction.sim +,,,script,./test.sh -f tsim/query/scalarNull.sim +,,,script,./test.sh -f tsim/query/session.sim +,,,script,./test.sh -f tsim/query/udf.sim +,,,script,./test.sh -f tsim/qnode/basic1.sim +,,,script,./test.sh -f tsim/snode/basic1.sim +,,,script,./test.sh -f tsim/mnode/basic1.sim +,,,script,./test.sh -f tsim/mnode/basic2.sim +,,,script,./test.sh -f tsim/mnode/basic3.sim +,,,script,./test.sh -f tsim/mnode/basic4.sim +,,,script,./test.sh -f tsim/mnode/basic5.sim +,,,script,./test.sh -f tsim/show/basic.sim +,,,script,./test.sh -f tsim/table/autocreate.sim +,,,script,./test.sh -f tsim/table/basic1.sim +,,,script,./test.sh -f tsim/table/basic2.sim +,,,script,./test.sh -f tsim/table/basic3.sim +,,,script,./test.sh -f tsim/table/bigint.sim +,,,script,./test.sh -f tsim/table/binary.sim +,,,script,./test.sh -f tsim/table/bool.sim +,,,script,./test.sh -f tsim/table/column_name.sim +,,,script,./test.sh -f tsim/table/column_num.sim +,,,script,./test.sh -f tsim/table/column_value.sim +,,,script,./test.sh -f tsim/table/column2.sim +,,,script,./test.sh -f tsim/table/createmulti.sim +,,,script,./test.sh -f tsim/table/date.sim +,,,script,./test.sh -f tsim/table/db.table.sim +,,,script,./test.sh -f tsim/table/delete_reuse1.sim +,,,script,./test.sh -f tsim/table/delete_reuse2.sim +,,,script,./test.sh -f tsim/table/delete_writing.sim +,,,script,./test.sh -f tsim/table/describe.sim +,,,script,./test.sh -f tsim/table/double.sim +,,,script,./test.sh -f tsim/table/float.sim +,,,script,./test.sh -f tsim/table/hash.sim +,,,script,./test.sh -f tsim/table/int.sim +,,,script,./test.sh -f tsim/table/limit.sim +,,,script,./test.sh -f tsim/table/smallint.sim +,,,script,./test.sh -f tsim/table/table_len.sim +,,,script,./test.sh -f tsim/table/table.sim +,,,script,./test.sh -f tsim/table/tinyint.sim +,,,script,./test.sh -f tsim/table/vgroup.sim +,,,script,./test.sh -f tsim/stream/basic0.sim -g +,,,script,./test.sh -f tsim/stream/basic2.sim +,,,script,./test.sh -f tsim/stream/drop_stream.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim +,,,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,,script,./test.sh -f tsim/stream/distributeInterval0.sim +,,,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,,script,./test.sh -f tsim/stream/distributeSession0.sim +,,,script,./test.sh -f tsim/stream/session0.sim +,,,script,./test.sh -f tsim/stream/session1.sim +,,,script,./test.sh -f tsim/stream/state0.sim +,,,script,./test.sh -f tsim/stream/triggerInterval0.sim +,,,script,./test.sh -f tsim/stream/triggerSession0.sim +,,,script,./test.sh -f tsim/stream/partitionby.sim +,,,script,./test.sh -f tsim/stream/partitionby1.sim +,,,script,./test.sh -f tsim/stream/schedSnode.sim +,,,script,./test.sh -f tsim/stream/windowClose.sim +,,,script,./test.sh -f tsim/stream/ignoreExpiredData.sim +,,,script,./test.sh -f tsim/stream/sliding.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnState.sim +,,,script,./test.sh -f tsim/stream/deleteInterval.sim +,,,script,./test.sh -f tsim/stream/deleteSession.sim +,,,script,./test.sh -f tsim/stream/deleteState.sim +,,,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim +,,,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim +,,,script,./test.sh -f tsim/stream/fillIntervalLinear.sim +,,,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +,,,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim +,,,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,,script,./test.sh -f tsim/trans/lossdata1.sim +,,,script,./test.sh -f tsim/trans/create_db.sim +,,,script,./test.sh -f tsim/tmq/basic1.sim +,,,script,./test.sh -f tsim/tmq/basic2.sim +,,,script,./test.sh -f tsim/tmq/basic3.sim +,,,script,./test.sh -f tsim/tmq/basic4.sim +,,,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim +,,,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +,,,script,./test.sh -f tsim/tmq/topic.sim +,,,script,./test.sh -f tsim/tmq/snapshot.sim +,,,script,./test.sh -f tsim/tmq/snapshot1.sim +,,,script,./test.sh -f tsim/stable/alter_comment.sim +,,,script,./test.sh -f tsim/stable/alter_count.sim +,,,script,./test.sh -f tsim/stable/alter_import.sim +,,,script,./test.sh -f tsim/stable/alter_insert1.sim +,,,script,./test.sh -f tsim/stable/alter_insert2.sim +,,,script,./test.sh -f tsim/stable/alter_metrics.sim +,,,script,./test.sh -f tsim/stable/column_add.sim +,,,script,./test.sh -f tsim/stable/column_drop.sim +,,,script,./test.sh -f tsim/stable/column_modify.sim +,,,script,./test.sh -f tsim/stable/disk.sim +,,,script,./test.sh -f tsim/stable/dnode3.sim +,,,script,./test.sh -f tsim/stable/metrics.sim +,,,script,./test.sh -f tsim/stable/refcount.sim +,,,script,./test.sh -f tsim/stable/tag_add.sim +,,,script,./test.sh -f tsim/stable/tag_drop.sim +,,,script,./test.sh -f tsim/stable/tag_filter.sim +,,,script,./test.sh -f tsim/stable/tag_modify.sim +,,,script,./test.sh -f tsim/stable/tag_rename.sim +,,,script,./test.sh -f tsim/stable/values.sim +,,,script,./test.sh -f tsim/stable/vnode3.sim +,,,script,./test.sh -f tsim/stable/metrics_idx.sim +,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +,,,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +,,,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +,,,script,./test.sh -f tsim/valgrind/checkError1.sim +,,,script,./test.sh -f tsim/valgrind/checkError2.sim +,,,script,./test.sh -f tsim/valgrind/checkError3.sim +,,,script,./test.sh -f tsim/valgrind/checkError4.sim +,,,script,./test.sh -f tsim/valgrind/checkError5.sim +,,,script,./test.sh -f tsim/valgrind/checkError6.sim +,,,script,./test.sh -f tsim/valgrind/checkError7.sim +,,,script,./test.sh -f tsim/valgrind/checkError8.sim +,,,script,./test.sh -f tsim/valgrind/checkUdf.sim +,,,script,./test.sh -f tsim/vnode/replica3_basic.sim +,,,script,./test.sh -f tsim/vnode/replica3_repeat.sim +,,,script,./test.sh -f tsim/vnode/replica3_vgroup.sim +,,,script,./test.sh -f tsim/vnode/replica3_many.sim +,,,script,./test.sh -f tsim/vnode/replica3_import.sim +,,,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode2.sim +,,,script,./test.sh -f tsim/vnode/stable_dnode3.sim +,,,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +,,,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim +,,,script,./test.sh -f tsim/sync/3Replica1VgElect.sim +,,,script,./test.sh -f tsim/sync/3Replica5VgElect.sim +,,,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim +,,,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim +,,,script,./test.sh -f tsim/catalog/alterInCurrent.sim +,,,script,./test.sh -f tsim/scalar/in.sim +,,,script,./test.sh -f tsim/scalar/scalar.sim +,,,script,./test.sh -f tsim/scalar/filter.sim +,,,script,./test.sh -f tsim/scalar/caseWhen.sim +,,,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim +,,,script,./test.sh -f tsim/alter/dnode.sim +,,,script,./test.sh -f tsim/alter/table.sim +,,,script,./test.sh -f tsim/cache/new_metrics.sim +,,,script,./test.sh -f tsim/cache/restart_table.sim +,,,script,./test.sh -f tsim/cache/restart_metrics.sim +,,,script,./test.sh -f tsim/column/commit.sim +,,,script,./test.sh -f tsim/column/metrics.sim +,,,script,./test.sh -f tsim/column/table.sim +,,,script,./test.sh -f tsim/compress/commitlog.sim +,,,script,./test.sh -f tsim/compress/compress2.sim +,,,script,./test.sh -f tsim/compress/compress.sim +,,,script,./test.sh -f tsim/compress/uncompress.sim +,,,script,./test.sh -f tsim/compute/avg.sim +,,,script,./test.sh -f tsim/compute/block_dist.sim +,,,script,./test.sh -f tsim/compute/bottom.sim +,,,script,./test.sh -f tsim/compute/count.sim +,,,script,./test.sh -f tsim/compute/diff.sim +,,,script,./test.sh -f tsim/compute/diff2.sim +,,,script,./test.sh -f tsim/compute/first.sim +,,,script,./test.sh -f tsim/compute/interval.sim +,,,script,./test.sh -f tsim/compute/last_row.sim +,,,script,./test.sh -f tsim/compute/last.sim +,,,script,./test.sh -f tsim/compute/leastsquare.sim +,,,script,./test.sh -f tsim/compute/max.sim +,,,script,./test.sh -f tsim/compute/min.sim +,,,script,./test.sh -f tsim/compute/null.sim +,,,script,./test.sh -f tsim/compute/percentile.sim +,,,script,./test.sh -f tsim/compute/stddev.sim +,,,script,./test.sh -f tsim/compute/sum.sim +,,,script,./test.sh -f tsim/compute/top.sim +,,,script,./test.sh -f tsim/field/2.sim +,,,script,./test.sh -f tsim/field/3.sim +,,,script,./test.sh -f tsim/field/4.sim +,,,script,./test.sh -f tsim/field/5.sim +,,,script,./test.sh -f tsim/field/6.sim +,,,script,./test.sh -f tsim/field/binary.sim +,,,script,./test.sh -f tsim/field/bigint.sim +,,,script,./test.sh -f tsim/field/bool.sim +,,,script,./test.sh -f tsim/field/double.sim +,,,script,./test.sh -f tsim/field/float.sim +,,,script,./test.sh -f tsim/field/int.sim +,,,script,./test.sh -f tsim/field/single.sim +,,,script,./test.sh -f tsim/field/smallint.sim +,,,script,./test.sh -f tsim/field/tinyint.sim +,,,script,./test.sh -f tsim/field/unsigined_bigint.sim +,,,script,./test.sh -f tsim/vector/metrics_field.sim +,,,script,./test.sh -f tsim/vector/metrics_mix.sim +,,,script,./test.sh -f tsim/vector/metrics_query.sim +,,,script,./test.sh -f tsim/vector/metrics_tag.sim +,,,script,./test.sh -f tsim/vector/metrics_time.sim +,,,script,./test.sh -f tsim/vector/multi.sim +,,,script,./test.sh -f tsim/vector/single.sim +,,,script,./test.sh -f tsim/vector/table_field.sim +,,,script,./test.sh -f tsim/vector/table_mix.sim +,,,script,./test.sh -f tsim/vector/table_query.sim +,,,script,./test.sh -f tsim/vector/table_time.sim +,,,script,./test.sh -f tsim/wal/kill.sim +,,,script,./test.sh -f tsim/tag/3.sim +,,,script,./test.sh -f tsim/tag/4.sim +,,,script,./test.sh -f tsim/tag/5.sim +,,,script,./test.sh -f tsim/tag/6.sim +,,,script,./test.sh -f tsim/tag/add.sim +,,,script,./test.sh -f tsim/tag/bigint.sim +,,,script,./test.sh -f tsim/tag/binary_binary.sim +,,,script,./test.sh -f tsim/tag/binary.sim +,,,script,./test.sh -f tsim/tag/bool_binary.sim +,,,script,./test.sh -f tsim/tag/bool_int.sim +,,,script,./test.sh -f tsim/tag/bool.sim +,,,script,./test.sh -f tsim/tag/change.sim +,,,script,./test.sh -f tsim/tag/column.sim +,,,script,./test.sh -f tsim/tag/commit.sim +,,,script,./test.sh -f tsim/tag/create.sim +,,,script,./test.sh -f tsim/tag/delete.sim +,,,script,./test.sh -f tsim/tag/double.sim +,,,script,./test.sh -f tsim/tag/filter.sim +,,,script,./test.sh -f tsim/tag/float.sim +,,,script,./test.sh -f tsim/tag/int_binary.sim +,,,script,./test.sh -f tsim/tag/int_float.sim +,,,script,./test.sh -f tsim/tag/int.sim +,,,script,./test.sh -f tsim/tag/set.sim +,,,script,./test.sh -f tsim/tag/smallint.sim +,,,script,./test.sh -f tsim/tag/tinyint.sim +,,,script,./test.sh -f tsim/tag/drop_tag.sim +,,,script,./test.sh -f tsim/tag/tbNameIn.sim +,,,script,./test.sh -f tmp/monitor.sim + +#system test + +,,,system-test,python3 ./test.py -f 0-others/taosShell.py +,,,system-test,python3 ./test.py -f 0-others/taosShellError.py +,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py +,,,system-test,python3 ./test.py -f 0-others/telemetry.py +,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py +,,,system-test,python3 ./test.py -f 0-others/udfTest.py +,,,system-test,python3 ./test.py -f 0-others/udf_create.py +,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,,system-test,python3 ./test.py -f 0-others/cachemodel.py +,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,,system-test,python3 ./test.py -f 0-others/udf_cfg2.py +,,,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,,system-test,python3 ./test.py -f 0-others/sysinfo.py +,,,system-test,python3 ./test.py -f 0-others/user_control.py +,,,system-test,python3 ./test.py -f 0-others/fsync.py +,,,system-test,python3 ./test.py -f 0-others/compatibility.py +,,,system-test,python3 ./test.py -f 1-insert/alter_database.py +,,,system-test,python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,,system-test,python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +,,,system-test,python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +,,,system-test,python3 ./test.py -f 1-insert/alter_stable.py +,,,system-test,python3 ./test.py -f 1-insert/alter_table.py +,,,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +,,,system-test,python3 ./test.py -f 1-insert/table_comment.py +,,,system-test,python3 ./test.py -f 1-insert/time_range_wise.py +,,,system-test,python3 ./test.py -f 1-insert/block_wise.py +,,,system-test,python3 ./test.py -f 1-insert/create_retentions.py +,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py +,,,system-test,python3 ./test.py -f 1-insert/mutil_stage.py +,,,system-test,python3 ./test.py -f 1-insert/table_param_ttl.py -R +,,,system-test,python3 ./test.py -f 1-insert/update_data_muti_rows.py +,,,system-test,python3 ./test.py -f 1-insert/db_tb_name_check.py +,,,system-test,python3 ./test.py -f 1-insert/database_pre_suf.py +,,,system-test,python3 ./test.py -f 0-others/show.py +,,,system-test,python3 ./test.py -f 2-query/abs.py +,,,system-test,python3 ./test.py -f 2-query/abs.py -R +,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py +,,,system-test,python3 ./test.py -f 2-query/and_or_for_byte.py -R +,,,system-test,python3 ./test.py -f 2-query/apercentile.py +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -R +,,,system-test,python3 ./test.py -f 2-query/arccos.py +,,,system-test,python3 ./test.py -f 2-query/arccos.py -R +,,,system-test,python3 ./test.py -f 2-query/arcsin.py +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -R +,,,system-test,python3 ./test.py -f 2-query/arctan.py +,,,system-test,python3 ./test.py -f 2-query/arctan.py -R +,,,system-test,python3 ./test.py -f 2-query/avg.py +,,,system-test,python3 ./test.py -f 2-query/avg.py -R +,,,system-test,python3 ./test.py -f 2-query/between.py +,,,system-test,python3 ./test.py -f 2-query/between.py -R +,,,system-test,python3 ./test.py -f 2-query/bottom.py +,,,system-test,python3 ./test.py -f 2-query/bottom.py -R +,,,system-test,python3 ./test.py -f 2-query/cast.py +,,,system-test,python3 ./test.py -f 2-query/cast.py -R +,,,system-test,python3 ./test.py -f 2-query/ceil.py +,,,system-test,python3 ./test.py -f 2-query/ceil.py -R +,,,system-test,python3 ./test.py -f 2-query/char_length.py +,,,system-test,python3 ./test.py -f 2-query/char_length.py -R +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -R +,,,system-test,python3 ./test.py -f 2-query/concat.py +,,,system-test,python3 ./test.py -f 2-query/concat.py -R +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R +,,,system-test,python3 ./test.py -f 2-query/cos.py +,,,system-test,python3 ./test.py -f 2-query/cos.py -R +,,,system-test,python3 ./test.py -f 2-query/count_partition.py +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -R +,,,system-test,python3 ./test.py -f 2-query/count.py +,,,system-test,python3 ./test.py -f 2-query/count.py -R +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +,,,system-test,python3 ./test.py -f 2-query/db.py +,,,system-test,python3 ./test.py -f 2-query/db.py -R +,,,system-test,python3 ./test.py -f 2-query/diff.py +,,,system-test,python3 ./test.py -f 2-query/diff.py -R +,,,system-test,python3 ./test.py -f 2-query/distinct.py +,,,system-test,python3 ./test.py -f 2-query/distinct.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -R +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -R +,,,system-test,python3 ./test.py -f 2-query/explain.py +,,,system-test,python3 ./test.py -f 2-query/explain.py -R +,,,system-test,python3 ./test.py -f 2-query/first.py +,,,system-test,python3 ./test.py -f 2-query/first.py -R +,,,system-test,python3 ./test.py -f 2-query/floor.py +,,,system-test,python3 ./test.py -f 2-query/floor.py -R +,,,system-test,python3 ./test.py -f 2-query/function_null.py +,,,system-test,python3 ./test.py -f 2-query/function_null.py -R +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -R +,,,system-test,python3 ./test.py -f 2-query/histogram.py +,,,system-test,python3 ./test.py -f 2-query/histogram.py -R +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -R +,,,system-test,python3 ./test.py -f 2-query/interp.py +,,,system-test,python3 ./test.py -f 2-query/interp.py -R +,,,system-test,python3 ./test.py -f 2-query/irate.py +,,,system-test,python3 ./test.py -f 2-query/irate.py -R +,,,system-test,python3 ./test.py -f 2-query/join.py +,,,system-test,python3 ./test.py -f 2-query/join.py -R +,,,system-test,python3 ./test.py -f 2-query/last_row.py +,,,system-test,python3 ./test.py -f 2-query/last_row.py -R +,,,system-test,python3 ./test.py -f 2-query/last.py +,,,system-test,python3 ./test.py -f 2-query/last.py -R +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -R +,,,system-test,python3 ./test.py -f 2-query/length.py +,,,system-test,python3 ./test.py -f 2-query/length.py -R +,,,system-test,python3 ./test.py -f 2-query/log.py +,,,system-test,python3 ./test.py -f 2-query/log.py -R +,,,system-test,python3 ./test.py -f 2-query/lower.py +,,,system-test,python3 ./test.py -f 2-query/lower.py -R +,,,system-test,python3 ./test.py -f 2-query/ltrim.py +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R +,,,system-test,python3 ./test.py -f 2-query/mavg.py +,,,system-test,python3 ./test.py -f 2-query/mavg.py -R +,,,system-test,python3 ./test.py -f 2-query/max_partition.py +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -R +,,,system-test,python3 ./test.py -f 2-query/max.py +,,,system-test,python3 ./test.py -f 2-query/max.py -R +,,,system-test,python3 ./test.py -f 2-query/min.py +,,,system-test,python3 ./test.py -f 2-query/min.py -R +,,,system-test,python3 ./test.py -f 2-query/mode.py +,,,system-test,python3 ./test.py -f 2-query/mode.py -R +,,,system-test,python3 ./test.py -f 2-query/Now.py +,,,system-test,python3 ./test.py -f 2-query/Now.py -R +,,,system-test,python3 ./test.py -f 2-query/percentile.py +,,,system-test,python3 ./test.py -f 2-query/percentile.py -R +,,,system-test,python3 ./test.py -f 2-query/pow.py +,,,system-test,python3 ./test.py -f 2-query/pow.py -R +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R +,,,system-test,python3 ./test.py -f 2-query/round.py +,,,system-test,python3 ./test.py -f 2-query/round.py -R +,,,system-test,python3 ./test.py -f 2-query/rtrim.py +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R +,,,system-test,python3 ./test.py -f 2-query/sample.py +,,,system-test,python3 ./test.py -f 2-query/sample.py -R +,,,system-test,python3 ./test.py -f 2-query/sin.py +,,,system-test,python3 ./test.py -f 2-query/sin.py -R +,,,system-test,python3 ./test.py -f 2-query/smaTest.py +,,,system-test,python3 ./test.py -f 2-query/smaTest.py -R +,,,system-test,python3 ./test.py -f 2-query/sml.py +,,,system-test,python3 ./test.py -f 2-query/sml.py -R +,,,system-test,python3 ./test.py -f 2-query/spread.py +,,,system-test,python3 ./test.py -f 2-query/spread.py -R +,,,system-test,python3 ./test.py -f 2-query/sqrt.py +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -R +,,,system-test,python3 ./test.py -f 2-query/statecount.py +,,,system-test,python3 ./test.py -f 2-query/statecount.py -R +,,,system-test,python3 ./test.py -f 2-query/stateduration.py +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -R +,,,system-test,python3 ./test.py -f 2-query/substr.py +,,,system-test,python3 ./test.py -f 2-query/substr.py -R +,,,system-test,python3 ./test.py -f 2-query/sum.py +,,,system-test,python3 ./test.py -f 2-query/sum.py -R +,,,system-test,python3 ./test.py -f 2-query/tail.py +,,,system-test,python3 ./test.py -f 2-query/tail.py -R +,,,system-test,python3 ./test.py -f 2-query/tan.py +,,,system-test,python3 ./test.py -f 2-query/tan.py -R +,,,system-test,python3 ./test.py -f 2-query/Timediff.py +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -R +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -R +,,,system-test,python3 ./test.py -f 2-query/timezone.py +,,,system-test,python3 ./test.py -f 2-query/timezone.py -R +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -R +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -R +,,,system-test,python3 ./test.py -f 2-query/Today.py +,,,system-test,python3 ./test.py -f 2-query/Today.py -R +,,,system-test,python3 ./test.py -f 2-query/top.py +,,,system-test,python3 ./test.py -f 2-query/top.py -R +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -R +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -R +,,,system-test,python3 ./test.py -f 2-query/twa.py +,,,system-test,python3 ./test.py -f 2-query/twa.py -R +,,,system-test,python3 ./test.py -f 2-query/union.py +,,,system-test,python3 ./test.py -f 2-query/union.py -R +,,,system-test,python3 ./test.py -f 2-query/unique.py +,,,system-test,python3 ./test.py -f 2-query/unique.py -R +,,,system-test,python3 ./test.py -f 2-query/upper.py +,,,system-test,python3 ./test.py -f 2-query/upper.py -R +,,,system-test,python3 ./test.py -f 2-query/varchar.py +,,,system-test,python3 ./test.py -f 2-query/varchar.py -R +,,,system-test,python3 ./test.py -f 1-insert/update_data.py +,,,system-test,python3 ./test.py -f 1-insert/tb_100w_data_order.py +,,,system-test,python3 ./test.py -f 1-insert/delete_stable.py +,,,system-test,python3 ./test.py -f 1-insert/delete_childtable.py +,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py +,,,system-test,python3 ./test.py -f 1-insert/keep_expired.py +,,,system-test,python3 ./test.py -f 2-query/join2.py +,,,system-test,python3 ./test.py -f 2-query/union1.py +,,,system-test,python3 ./test.py -f 2-query/concat2.py +,,,system-test,python3 ./test.py -f 2-query/json_tag.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py +,,,system-test,python3 ./test.py -f 2-query/stablity.py +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py +,,,system-test,python3 ./test.py -f 2-query/elapsed.py +,,,system-test,python3 ./test.py -f 2-query/csum.py +,,,system-test,python3 ./test.py -f 2-query/function_diff.py +,,,system-test,python3 ./test.py -f 2-query/queryQnode.py +,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py +,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 +,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 +,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +,,,system-test,python3 ./test.py -f 7-tmq/create_wrong_topic.py +,,,system-test,python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +,,,system-test,python3 ./test.py -f 7-tmq/basic5.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb0.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb1.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb2.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb3.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeDb4.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb0.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb1.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb2.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb3.py +,,,system-test,python3 ./test.py -f 7-tmq/subscribeStb4.py +,,,system-test,python3 ./test.py -f 7-tmq/db.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqError.py +,,,system-test,python3 ./test.py -f 7-tmq/schema.py +,,,system-test,python3 ./test.py -f 7-tmq/stbFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqCheckData1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqShow.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqAlterSchema.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +,,,system-test,python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py +,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/cast.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/between.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/length.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/char_length.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/upper.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/lower.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/join.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/explain.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/timezone.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Now.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Today.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/max.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/count.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/last.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/first.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/diff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/top.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/bottom.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/percentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/abs.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ceil.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/floor.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/round.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/log.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/pow.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sin.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/cos.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tan.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arccos.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/arctan.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mavg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/twa.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/irate.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_null.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/count_partition.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/max_partition.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sml.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 + +#develop test +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R + +#docs-examples test +,,,docs-examples-test,bash python.sh +,,,docs-examples-test,bash node.sh +,,,docs-examples-test,bash csharp.sh +,,,docs-examples-test,bash jdbc.sh +,,,docs-examples-test,bash go.sh diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 52864d668f..4e06643ac4 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -44,16 +44,35 @@ ulimit -c unlimited if [ $ent -eq 0 ]; then REP_DIR=/home/TDengine - REP_MOUNT_PARAM=$WORKDIR/TDengine:/home/TDengine + REP_REAL_PATH=$WORKDIR/TDengine + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine else REP_DIR=/home/TDinternal - REP_MOUNT_PARAM=$WORKDIR/TDinternal:/home/TDinternal + REP_REAL_PATH=$WORKDIR/TDinternal + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal fi - +date docker run \ -v $REP_MOUNT_PARAM \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT" +if [[ -d ${WORKDIR}/debugNoSan ]] ;then + echo "delete ${WORKDIR}/debugNoSan" + rm -rf ${WORKDIR}/debugNoSan +fi +if [[ -d ${WORKDIR}/debugSan ]] ;then + echo "delete ${WORKDIR}/debugSan" + rm -rf ${WORKDIR}/debugSan +fi + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan +date +docker run \ + -v $REP_MOUNT_PARAM \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DSANITIZER=true;make -j $THREAD_COUNT" + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan + ret=$? exit $ret diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 26f481e571..b5d57265be 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -164,8 +164,17 @@ function run_thread() { if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local exec_dir=`echo "$line"|cut -d, -f3` - local case_cmd=`echo "$line"|cut -d, -f4` + local case_build_san=`echo "$line"|cut -d, -f3` + if [ "${case_build_san}" == "y" ]; then + case_build_san="y" + elif [[ "${case_build_san}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then + case_build_san="n" + else + usage + exit 1 + fi + local exec_dir=`echo "$line"|cut -d, -f4` + local case_cmd=`echo "$line"|cut -d, -f5` local case_file="" echo "$case_cmd"|grep -q "\.sh" if [ $? -eq 0 ]; then @@ -191,7 +200,7 @@ function run_thread() { if [ ! -z "$case_path" ]; then mkdir -p $log_dir/$case_path fi - cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param}" + cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 @@ -199,7 +208,7 @@ function run_thread() { start_time=`date +%s` local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4` + local case_info=`echo "$line"|cut -d, -f 3,4,5` while [ ${redo_count} -lt 6 ]; do if [ -f $case_log_file ]; then cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt @@ -413,7 +422,7 @@ if [ -f "${failed_case_file}" ]; then continue fi fi - line=`echo "$line"|cut -d, -f 3,4` + line=`echo "$line"|cut -d, -f 3,4,5` echo -e "$i. $line \e[31m failed\e[0m" >&2 i=$(( i + 1 )) done <${failed_case_file} diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index f58aaaf29d..b5deb09341 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -8,11 +8,12 @@ function usage() { echo -e "\t -t thread number" echo -e "\t -e enterprise edition" echo -e "\t -o default timeout value" + echo -e "\t -s build with sanitizer" echo -e "\t -h help" } ent=0 -while getopts "w:d:c:t:o:eh" opt; do +while getopts "w:d:c:t:o:s:eh" opt; do case $opt in w) WORKDIR=$OPTARG @@ -32,6 +33,9 @@ while getopts "w:d:c:t:o:eh" opt; do o) extra_param="-o $OPTARG" ;; + s) + buildSan=$OPTARG + ;; h) usage exit 0 @@ -60,24 +64,37 @@ if [ -z "$thread_no" ]; then usage exit 1 fi + +#select whether the compilation environment includes sanitizer +if [ "${buildSan}" == "y" ]; then + DEBUGPATH="debugSan" +elif [[ "${buildSan}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then + DEBUGPATH="debugNoSan" +else + usage + exit 1 +fi + if [ $ent -ne 0 ]; then # enterprise edition extra_param="$extra_param -e" INTERNAL_REPDIR=$WORKDIR/TDinternal REPDIR=$INTERNAL_REPDIR/community + REPDIR_DEBUG=$WORKDIR/$DEBUGPATH/ CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" - REP_MOUNT_LIB="$INTERNAL_REPDIR/debug/build/lib:/home/TDinternal/debug/build/lib:ro" - + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/" + REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro" else # community edition REPDIR=$WORKDIR/TDengine + REPDIR_DEBUG=$WORKDIR/$DEBUGPATH/ CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" - REP_MOUNT_LIB="$REPDIR/debug/build/lib:/home/TDengine/debug/build/lib:ro" - + REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDengine/debug/" + REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro" fi ulimit -c unlimited @@ -107,6 +124,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_DEBUG \ -v $REP_MOUNT_LIB \ -v $MOUNT_DIR \ -v ${SOURCEDIR}:/usr/local/src/ \ diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt deleted file mode 100644 index e6fdecf35e..0000000000 --- a/tests/script/jenkins/basic.txt +++ /dev/null @@ -1,457 +0,0 @@ - -#======================b1-start=============== - -# ---- user ---- -./test.sh -f tsim/user/basic.sim -./test.sh -f tsim/user/password.sim -./test.sh -f tsim/user/privilege_db.sim -./test.sh -f tsim/user/privilege_sysinfo.sim - -# ---- db ---- -./test.sh -f tsim/db/alter_option.sim -./test.sh -f tsim/db/alter_replica_13.sim -./test.sh -f tsim/db/alter_replica_31.sim -./test.sh -f tsim/db/basic1.sim -./test.sh -f tsim/db/basic2.sim -./test.sh -f tsim/db/basic3.sim -./test.sh -f tsim/db/basic4.sim -./test.sh -f tsim/db/basic5.sim -./test.sh -f tsim/db/basic6.sim -./test.sh -f tsim/db/commit.sim -./test.sh -f tsim/db/create_all_options.sim -./test.sh -f tsim/db/delete_reuse1.sim -./test.sh -f tsim/db/delete_reuse2.sim -./test.sh -f tsim/db/delete_reusevnode.sim -./test.sh -f tsim/db/delete_reusevnode2.sim -./test.sh -f tsim/db/delete_writing1.sim -./test.sh -f tsim/db/delete_writing2.sim -./test.sh -f tsim/db/error1.sim -./test.sh -f tsim/db/keep.sim -./test.sh -f tsim/db/len.sim -./test.sh -f tsim/db/repeat.sim -./test.sh -f tsim/db/show_create_db.sim -./test.sh -f tsim/db/show_create_table.sim -./test.sh -f tsim/db/tables.sim -./test.sh -f tsim/db/taosdlog.sim - -# ---- dnode -./test.sh -f tsim/dnode/balance_replica1.sim -./test.sh -f tsim/dnode/balance_replica3.sim -./test.sh -f tsim/dnode/balance1.sim -./test.sh -f tsim/dnode/balance2.sim -./test.sh -f tsim/dnode/balance3.sim -./test.sh -f tsim/dnode/balancex.sim -./test.sh -f tsim/dnode/create_dnode.sim -./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim -./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -./test.sh -f tsim/dnode/drop_dnode_force.sim -./test.sh -f tsim/dnode/offline_reason.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim -./test.sh -f tsim/dnode/vnode_clean.sim -./test.sh -f tsim/dnode/use_dropped_dnode.sim -./test.sh -f tsim/dnode/split_vgroup_replica1.sim -./test.sh -f tsim/dnode/split_vgroup_replica3.sim - -# ---- import ---- -./test.sh -f tsim/import/basic.sim -./test.sh -f tsim/import/commit.sim -./test.sh -f tsim/import/large.sim -./test.sh -f tsim/import/replica1.sim - -# ---- insert ---- -./test.sh -f tsim/insert/backquote.sim -./test.sh -f tsim/insert/basic.sim -./test.sh -f tsim/insert/basic0.sim -./test.sh -f tsim/insert/basic1.sim -./test.sh -f tsim/insert/basic2.sim -./test.sh -f tsim/insert/commit-merge0.sim -./test.sh -f tsim/insert/insert_drop.sim -./test.sh -f tsim/insert/insert_select.sim -./test.sh -f tsim/insert/null.sim -./test.sh -f tsim/insert/query_block1_file.sim -./test.sh -f tsim/insert/query_block1_memory.sim -./test.sh -f tsim/insert/query_block2_file.sim -./test.sh -f tsim/insert/query_block2_memory.sim -./test.sh -f tsim/insert/query_file_memory.sim -./test.sh -f tsim/insert/query_multi_file.sim -./test.sh -f tsim/insert/tcp.sim -./test.sh -f tsim/insert/update0.sim -./test.sh -f tsim/insert/update1_sort_merge.sim - -# ---- parser ---- -./test.sh -f tsim/parser/alter__for_community_version.sim -./test.sh -f tsim/parser/alter_column.sim -./test.sh -f tsim/parser/alter_stable.sim -./test.sh -f tsim/parser/alter.sim -./test.sh -f tsim/parser/alter1.sim -./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -./test.sh -f tsim/parser/auto_create_tb.sim -./test.sh -f tsim/parser/between_and.sim -./test.sh -f tsim/parser/binary_escapeCharacter.sim -./test.sh -f tsim/parser/col_arithmetic_operation.sim -./test.sh -f tsim/parser/columnValue_bigint.sim -./test.sh -f tsim/parser/columnValue_bool.sim -./test.sh -f tsim/parser/columnValue_double.sim -./test.sh -f tsim/parser/columnValue_float.sim -./test.sh -f tsim/parser/columnValue_int.sim -./test.sh -f tsim/parser/columnValue_smallint.sim -./test.sh -f tsim/parser/columnValue_tinyint.sim -./test.sh -f tsim/parser/columnValue_unsign.sim -./test.sh -f tsim/parser/commit.sim -./test.sh -f tsim/parser/condition.sim -./test.sh -f tsim/parser/constCol.sim -./test.sh -f tsim/parser/create_db.sim -./test.sh -f tsim/parser/create_mt.sim -./test.sh -f tsim/parser/create_tb_with_tag_name.sim -./test.sh -f tsim/parser/create_tb.sim -./test.sh -f tsim/parser/dbtbnameValidate.sim -./test.sh -f tsim/parser/distinct.sim -# TD-17623 ./test.sh -f tsim/parser/fill_stb.sim -./test.sh -f tsim/parser/fill_us.sim -./test.sh -f tsim/parser/fill.sim -./test.sh -f tsim/parser/first_last.sim -./test.sh -f tsim/parser/fourArithmetic-basic.sim -./test.sh -f tsim/parser/function.sim -./test.sh -f tsim/parser/groupby-basic.sim -./test.sh -f tsim/parser/groupby.sim -./test.sh -f tsim/parser/having_child.sim -./test.sh -f tsim/parser/having.sim -./test.sh -f tsim/parser/import_commit1.sim -./test.sh -f tsim/parser/import_commit2.sim -./test.sh -f tsim/parser/import_commit3.sim -./test.sh -f tsim/parser/import_file.sim -./test.sh -f tsim/parser/import.sim -./test.sh -f tsim/parser/insert_multiTbl.sim -./test.sh -f tsim/parser/insert_tb.sim -# TD-18293 ./test.sh -f tsim/parser/interp.sim -./test.sh -f tsim/parser/join_manyblocks.sim -./test.sh -f tsim/parser/join_multitables.sim -./test.sh -f tsim/parser/join_multivnode.sim -./test.sh -f tsim/parser/join.sim -./test.sh -f tsim/parser/last_cache.sim -./test.sh -f tsim/parser/last_groupby.sim -./test.sh -f tsim/parser/lastrow.sim -./test.sh -f tsim/parser/lastrow2.sim -./test.sh -f tsim/parser/like.sim -./test.sh -f tsim/parser/limit.sim -./test.sh -f tsim/parser/limit1.sim -# TD-17623 ./test.sh -f tsim/parser/limit2.sim -./test.sh -f tsim/parser/mixed_blocks.sim -./test.sh -f tsim/parser/nchar.sim -./test.sh -f tsim/parser/nestquery.sim -./test.sh -f tsim/parser/null_char.sim -./test.sh -f tsim/parser/precision_ns.sim -./test.sh -f tsim/parser/projection_limit_offset.sim -./test.sh -f tsim/parser/regex.sim -./test.sh -f tsim/parser/select_across_vnodes.sim -./test.sh -f tsim/parser/select_distinct_tag.sim -./test.sh -f tsim/parser/select_from_cache_disk.sim -./test.sh -f tsim/parser/select_with_tags.sim -./test.sh -f tsim/parser/selectResNum.sim -./test.sh -f tsim/parser/set_tag_vals.sim -./test.sh -f tsim/parser/single_row_in_tb.sim -./test.sh -f tsim/parser/sliding.sim -./test.sh -f tsim/parser/slimit_alter_tags.sim -./test.sh -f tsim/parser/slimit.sim -./test.sh -f tsim/parser/slimit1.sim -./test.sh -f tsim/parser/stableOp.sim -./test.sh -f tsim/parser/tags_dynamically_specifiy.sim -./test.sh -f tsim/parser/tags_filter.sim -./test.sh -f tsim/parser/tbnameIn.sim -./test.sh -f tsim/parser/timestamp.sim -./test.sh -f tsim/parser/top_groupby.sim -./test.sh -f tsim/parser/topbot.sim -./test.sh -f tsim/parser/union.sim -./test.sh -f tsim/parser/union_sysinfo.sim -./test.sh -f tsim/parser/where.sim - -# ---- query ---- -./test.sh -f tsim/query/charScalarFunction.sim -./test.sh -f tsim/query/explain.sim -./test.sh -f tsim/query/interval-offset.sim -./test.sh -f tsim/query/interval.sim -./test.sh -f tsim/query/scalarFunction.sim -./test.sh -f tsim/query/scalarNull.sim -./test.sh -f tsim/query/session.sim -./test.sh -f tsim/query/udf.sim - -# ---- qnode -./test.sh -f tsim/qnode/basic1.sim - -# ---- snode ---- -./test.sh -f tsim/snode/basic1.sim - -# ---- mnode -./test.sh -f tsim/mnode/basic1.sim -./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim -./test.sh -f tsim/mnode/basic5.sim - -# ---- show ---- -./test.sh -f tsim/show/basic.sim - -# ---- table ---- -./test.sh -f tsim/table/autocreate.sim -./test.sh -f tsim/table/basic1.sim -./test.sh -f tsim/table/basic2.sim -./test.sh -f tsim/table/basic3.sim -./test.sh -f tsim/table/bigint.sim -./test.sh -f tsim/table/binary.sim -./test.sh -f tsim/table/bool.sim -./test.sh -f tsim/table/column_name.sim -./test.sh -f tsim/table/column_num.sim -./test.sh -f tsim/table/column_value.sim -./test.sh -f tsim/table/column2.sim -./test.sh -f tsim/table/createmulti.sim -./test.sh -f tsim/table/date.sim -./test.sh -f tsim/table/db.table.sim -./test.sh -f tsim/table/delete_reuse1.sim -./test.sh -f tsim/table/delete_reuse2.sim -./test.sh -f tsim/table/delete_writing.sim -./test.sh -f tsim/table/describe.sim -./test.sh -f tsim/table/double.sim -./test.sh -f tsim/table/float.sim -./test.sh -f tsim/table/hash.sim -./test.sh -f tsim/table/int.sim -./test.sh -f tsim/table/limit.sim -./test.sh -f tsim/table/smallint.sim -./test.sh -f tsim/table/table_len.sim -./test.sh -f tsim/table/table.sim -./test.sh -f tsim/table/tinyint.sim -./test.sh -f tsim/table/vgroup.sim - -# ---- stream -./test.sh -f tsim/stream/basic0.sim -g -# TD-20201 ./test.sh -f tsim/stream/basic1.sim -./test.sh -f tsim/stream/basic2.sim -./test.sh -f tsim/stream/drop_stream.sim -./test.sh -f tsim/stream/fillHistoryBasic1.sim -./test.sh -f tsim/stream/fillHistoryBasic2.sim -./test.sh -f tsim/stream/fillHistoryBasic3.sim -./test.sh -f tsim/stream/distributeInterval0.sim -./test.sh -f tsim/stream/distributeIntervalRetrive0.sim -./test.sh -f tsim/stream/distributeSession0.sim -./test.sh -f tsim/stream/session0.sim -./test.sh -f tsim/stream/session1.sim -./test.sh -f tsim/stream/state0.sim -./test.sh -f tsim/stream/triggerInterval0.sim -./test.sh -f tsim/stream/triggerSession0.sim -./test.sh -f tsim/stream/partitionby.sim -./test.sh -f tsim/stream/partitionby1.sim -./test.sh -f tsim/stream/schedSnode.sim -./test.sh -f tsim/stream/windowClose.sim -./test.sh -f tsim/stream/ignoreExpiredData.sim -./test.sh -f tsim/stream/sliding.sim -./test.sh -f tsim/stream/partitionbyColumnInterval.sim -./test.sh -f tsim/stream/partitionbyColumnSession.sim -./test.sh -f tsim/stream/partitionbyColumnState.sim -./test.sh -f tsim/stream/deleteInterval.sim -./test.sh -f tsim/stream/deleteSession.sim -./test.sh -f tsim/stream/deleteState.sim -./test.sh -f tsim/stream/fillIntervalDelete0.sim -./test.sh -f tsim/stream/fillIntervalDelete1.sim -./test.sh -f tsim/stream/fillIntervalLinear.sim -./test.sh -f tsim/stream/fillIntervalPartitionBy.sim -./test.sh -f tsim/stream/fillIntervalPrevNext.sim -./test.sh -f tsim/stream/fillIntervalValue.sim - -# ---- transaction ---- -./test.sh -f tsim/trans/lossdata1.sim -./test.sh -f tsim/trans/create_db.sim - -# ---- tmq -./test.sh -f tsim/tmq/basic1.sim -./test.sh -f tsim/tmq/basic2.sim -./test.sh -f tsim/tmq/basic3.sim -./test.sh -f tsim/tmq/basic4.sim -./test.sh -f tsim/tmq/basic1Of2Cons.sim -./test.sh -f tsim/tmq/basic2Of2Cons.sim -./test.sh -f tsim/tmq/basic3Of2Cons.sim -./test.sh -f tsim/tmq/basic4Of2Cons.sim -./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim -./test.sh -f tsim/tmq/topic.sim -./test.sh -f tsim/tmq/snapshot.sim -./test.sh -f tsim/tmq/snapshot1.sim - -# --- stable ---- -./test.sh -f tsim/stable/alter_comment.sim -./test.sh -f tsim/stable/alter_count.sim -./test.sh -f tsim/stable/alter_import.sim -./test.sh -f tsim/stable/alter_insert1.sim -./test.sh -f tsim/stable/alter_insert2.sim -./test.sh -f tsim/stable/alter_metrics.sim -./test.sh -f tsim/stable/column_add.sim -./test.sh -f tsim/stable/column_drop.sim -./test.sh -f tsim/stable/column_modify.sim -./test.sh -f tsim/stable/disk.sim -./test.sh -f tsim/stable/dnode3.sim -./test.sh -f tsim/stable/metrics.sim -./test.sh -f tsim/stable/refcount.sim -./test.sh -f tsim/stable/tag_add.sim -./test.sh -f tsim/stable/tag_drop.sim -./test.sh -f tsim/stable/tag_filter.sim -./test.sh -f tsim/stable/tag_modify.sim -./test.sh -f tsim/stable/tag_rename.sim -./test.sh -f tsim/stable/values.sim -./test.sh -f tsim/stable/vnode3.sim -./test.sh -f tsim/stable/metrics_idx.sim - -# --- sma -./test.sh -f tsim/sma/drop_sma.sim -./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim - -# --- valgrind ---- -./test.sh -f tsim/valgrind/checkError1.sim -./test.sh -f tsim/valgrind/checkError2.sim -./test.sh -f tsim/valgrind/checkError3.sim -./test.sh -f tsim/valgrind/checkError4.sim -./test.sh -f tsim/valgrind/checkError5.sim -./test.sh -f tsim/valgrind/checkError6.sim -./test.sh -f tsim/valgrind/checkError7.sim -./test.sh -f tsim/valgrind/checkError8.sim -./test.sh -f tsim/valgrind/checkUdf.sim - -# --- vnode ---- -./test.sh -f tsim/vnode/replica3_basic.sim -./test.sh -f tsim/vnode/replica3_repeat.sim -./test.sh -f tsim/vnode/replica3_vgroup.sim -./test.sh -f tsim/vnode/replica3_many.sim -./test.sh -f tsim/vnode/replica3_import.sim -./test.sh -f tsim/vnode/stable_balance_replica1.sim -./test.sh -f tsim/vnode/stable_dnode2_stop.sim -./test.sh -f tsim/vnode/stable_dnode2.sim -./test.sh -f tsim/vnode/stable_dnode3.sim -./test.sh -f tsim/vnode/stable_replica3_dnode6.sim -./test.sh -f tsim/vnode/stable_replica3_vnode3.sim - -# --- sync -./test.sh -f tsim/sync/3Replica1VgElect.sim -./test.sh -f tsim/sync/3Replica5VgElect.sim -./test.sh -f tsim/sync/oneReplica1VgElect.sim -./test.sh -f tsim/sync/oneReplica5VgElect.sim - -# --- catalog ---- -./test.sh -f tsim/catalog/alterInCurrent.sim - -# --- scalar ---- -./test.sh -f tsim/scalar/in.sim -./test.sh -f tsim/scalar/scalar.sim -./test.sh -f tsim/scalar/filter.sim -./test.sh -f tsim/scalar/caseWhen.sim - -# ---- alter ---- -./test.sh -f tsim/alter/cached_schema_after_alter.sim -./test.sh -f tsim/alter/dnode.sim -./test.sh -f tsim/alter/table.sim - -# ---- cache ---- -./test.sh -f tsim/cache/new_metrics.sim -./test.sh -f tsim/cache/restart_table.sim -./test.sh -f tsim/cache/restart_metrics.sim - -# ---- column ---- -./test.sh -f tsim/column/commit.sim -./test.sh -f tsim/column/metrics.sim -./test.sh -f tsim/column/table.sim - -# ---- compress ---- -./test.sh -f tsim/compress/commitlog.sim -./test.sh -f tsim/compress/compress2.sim -./test.sh -f tsim/compress/compress.sim -./test.sh -f tsim/compress/uncompress.sim - -# ---- compute ---- -./test.sh -f tsim/compute/avg.sim -./test.sh -f tsim/compute/block_dist.sim -./test.sh -f tsim/compute/bottom.sim -./test.sh -f tsim/compute/count.sim -./test.sh -f tsim/compute/diff.sim -./test.sh -f tsim/compute/diff2.sim -./test.sh -f tsim/compute/first.sim -./test.sh -f tsim/compute/interval.sim -./test.sh -f tsim/compute/last_row.sim -./test.sh -f tsim/compute/last.sim -./test.sh -f tsim/compute/leastsquare.sim -./test.sh -f tsim/compute/max.sim -./test.sh -f tsim/compute/min.sim -./test.sh -f tsim/compute/null.sim -./test.sh -f tsim/compute/percentile.sim -./test.sh -f tsim/compute/stddev.sim -./test.sh -f tsim/compute/sum.sim -./test.sh -f tsim/compute/top.sim - -# ---- field ---- -./test.sh -f tsim/field/2.sim -./test.sh -f tsim/field/3.sim -./test.sh -f tsim/field/4.sim -./test.sh -f tsim/field/5.sim -./test.sh -f tsim/field/6.sim -./test.sh -f tsim/field/binary.sim -./test.sh -f tsim/field/bigint.sim -./test.sh -f tsim/field/bool.sim -./test.sh -f tsim/field/double.sim -./test.sh -f tsim/field/float.sim -./test.sh -f tsim/field/int.sim -./test.sh -f tsim/field/single.sim -./test.sh -f tsim/field/smallint.sim -./test.sh -f tsim/field/tinyint.sim -./test.sh -f tsim/field/unsigined_bigint.sim - -# ---- vector ---- -./test.sh -f tsim/vector/metrics_field.sim -./test.sh -f tsim/vector/metrics_mix.sim -./test.sh -f tsim/vector/metrics_query.sim -./test.sh -f tsim/vector/metrics_tag.sim -./test.sh -f tsim/vector/metrics_time.sim -./test.sh -f tsim/vector/multi.sim -./test.sh -f tsim/vector/single.sim -./test.sh -f tsim/vector/table_field.sim -./test.sh -f tsim/vector/table_mix.sim -./test.sh -f tsim/vector/table_query.sim -./test.sh -f tsim/vector/table_time.sim - -# ---- wal ---- -./test.sh -f tsim/wal/kill.sim - -# ---- tag ---- -./test.sh -f tsim/tag/3.sim -./test.sh -f tsim/tag/4.sim -./test.sh -f tsim/tag/5.sim -./test.sh -f tsim/tag/6.sim -./test.sh -f tsim/tag/add.sim -./test.sh -f tsim/tag/bigint.sim -./test.sh -f tsim/tag/binary_binary.sim -./test.sh -f tsim/tag/binary.sim -./test.sh -f tsim/tag/bool_binary.sim -./test.sh -f tsim/tag/bool_int.sim -./test.sh -f tsim/tag/bool.sim -./test.sh -f tsim/tag/change.sim -./test.sh -f tsim/tag/column.sim -./test.sh -f tsim/tag/commit.sim -./test.sh -f tsim/tag/create.sim -./test.sh -f tsim/tag/delete.sim -./test.sh -f tsim/tag/double.sim -./test.sh -f tsim/tag/filter.sim -./test.sh -f tsim/tag/float.sim -./test.sh -f tsim/tag/int_binary.sim -./test.sh -f tsim/tag/int_float.sim -./test.sh -f tsim/tag/int.sim -./test.sh -f tsim/tag/set.sim -./test.sh -f tsim/tag/smallint.sim -./test.sh -f tsim/tag/tinyint.sim -./test.sh -f tsim/tag/drop_tag.sim -./test.sh -f tsim/tag/tbNameIn.sim - -./test.sh -f tmp/monitor.sim -#======================b1-end=============== diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index b6139cea13..63018aea8c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -38,7 +38,7 @@ endi print ============= create database #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | WAL_FSYNC_PERIOD value [0 ~ 180000 ms] @@ -78,7 +78,7 @@ endi if $data7_db != 1440000m,1440000m,1440000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 5ac2ee6c4e..7012fbac6c 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -37,7 +37,7 @@ endi print ============= create database with all options #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] @@ -98,7 +98,7 @@ endi if $data7_db != 5256000m,5256000m,5256000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index ae78c806ca..db9a25365e 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -733,6 +733,7 @@ sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 group by f1 having sum(f1) > 2 order by f1; if $rows != 3 then + print expect 3 , actual: $rows return -1 endi if $data00 != 2.000000000 then diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 1918c0ef76..aec18a1cf1 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -10,37 +10,43 @@ from util.sql import * from util.cases import * from util.dnodes import * + class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) - self.buffer_boundary = [3,4097,8193,12289,16384] - self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + tdSql.init(conn.cursor(), logSql) + self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + self.buffer_error = [self.buffer_boundary[0] - + 1, self.buffer_boundary[-1]+1, 12289, 256] # pages_boundary >= 64 - self.pages_boundary = [64,128,512] + self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: tdSql.execute(f'alter database db buffer {buffer}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8], buffer) tdSql.execute('drop database db') tdSql.execute('create database db vgroups 10') for buffer in self.buffer_error: tdSql.error(f'alter database db buffer {buffer}') tdSql.execute('drop database db') - + def alter_pages(self): tdSql.execute('create database db') for pages in self.pages_boundary: tdSql.execute(f'alter database db pages {pages}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10], pages) tdSql.execute('drop database db') tdSql.execute('create database db') - tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') @@ -55,5 +61,6 @@ class TDTestCase: tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh deleted file mode 100755 index 0729d36f12..0000000000 --- a/tests/system-test/fulltest.sh +++ /dev/null @@ -1,656 +0,0 @@ -#!/bin/bash -set -e -set -x - -python3 ./test.py -f 0-others/taosShell.py -python3 ./test.py -f 0-others/taosShellError.py -python3 ./test.py -f 0-others/taosShellNetChk.py -python3 ./test.py -f 0-others/telemetry.py -python3 ./test.py -f 0-others/taosdMonitor.py -python3 ./test.py -f 0-others/udfTest.py -python3 ./test.py -f 0-others/udf_create.py -python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachemodel.py -python3 ./test.py -f 0-others/udf_cfg1.py -python3 ./test.py -f 0-others/udf_cfg2.py -python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 -python3 ./test.py -f 0-others/sysinfo.py -python3 ./test.py -f 0-others/user_control.py -python3 ./test.py -f 0-others/fsync.py -python3 ./test.py -f 0-others/compatibility.py -python3 ./test.py -f 1-insert/alter_database.py -python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -python3 ./test.py -f 1-insert/alter_stable.py -python3 ./test.py -f 1-insert/alter_table.py -python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -python3 ./test.py -f 1-insert/table_comment.py -python3 ./test.py -f 1-insert/time_range_wise.py -python3 ./test.py -f 1-insert/block_wise.py -python3 ./test.py -f 1-insert/create_retentions.py -python3 ./test.py -f 1-insert/table_param_ttl.py -python3 ./test.py -f 1-insert/mutil_stage.py -python3 ./test.py -f 1-insert/table_param_ttl.py -R -python3 ./test.py -f 1-insert/update_data_muti_rows.py -python3 ./test.py -f 1-insert/db_tb_name_check.py -python3 ./test.py -f 1-insert/database_pre_suf.py -python3 ./test.py -f 0-others/show.py -python3 ./test.py -f 2-query/abs.py -python3 ./test.py -f 2-query/abs.py -R -python3 ./test.py -f 2-query/and_or_for_byte.py -python3 ./test.py -f 2-query/and_or_for_byte.py -R -python3 ./test.py -f 2-query/apercentile.py -python3 ./test.py -f 2-query/apercentile.py -R -python3 ./test.py -f 2-query/arccos.py -python3 ./test.py -f 2-query/arccos.py -R -python3 ./test.py -f 2-query/arcsin.py -python3 ./test.py -f 2-query/arcsin.py -R -python3 ./test.py -f 2-query/arctan.py -python3 ./test.py -f 2-query/arctan.py -R -python3 ./test.py -f 2-query/avg.py -python3 ./test.py -f 2-query/avg.py -R -python3 ./test.py -f 2-query/between.py -python3 ./test.py -f 2-query/between.py -R -python3 ./test.py -f 2-query/bottom.py -python3 ./test.py -f 2-query/bottom.py -R -python3 ./test.py -f 2-query/cast.py -python3 ./test.py -f 2-query/cast.py -R -python3 ./test.py -f 2-query/ceil.py -python3 ./test.py -f 2-query/ceil.py -R -python3 ./test.py -f 2-query/char_length.py -python3 ./test.py -f 2-query/char_length.py -R -python3 ./test.py -f 2-query/check_tsdb.py -python3 ./test.py -f 2-query/check_tsdb.py -R -python3 ./test.py -f 2-query/concat.py -python3 ./test.py -f 2-query/concat.py -R -python3 ./test.py -f 2-query/concat_ws.py -python3 ./test.py -f 2-query/concat_ws.py -R -python3 ./test.py -f 2-query/concat_ws2.py -python3 ./test.py -f 2-query/concat_ws2.py -R -python3 ./test.py -f 2-query/cos.py -python3 ./test.py -f 2-query/cos.py -R -python3 ./test.py -f 2-query/count_partition.py -python3 ./test.py -f 2-query/count_partition.py -R -python3 ./test.py -f 2-query/count.py -python3 ./test.py -f 2-query/count.py -R -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R -python3 ./test.py -f 2-query/db.py -python3 ./test.py -f 2-query/db.py -R -python3 ./test.py -f 2-query/diff.py -python3 ./test.py -f 2-query/diff.py -R -python3 ./test.py -f 2-query/distinct.py -python3 ./test.py -f 2-query/distinct.py -R -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R -python3 ./test.py -f 2-query/distribute_agg_avg.py -python3 ./test.py -f 2-query/distribute_agg_avg.py -R -python3 ./test.py -f 2-query/distribute_agg_count.py -python3 ./test.py -f 2-query/distribute_agg_count.py -R -python3 ./test.py -f 2-query/distribute_agg_max.py -python3 ./test.py -f 2-query/distribute_agg_max.py -R -python3 ./test.py -f 2-query/distribute_agg_min.py -python3 ./test.py -f 2-query/distribute_agg_min.py -R -python3 ./test.py -f 2-query/distribute_agg_spread.py -python3 ./test.py -f 2-query/distribute_agg_spread.py -R -python3 ./test.py -f 2-query/distribute_agg_stddev.py -python3 ./test.py -f 2-query/distribute_agg_stddev.py -R -python3 ./test.py -f 2-query/distribute_agg_sum.py -python3 ./test.py -f 2-query/distribute_agg_sum.py -R -python3 ./test.py -f 2-query/explain.py -python3 ./test.py -f 2-query/explain.py -R -python3 ./test.py -f 2-query/first.py -python3 ./test.py -f 2-query/first.py -R -python3 ./test.py -f 2-query/floor.py -python3 ./test.py -f 2-query/floor.py -R -python3 ./test.py -f 2-query/function_null.py -python3 ./test.py -f 2-query/function_null.py -R -python3 ./test.py -f 2-query/function_stateduration.py -python3 ./test.py -f 2-query/function_stateduration.py -R -python3 ./test.py -f 2-query/histogram.py -python3 ./test.py -f 2-query/histogram.py -R -python3 ./test.py -f 2-query/hyperloglog.py -python3 ./test.py -f 2-query/hyperloglog.py -R -python3 ./test.py -f 2-query/interp.py -python3 ./test.py -f 2-query/interp.py -R -python3 ./test.py -f 2-query/irate.py -python3 ./test.py -f 2-query/irate.py -R -python3 ./test.py -f 2-query/join.py -python3 ./test.py -f 2-query/join.py -R -python3 ./test.py -f 2-query/last_row.py -python3 ./test.py -f 2-query/last_row.py -R -python3 ./test.py -f 2-query/last.py -python3 ./test.py -f 2-query/last.py -R -python3 ./test.py -f 2-query/leastsquares.py -python3 ./test.py -f 2-query/leastsquares.py -R -python3 ./test.py -f 2-query/length.py -python3 ./test.py -f 2-query/length.py -R -python3 ./test.py -f 2-query/log.py -python3 ./test.py -f 2-query/log.py -R -python3 ./test.py -f 2-query/lower.py -python3 ./test.py -f 2-query/lower.py -R -python3 ./test.py -f 2-query/ltrim.py -python3 ./test.py -f 2-query/ltrim.py -R -python3 ./test.py -f 2-query/mavg.py -python3 ./test.py -f 2-query/mavg.py -R -python3 ./test.py -f 2-query/max_partition.py -python3 ./test.py -f 2-query/max_partition.py -R -python3 ./test.py -f 2-query/max.py -python3 ./test.py -f 2-query/max.py -R -python3 ./test.py -f 2-query/min.py -python3 ./test.py -f 2-query/min.py -R -python3 ./test.py -f 2-query/mode.py -python3 ./test.py -f 2-query/mode.py -R -python3 ./test.py -f 2-query/Now.py -python3 ./test.py -f 2-query/Now.py -R -python3 ./test.py -f 2-query/percentile.py -python3 ./test.py -f 2-query/percentile.py -R -python3 ./test.py -f 2-query/pow.py -python3 ./test.py -f 2-query/pow.py -R -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R -python3 ./test.py -f 2-query/round.py -python3 ./test.py -f 2-query/round.py -R -python3 ./test.py -f 2-query/rtrim.py -python3 ./test.py -f 2-query/rtrim.py -R -python3 ./test.py -f 2-query/sample.py -python3 ./test.py -f 2-query/sample.py -R -python3 ./test.py -f 2-query/sin.py -python3 ./test.py -f 2-query/sin.py -R -python3 ./test.py -f 2-query/smaTest.py -python3 ./test.py -f 2-query/smaTest.py -R -python3 ./test.py -f 2-query/sml.py -python3 ./test.py -f 2-query/sml.py -R -python3 ./test.py -f 2-query/spread.py -python3 ./test.py -f 2-query/spread.py -R -python3 ./test.py -f 2-query/sqrt.py -python3 ./test.py -f 2-query/sqrt.py -R -python3 ./test.py -f 2-query/statecount.py -python3 ./test.py -f 2-query/statecount.py -R -python3 ./test.py -f 2-query/stateduration.py -python3 ./test.py -f 2-query/stateduration.py -R -python3 ./test.py -f 2-query/substr.py -python3 ./test.py -f 2-query/substr.py -R -python3 ./test.py -f 2-query/sum.py -python3 ./test.py -f 2-query/sum.py -R -python3 ./test.py -f 2-query/tail.py -python3 ./test.py -f 2-query/tail.py -R -python3 ./test.py -f 2-query/tan.py -python3 ./test.py -f 2-query/tan.py -R -python3 ./test.py -f 2-query/Timediff.py -python3 ./test.py -f 2-query/Timediff.py -R -python3 ./test.py -f 2-query/timetruncate.py -python3 ./test.py -f 2-query/timetruncate.py -R -python3 ./test.py -f 2-query/timezone.py -python3 ./test.py -f 2-query/timezone.py -R -python3 ./test.py -f 2-query/To_iso8601.py -python3 ./test.py -f 2-query/To_iso8601.py -R -python3 ./test.py -f 2-query/To_unixtimestamp.py -python3 ./test.py -f 2-query/To_unixtimestamp.py -R -python3 ./test.py -f 2-query/Today.py -python3 ./test.py -f 2-query/Today.py -R -python3 ./test.py -f 2-query/top.py -python3 ./test.py -f 2-query/top.py -R -python3 ./test.py -f 2-query/tsbsQuery.py -python3 ./test.py -f 2-query/tsbsQuery.py -R -python3 ./test.py -f 2-query/ttl_comment.py -python3 ./test.py -f 2-query/ttl_comment.py -R -python3 ./test.py -f 2-query/twa.py -python3 ./test.py -f 2-query/twa.py -R -python3 ./test.py -f 2-query/union.py -python3 ./test.py -f 2-query/union.py -R -python3 ./test.py -f 2-query/unique.py -python3 ./test.py -f 2-query/unique.py -R -python3 ./test.py -f 2-query/upper.py -python3 ./test.py -f 2-query/upper.py -R -python3 ./test.py -f 2-query/varchar.py -python3 ./test.py -f 2-query/varchar.py -R - - -python3 ./test.py -f 1-insert/update_data.py -python3 ./test.py -f 1-insert/tb_100w_data_order.py - -# TD-20200 python3 ./test.py -f 1-insert/delete_data.py -python3 ./test.py -f 1-insert/delete_stable.py -python3 ./test.py -f 1-insert/delete_childtable.py -python3 ./test.py -f 1-insert/delete_normaltable.py -python3 ./test.py -f 1-insert/keep_expired.py - -python3 ./test.py -f 2-query/join2.py -python3 ./test.py -f 2-query/union1.py -python3 ./test.py -f 2-query/concat2.py - - -python3 ./test.py -f 2-query/json_tag.py - -python3 ./test.py -f 2-query/nestedQuery.py -python3 ./test.py -f 2-query/nestedQuery_str.py -python3 ./test.py -f 2-query/nestedQuery_math.py -python3 ./test.py -f 2-query/nestedQuery_time.py -python3 ./test.py -f 2-query/stablity.py -python3 ./test.py -f 2-query/stablity_1.py - -python3 ./test.py -f 2-query/elapsed.py -python3 ./test.py -f 2-query/csum.py -python3 ./test.py -f 2-query/function_diff.py -python3 ./test.py -f 2-query/queryQnode.py - -python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False -python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 - - -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 - -# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -# TD-19646 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 - -python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 - -# vnode case -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py -N 4 -M 1 -python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - -python3 ./test.py -f 7-tmq/create_wrong_topic.py -python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -python3 ./test.py -f 7-tmq/basic5.py -python3 ./test.py -f 7-tmq/subscribeDb.py -python3 ./test.py -f 7-tmq/subscribeDb0.py -python3 ./test.py -f 7-tmq/subscribeDb1.py -python3 ./test.py -f 7-tmq/subscribeDb2.py -python3 ./test.py -f 7-tmq/subscribeDb3.py -python3 ./test.py -f 7-tmq/subscribeDb4.py -python3 ./test.py -f 7-tmq/subscribeStb.py -python3 ./test.py -f 7-tmq/subscribeStb0.py -python3 ./test.py -f 7-tmq/subscribeStb1.py -python3 ./test.py -f 7-tmq/subscribeStb2.py -python3 ./test.py -f 7-tmq/subscribeStb3.py -python3 ./test.py -f 7-tmq/subscribeStb4.py -python3 ./test.py -f 7-tmq/db.py -python3 ./test.py -f 7-tmq/tmqError.py -python3 ./test.py -f 7-tmq/schema.py -python3 ./test.py -f 7-tmq/stbFilter.py -python3 ./test.py -f 7-tmq/tmqCheckData.py -python3 ./test.py -f 7-tmq/tmqCheckData1.py -#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 -python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -python3 ./test.py -f 7-tmq/tmqShow.py -python3 ./test.py -f 7-tmq/tmqAlterSchema.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -python3 ./test.py -f 7-tmq/tmqDropStb.py -python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -python3 ./test.py -f 7-tmq/tmqUdf.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -python3 ./test.py -f 7-tmq/tmq_taosx.py -python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py - -python3 ./test.py -f 99-TDcase/TD-19201.py - -#------------querPolicy 2----------- - -python3 ./test.py -f 2-query/between.py -Q 2 -python3 ./test.py -f 2-query/distinct.py -Q 2 -python3 ./test.py -f 2-query/varchar.py -Q 2 -python3 ./test.py -f 2-query/ltrim.py -Q 2 -python3 ./test.py -f 2-query/rtrim.py -Q 2 -python3 ./test.py -f 2-query/length.py -Q 2 -python3 ./test.py -f 2-query/char_length.py -Q 2 -python3 ./test.py -f 2-query/upper.py -Q 2 -python3 ./test.py -f 2-query/lower.py -Q 2 -python3 ./test.py -f 2-query/join.py -Q 2 -python3 ./test.py -f 2-query/join2.py -Q 2 -python3 ./test.py -f 2-query/cast.py -Q 2 -python3 ./test.py -f 2-query/substr.py -Q 2 -python3 ./test.py -f 2-query/union.py -Q 2 -python3 ./test.py -f 2-query/union1.py -Q 2 -python3 ./test.py -f 2-query/concat.py -Q 2 -python3 ./test.py -f 2-query/concat2.py -Q 2 -python3 ./test.py -f 2-query/concat_ws.py -Q 2 -python3 ./test.py -f 2-query/concat_ws2.py -Q 2 -python3 ./test.py -f 2-query/check_tsdb.py -Q 2 -python3 ./test.py -f 2-query/spread.py -Q 2 -python3 ./test.py -f 2-query/hyperloglog.py -Q 2 -python3 ./test.py -f 2-query/explain.py -Q 2 -python3 ./test.py -f 2-query/leastsquares.py -Q 2 -python3 ./test.py -f 2-query/timezone.py -Q 2 -python3 ./test.py -f 2-query/Now.py -Q 2 -python3 ./test.py -f 2-query/Today.py -Q 2 -python3 ./test.py -f 2-query/max.py -Q 2 -python3 ./test.py -f 2-query/min.py -Q 2 -python3 ./test.py -f 2-query/mode.py -Q 2 -python3 ./test.py -f 2-query/count.py -Q 2 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 -python3 ./test.py -f 2-query/last.py -Q 2 -python3 ./test.py -f 2-query/first.py -Q 2 -python3 ./test.py -f 2-query/To_iso8601.py -Q 2 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 -python3 ./test.py -f 2-query/timetruncate.py -Q 2 -python3 ./test.py -f 2-query/diff.py -Q 2 -python3 ./test.py -f 2-query/Timediff.py -Q 2 -python3 ./test.py -f 2-query/json_tag.py -Q 2 -python3 ./test.py -f 2-query/top.py -Q 2 -python3 ./test.py -f 2-query/bottom.py -Q 2 -python3 ./test.py -f 2-query/percentile.py -Q 2 -python3 ./test.py -f 2-query/apercentile.py -Q 2 -python3 ./test.py -f 2-query/abs.py -Q 2 -python3 ./test.py -f 2-query/ceil.py -Q 2 -python3 ./test.py -f 2-query/floor.py -Q 2 -python3 ./test.py -f 2-query/round.py -Q 2 -python3 ./test.py -f 2-query/log.py -Q 2 -python3 ./test.py -f 2-query/pow.py -Q 2 -python3 ./test.py -f 2-query/sqrt.py -Q 2 -python3 ./test.py -f 2-query/sin.py -Q 2 -python3 ./test.py -f 2-query/cos.py -Q 2 -python3 ./test.py -f 2-query/tan.py -Q 2 -python3 ./test.py -f 2-query/arcsin.py -Q 2 -python3 ./test.py -f 2-query/arccos.py -Q 2 -python3 ./test.py -f 2-query/arctan.py -Q 2 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -python3 ./test.py -f 2-query/interp.py -Q 2 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -python3 ./test.py -f 2-query/stablity.py -Q 2 -python3 ./test.py -f 2-query/stablity_1.py -Q 2 - -python3 ./test.py -f 2-query/avg.py -Q 2 -python3 ./test.py -f 2-query/elapsed.py -Q 2 -python3 ./test.py -f 2-query/csum.py -Q 2 -python3 ./test.py -f 2-query/mavg.py -Q 2 -python3 ./test.py -f 2-query/sample.py -Q 2 -python3 ./test.py -f 2-query/function_diff.py -Q 2 -python3 ./test.py -f 2-query/unique.py -Q 2 -python3 ./test.py -f 2-query/stateduration.py -Q 2 -python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -python3 ./test.py -f 2-query/statecount.py -Q 2 -python3 ./test.py -f 2-query/tail.py -Q 2 -python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -python3 ./test.py -f 2-query/twa.py -Q 2 -python3 ./test.py -f 2-query/irate.py -Q 2 -python3 ./test.py -f 2-query/function_null.py -Q 2 -python3 ./test.py -f 2-query/count_partition.py -Q 2 -python3 ./test.py -f 2-query/max_partition.py -Q 2 -python3 ./test.py -f 2-query/last_row.py -Q 2 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 -python3 ./test.py -f 2-query/sml.py -Q 2 - -#------------querPolicy 3----------- -python3 ./test.py -f 2-query/between.py -Q 3 -python3 ./test.py -f 2-query/distinct.py -Q 3 -python3 ./test.py -f 2-query/varchar.py -Q 3 -python3 ./test.py -f 2-query/ltrim.py -Q 3 -python3 ./test.py -f 2-query/rtrim.py -Q 3 -python3 ./test.py -f 2-query/length.py -Q 3 -python3 ./test.py -f 2-query/char_length.py -Q 3 -python3 ./test.py -f 2-query/upper.py -Q 3 -python3 ./test.py -f 2-query/lower.py -Q 3 -python3 ./test.py -f 2-query/join.py -Q 3 -python3 ./test.py -f 2-query/join2.py -Q 3 -python3 ./test.py -f 2-query/cast.py -Q 3 -python3 ./test.py -f 2-query/substr.py -Q 3 -python3 ./test.py -f 2-query/union.py -Q 3 -python3 ./test.py -f 2-query/union1.py -Q 3 -python3 ./test.py -f 2-query/concat.py -Q 3 -python3 ./test.py -f 2-query/concat2.py -Q 3 -python3 ./test.py -f 2-query/concat_ws.py -Q 3 -python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -python3 ./test.py -f 2-query/spread.py -Q 3 -python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -python3 ./test.py -f 2-query/explain.py -Q 3 -python3 ./test.py -f 2-query/leastsquares.py -Q 3 -python3 ./test.py -f 2-query/timezone.py -Q 3 -python3 ./test.py -f 2-query/Now.py -Q 3 -python3 ./test.py -f 2-query/Today.py -Q 3 -python3 ./test.py -f 2-query/max.py -Q 3 -python3 ./test.py -f 2-query/min.py -Q 3 -python3 ./test.py -f 2-query/mode.py -Q 3 -python3 ./test.py -f 2-query/count.py -Q 3 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 -python3 ./test.py -f 2-query/last.py -Q 3 -python3 ./test.py -f 2-query/first.py -Q 3 -python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -python3 ./test.py -f 2-query/timetruncate.py -Q 3 -python3 ./test.py -f 2-query/diff.py -Q 3 -python3 ./test.py -f 2-query/Timediff.py -Q 3 -python3 ./test.py -f 2-query/json_tag.py -Q 3 -python3 ./test.py -f 2-query/top.py -Q 3 -python3 ./test.py -f 2-query/bottom.py -Q 3 -python3 ./test.py -f 2-query/percentile.py -Q 3 -python3 ./test.py -f 2-query/apercentile.py -Q 3 -python3 ./test.py -f 2-query/abs.py -Q 3 -python3 ./test.py -f 2-query/ceil.py -Q 3 -python3 ./test.py -f 2-query/floor.py -Q 3 -python3 ./test.py -f 2-query/round.py -Q 3 -python3 ./test.py -f 2-query/log.py -Q 3 -python3 ./test.py -f 2-query/pow.py -Q 3 -python3 ./test.py -f 2-query/sqrt.py -Q 3 -python3 ./test.py -f 2-query/sin.py -Q 3 -python3 ./test.py -f 2-query/cos.py -Q 3 -python3 ./test.py -f 2-query/tan.py -Q 3 -python3 ./test.py -f 2-query/arcsin.py -Q 3 -python3 ./test.py -f 2-query/arccos.py -Q 3 -python3 ./test.py -f 2-query/arctan.py -Q 3 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -python3 ./test.py -f 2-query/stablity.py -Q 3 -python3 ./test.py -f 2-query/stablity_1.py -Q 3 - -python3 ./test.py -f 2-query/avg.py -Q 3 -python3 ./test.py -f 2-query/elapsed.py -Q 3 -python3 ./test.py -f 2-query/csum.py -Q 3 -python3 ./test.py -f 2-query/mavg.py -Q 3 -python3 ./test.py -f 2-query/sample.py -Q 3 -python3 ./test.py -f 2-query/function_diff.py -Q 3 -python3 ./test.py -f 2-query/unique.py -Q 3 -python3 ./test.py -f 2-query/stateduration.py -Q 3 -python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -python3 ./test.py -f 2-query/statecount.py -Q 3 -python3 ./test.py -f 2-query/tail.py -Q 3 -python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -python3 ./test.py -f 2-query/twa.py -Q 3 -python3 ./test.py -f 2-query/irate.py -Q 3 -python3 ./test.py -f 2-query/function_null.py -Q 3 -python3 ./test.py -f 2-query/count_partition.py -Q 3 -python3 ./test.py -f 2-query/max_partition.py -Q 3 -python3 ./test.py -f 2-query/last_row.py -Q 3 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -python3 ./test.py -f 2-query/sml.py -Q 3 -python3 ./test.py -f 2-query/interp.py -Q 3 - - -#------------querPolicy 4----------- - -python3 ./test.py -f 2-query/between.py -Q 4 -python3 ./test.py -f 2-query/distinct.py -Q 4 -python3 ./test.py -f 2-query/varchar.py -Q 4 -python3 ./test.py -f 2-query/ltrim.py -Q 4 -python3 ./test.py -f 2-query/rtrim.py -Q 4 -python3 ./test.py -f 2-query/length.py -Q 4 -python3 ./test.py -f 2-query/char_length.py -Q 4 -python3 ./test.py -f 2-query/upper.py -Q 4 -python3 ./test.py -f 2-query/lower.py -Q 4 -python3 ./test.py -f 2-query/join.py -Q 4 -python3 ./test.py -f 2-query/join2.py -Q 4 -#python3 ./test.py -f 2-query/cast.py -Q 4 -python3 ./test.py -f 2-query/substr.py -Q 4 -python3 ./test.py -f 2-query/union.py -Q 4 -python3 ./test.py -f 2-query/union1.py -Q 4 -python3 ./test.py -f 2-query/concat.py -Q 4 -python3 ./test.py -f 2-query/concat2.py -Q 4 -python3 ./test.py -f 2-query/concat_ws.py -Q 4 -python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -python3 ./test.py -f 2-query/spread.py -Q 4 -python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -python3 ./test.py -f 2-query/explain.py -Q 4 -python3 ./test.py -f 2-query/leastsquares.py -Q 4 -python3 ./test.py -f 2-query/timezone.py -Q 4 -python3 ./test.py -f 2-query/Now.py -Q 4 -python3 ./test.py -f 2-query/Today.py -Q 4 -python3 ./test.py -f 2-query/max.py -Q 4 -python3 ./test.py -f 2-query/min.py -Q 4 -python3 ./test.py -f 2-query/mode.py -Q 4 -python3 ./test.py -f 2-query/count.py -Q 4 -python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 -python3 ./test.py -f 2-query/last.py -Q 4 -python3 ./test.py -f 2-query/first.py -Q 4 -python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -python3 ./test.py -f 2-query/timetruncate.py -Q 4 -python3 ./test.py -f 2-query/diff.py -Q 4 -python3 ./test.py -f 2-query/Timediff.py -Q 4 -python3 ./test.py -f 2-query/json_tag.py -Q 4 -python3 ./test.py -f 2-query/top.py -Q 4 -python3 ./test.py -f 2-query/bottom.py -Q 4 -python3 ./test.py -f 2-query/percentile.py -Q 4 -python3 ./test.py -f 2-query/apercentile.py -Q 4 -python3 ./test.py -f 2-query/abs.py -Q 4 -python3 ./test.py -f 2-query/ceil.py -Q 4 -python3 ./test.py -f 2-query/floor.py -Q 4 -python3 ./test.py -f 2-query/round.py -Q 4 -python3 ./test.py -f 2-query/log.py -Q 4 -python3 ./test.py -f 2-query/pow.py -Q 4 -python3 ./test.py -f 2-query/sqrt.py -Q 4 -python3 ./test.py -f 2-query/sin.py -Q 4 -python3 ./test.py -f 2-query/cos.py -Q 4 -python3 ./test.py -f 2-query/tan.py -Q 4 -python3 ./test.py -f 2-query/arcsin.py -Q 4 -python3 ./test.py -f 2-query/arccos.py -Q 4 -python3 ./test.py -f 2-query/arctan.py -Q 4 -python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 - -python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -python3 ./test.py -f 2-query/stablity.py -Q 4 -python3 ./test.py -f 2-query/stablity_1.py -Q 4 - - -python3 ./test.py -f 2-query/avg.py -Q 4 -python3 ./test.py -f 2-query/elapsed.py -Q 4 -python3 ./test.py -f 2-query/csum.py -Q 4 -python3 ./test.py -f 2-query/mavg.py -Q 4 -python3 ./test.py -f 2-query/sample.py -Q 4 -python3 ./test.py -f 2-query/function_diff.py -Q 4 -python3 ./test.py -f 2-query/unique.py -Q 4 -python3 ./test.py -f 2-query/stateduration.py -Q 4 -python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -python3 ./test.py -f 2-query/statecount.py -Q 4 -python3 ./test.py -f 2-query/tail.py -Q 4 -python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -python3 ./test.py -f 2-query/twa.py -Q 4 -python3 ./test.py -f 2-query/irate.py -Q 4 -python3 ./test.py -f 2-query/function_null.py -Q 4 -python3 ./test.py -f 2-query/count_partition.py -Q 4 -python3 ./test.py -f 2-query/max_partition.py -Q 4 -python3 ./test.py -f 2-query/last_row.py -Q 4 -python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 -python3 ./test.py -f 2-query/sml.py -Q 4 -python3 ./test.py -f 2-query/interp.py -Q 4