diff --git a/.travis.yml b/.travis.yml index 17ede412d2..a86f2463e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ matrix: ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $? cd ${TRAVIS_BUILD_DIR}/tests/pytest - ./smoketest.sh -g 2>&1 | tee mem-error-out.txt + ./valgrind-test.sh -g 2>&1 | tee mem-error-out.txt sleep 1 # Color setting @@ -59,12 +59,12 @@ matrix: GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - grep 'ERROR SUMMARY' mem-error-out.txt | uniq | tee uniq-mem-error-out.txt + grep 'ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` do if [ -n "$memError" ]; then - if [ "$memError" -gt 16 ]; then + if [ "$memError" -gt 5 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ More than our threshold! ## ${NC}" travis_terminate $memError @@ -72,11 +72,11 @@ matrix: fi done - grep 'definitely lost:' mem-error-out.txt | uniq | tee uniq-definitely-lost-out.txt - for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $4}'` + grep 'definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt + for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` do if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 16 ]; then + if [ "$defiMemError" -gt 3 ]; then echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $defiMemError @@ -115,7 +115,7 @@ matrix: # The command that will be added as an argument to "cov-build" to compile your project for analysis, # ** likely specific to your build ** - build_command: make > /dev/null + build_command: make # Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'. # Take care in resource usage, and consider the build frequency allowances per @@ -163,11 +163,9 @@ matrix: cd ${TRAVIS_BUILD_DIR}/tests - ./test-all.sh + ./test-all.sh $TRAVIS_EVENT_TYPE COVER - if [ "$?" -ne "0" ]; then - travis_terminate $? - fi + TEST_RESULT=$? pkill taosd sleep 1 @@ -199,6 +197,9 @@ matrix: echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} " fi + if [ "$TEST_RESULT" -ne "0" ]; then + travis_terminate $? + fi ;; esac diff --git a/README.md b/README.md index 158ae040fa..0babfe7776 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo 1. sudo apt install build-essential cmake net-tools python-pip python-setuptools python3-pip - python3-setuptools valgrind + python3-setuptools valgrind psmisc curl 2. git clone ; cd TDengine diff --git a/src/client/inc/tscLog.h b/src/client/inc/tscLog.h index e82b824123..346e1a6795 100644 --- a/src/client/inc/tscLog.h +++ b/src/client/inc/tscLog.h @@ -22,25 +22,25 @@ extern "C" { #include "tlog.h" -extern int32_t cdebugFlag; +extern int32_t cDebugFlag; #define tscError(...) \ - if (cdebugFlag & DEBUG_ERROR) { \ + if (cDebugFlag & DEBUG_ERROR) { \ taosPrintLog("ERROR TSC ", 255, __VA_ARGS__); \ } #define tscWarn(...) \ - if (cdebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN TSC ", cdebugFlag, __VA_ARGS__); \ + if (cDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN TSC ", cDebugFlag, __VA_ARGS__); \ } #define tscTrace(...) \ - if (cdebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TSC ", cdebugFlag, __VA_ARGS__); \ + if (cDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); \ } #define tscPrint(...) \ { taosPrintLog("TSC ", 255, __VA_ARGS__); } #define tscDump(...) \ - if (cdebugFlag & DEBUG_TRACE) { \ - taosPrintLongString("TSC ", cdebugFlag, __VA_ARGS__); \ + if (cDebugFlag & DEBUG_TRACE) { \ + taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); \ } #ifdef __cplusplus diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 61a5fdd311..6ea1ee6440 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -365,7 +365,7 @@ void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscProcessMsgFromServer(SRpcMsg *rpcMsg); +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewMeterMeta(SSqlObj *pSql, char *tableId); diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 05b7a2be1e..812c197ef7 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -23,16 +23,16 @@ #include "ttime.h" #define jniError(...) \ - if (jnidebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR JNI ", jnidebugFlag, __VA_ARGS__); \ + if (jniDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); \ } #define jniWarn(...) \ - if (jnidebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN JNI ", jnidebugFlag, __VA_ARGS__); \ + if (jniDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN JNI ", jniDebugFlag, __VA_ARGS__); \ } #define jniTrace(...) \ - if (jnidebugFlag & DEBUG_TRACE) { \ - taosPrintLog("JNI ", jnidebugFlag, __VA_ARGS__); \ + if (jniDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ } #define jniPrint(...) \ { taosPrintLog("JNI ", 255, __VA_ARGS__); } diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index c1944e96c9..6fb8df2444 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1691,10 +1691,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - if (pCtx->order == TSDB_ORDER_ASC) { - return; - } - + assert(pCtx->order != TSDB_ORDER_ASC); void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -2912,7 +2909,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { } static void date_col_output_function(SQLFunctionCtx *pCtx) { - if (pCtx->scanFlag == SUPPLEMENTARY_SCAN) { + if (pCtx->scanFlag == REVERSE_SCAN) { return; } @@ -2969,11 +2966,12 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { char* output = pCtx->aOutputBuf; if (pCtx->tag.nType == TSDB_DATA_TYPE_BINARY || pCtx->tag.nType == TSDB_DATA_TYPE_NCHAR) { - *(int16_t*) output = pCtx->tag.nLen; - output += VARSTR_HEADER_SIZE; + varDataSetLen(output, pCtx->tag.nLen); + tVariantDump(&pCtx->tag, varDataVal(output), pCtx->outputType); + } else { + tVariantDump(&pCtx->tag, output, pCtx->outputType); } - tVariantDump(&pCtx->tag, output, pCtx->outputType); pCtx->aOutputBuf += pCtx->outputBytes; } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cc03f7ad9..a81ad19e4f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5563,8 +5563,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - *(VarDataLenT*)tagVal = pList->a[i].pVar.nLen; - ret = tVariantDump(&(pList->a[i].pVar), tagVal + VARSTR_HEADER_SIZE, pTagSchema[i].type); + ret = tVariantDump(&(pList->a[i].pVar), varDataVal(tagVal), pTagSchema[i].type); + varDataSetLen(tagVal, pList->a[i].pVar.nLen); } else { ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type); } @@ -5911,7 +5911,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* } if (pSqlExpr->pLeft == NULL) { - if (pSqlExpr->nSQLOptr >= TK_TINYINT && pSqlExpr->nSQLOptr <= TK_DOUBLE) { + if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) { *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_VALUE; (*pExpr)->pVal = calloc(1, sizeof(tVariant)); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 119a84e964..98cbe9dbde 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -221,7 +221,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } -void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { SSqlObj *pSql = (SSqlObj *)rpcMsg->handle; if (pSql == NULL) { tscError("%p sql is already released", pSql->signature); @@ -245,6 +245,12 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { return; } + if (pCmd->command < TSDB_SQL_MGMT) { + if (pIpSet) pSql->ipList = *pIpSet; + } else { + if (pIpSet) tscMgmtIpSet = *pIpSet; + } + if (rpcMsg->pCont == NULL) { rpcMsg->code = TSDB_CODE_NETWORK_UNAVAIL; } else { @@ -492,13 +498,15 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(pQueryInfo->type); // todo valid the vgroupId at the client side - if (UTIL_TABLE_IS_SUPERTABLE(pQueryInfo->pTableMetaInfo[0])) { - SVgroupsInfo* pVgroupInfo = pQueryInfo->pTableMetaInfo[0]->vgroupList; - assert(pVgroupInfo->numOfVgroups == 1); // todo fix me + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + int32_t vgIndex = pTableMetaInfo->vgroupIndex; - pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[0].vgId); + SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; + pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); } else { - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[0]->pTableMeta; + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index a45db94b49..14f0fa07ca 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -172,7 +172,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscTrace("try to create a connection to %s", ip); + tscTrace("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL); if (pObj != NULL) { @@ -191,7 +191,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } - tscTrace("%p DB connection is opening", pObj); + tscTrace("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn); // version compare only requires the first 3 segments of the version string int code = taosCheckVersion(version, taos_get_server_info(pObj), 3); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d3a81adca6..7a1c08d056 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1436,8 +1436,8 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO return; } else { // reach the maximum retry count, abort atomic_val_compare_exchange_32(&pState->code, TSDB_CODE_SUCCESS, numOfRows); - tscError("%p sub:%p retrieve failed,code:%d,orderOfSub:%d failed.no more retry,set global code:%d", pPObj, pSql, - numOfRows, subqueryIndex, tstrerror(pState->code)); + tscError("%p sub:%p retrieve failed,code:%s,orderOfSub:%d failed.no more retry,set global code:%s", pPObj, pSql, + tstrerror(numOfRows), subqueryIndex, tstrerror(pState->code)); } } @@ -1450,7 +1450,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } // all subqueries are failed - tscError("%p retrieve from %d vnode(s) completed,code:%d.FAILED.", pPObj, pState->numOfTotal, pState->code); + tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pPObj, pState->numOfTotal, tstrerror(pState->code)); pPObj->res.code = pState->code; // release allocated resource diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index d713a9ee5f..5d8652a631 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -63,12 +63,15 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) { rpcInit.user = (char*)user; rpcInit.idleTime = 2000; rpcInit.ckey = "key"; + rpcInit.spi = 1; rpcInit.secret = secretEncrypt; *pDnodeConn = rpcOpen(&rpcInit); if (*pDnodeConn == NULL) { tscError("failed to init connection to TDengine"); return -1; + } else { + tscTrace("dnodeConn:%p is created, user:%s", *pDnodeConn, user); } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ccb363d9e8..c25e91a669 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -766,7 +766,7 @@ void tscCloseTscObj(STscObj* pObj) { rpcClose(pObj->pDnodeConn); } - tscTrace("%p DB connection is closed", pObj); + tscTrace("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn); tfree(pObj); } @@ -1144,23 +1144,21 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { } SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { - SColumnFilterInfo* pFilter = NULL; - if (numOfFilters > 0) { - pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); - } else { + if (numOfFilters == 0) { assert(src == NULL); return NULL; } + SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1; - - char* pTmp = calloc(1, len); - pFilter[j].pz = (int64_t) pTmp; + pFilter[j].pz = (int64_t) calloc(1, len); - memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len); } } @@ -1675,10 +1673,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->pTableMeta = pTableMeta; if (vgroupList != NULL) { - assert(vgroupList->numOfVgroups == 1); // todo fix me - size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; - pTableMetaInfo->vgroupList = malloc(size); memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index ec52bcd2ae..51a5dad486 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -115,10 +115,8 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - break; default: return POINTER_SHIFT(row, offset); - break; } } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 05afbab642..b077f40945 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -141,9 +141,9 @@ extern int32_t tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t dDebugFlag; extern int32_t vDebugFlag; -extern int32_t mdebugFlag; -extern int32_t cdebugFlag; -extern int32_t jnidebugFlag; +extern int32_t mDebugFlag; +extern int32_t cDebugFlag; +extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; extern int32_t sdbDebugFlag; extern int32_t httpDebugFlag; @@ -151,8 +151,8 @@ extern int32_t monitorDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t debugFlag; -extern int32_t odbcdebugFlag; -extern int32_t qdebugFlag; +extern int32_t odbcDebugFlag; +extern int32_t qDebugFlag; extern uint32_t taosMaxTmrCtrl; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 991c1b403b..90637265b3 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -127,16 +127,16 @@ int32_t tsRestRowLimit = 10240; int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; int32_t tsNumOfLogLines = 10000000; -int32_t mdebugFlag = 135; +int32_t mDebugFlag = 135; int32_t sdbDebugFlag = 135; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; -int32_t cdebugFlag = 135; -int32_t jnidebugFlag = 131; -int32_t odbcdebugFlag = 131; +int32_t cDebugFlag = 135; +int32_t jniDebugFlag = 131; +int32_t odbcDebugFlag = 131; int32_t httpDebugFlag = 131; int32_t monitorDebugFlag = 131; -int32_t qdebugFlag = 131; +int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 135; int32_t uDebugFlag = 131; int32_t debugFlag = 131; @@ -204,19 +204,19 @@ static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { - mdebugFlag = debugFlag; + mDebugFlag = debugFlag; sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; - cdebugFlag = debugFlag; - jnidebugFlag = debugFlag; - odbcdebugFlag = debugFlag; + cDebugFlag = debugFlag; + jniDebugFlag = debugFlag; + odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; monitorDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; sDebugFlag = debugFlag; - //qdebugFlag = debugFlag; + //qDebugFlag = debugFlag; } uPrint("all debug flag are set to %d", debugFlag); } @@ -1002,7 +1002,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "mDebugFlag"; - cfg.ptr = &mdebugFlag; + cfg.ptr = &mDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; cfg.minValue = 0; @@ -1062,7 +1062,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "cDebugFlag"; - cfg.ptr = &cdebugFlag; + cfg.ptr = &cDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; @@ -1072,7 +1072,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "jniDebugFlag"; - cfg.ptr = &jnidebugFlag; + cfg.ptr = &jniDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; @@ -1082,7 +1082,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "odbcDebugFlag"; - cfg.ptr = &odbcdebugFlag; + cfg.ptr = &odbcDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; @@ -1122,7 +1122,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "qDebugFlag"; - cfg.ptr = &qdebugFlag; + cfg.ptr = &qDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 5b62192fa8..dc5f7d192f 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -167,9 +167,9 @@ static int32_t dnodeInitSystem() { if (dnodeInitRead() != 0) return -1; if (dnodeInitWrite() != 0) return -1; if (dnodeInitClient() != 0) return -1; - if (dnodeInitModules() != 0) return -1; if (dnodeInitServer() != 0) return -1; if (dnodeInitMgmt() != 0) return -1; + if (dnodeInitModules() != 0) return -1; if (dnodeInitShell() != 0) return -1; dnodeStartModules(); @@ -184,14 +184,14 @@ static void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) { dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED); dnodeCleanupShell(); - dnodeCleanupServer(); + dnodeCleanUpModules(); dnodeCleanupMgmt(); + dnodeCleanupServer(); dnodeCleanupClient(); dnodeCleanupWrite(); dnodeCleanupRead(); - dnodeCleanUpModules(); - taos_cleanup(); dnodeCleanupStorage(); + taos_cleanup(); taosCloseLog(); } } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 62d22573ff..36a7c98807 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -266,9 +266,12 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { return taosCfgDynamicOptions(pCfg->config); } +void dnodeUpdateIpSet(SRpcIpSet *pIpSet) { + dPrint("mnode IP list is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse); + for (int i = 0; i < pIpSet->numOfIps; ++i) { + dPrint("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i]) + } -void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { - dTrace("mgmt IP list is changed for ufp is called"); tsMnodeIpSet = *pIpSet; } diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index b1e5d9fd38..c91da4953d 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -29,11 +29,11 @@ #include "dnodeVWrite.h" #include "mnode.h" -extern void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); +extern void dnodeUpdateIpSet(SRpcIpSet *pIpSet); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg); +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg); +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet); static void *tsDnodeServerRpc = NULL; static void *tsDnodeClientRpc = NULL; @@ -52,7 +52,8 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mgmtProcessReqMsgFromDnode; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = mgmtProcessReqMsgFromDnode; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = mgmtProcessReqMsgFromDnode; - + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = mgmtProcessReqMsgFromDnode; + SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = tsDnodeDnodePort; @@ -61,7 +62,7 @@ int32_t dnodeInitServer() { rpcInit.cfp = dnodeProcessReqMsgFromDnode; rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 2000; + rpcInit.idleTime = tsShellActivityTimer * 1000; tsDnodeServerRpc = rpcOpen(&rpcInit); if (tsDnodeServerRpc == NULL) { @@ -81,7 +82,7 @@ void dnodeCleanupServer() { } } -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg) { +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { SRpcMsg rspMsg; rspMsg.handle = pMsg->handle; rspMsg.pCont = NULL; @@ -119,10 +120,9 @@ int32_t dnodeInitClient() { rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessRspFromDnode; - rpcInit.ufp = dnodeUpdateIpSet; rpcInit.sessions = 100; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 2000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "t"; rpcInit.ckey = "key"; rpcInit.secret = "secret"; @@ -145,9 +145,10 @@ void dnodeCleanupClient() { } } -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg) { +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { if (dnodeProcessRspMsgFp[pMsg->msgType]) { + if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) dnodeUpdateIpSet(pIpSet); (*dnodeProcessRspMsgFp[pMsg->msgType])(pMsg); } else { dError("RPC %p, msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]); @@ -163,3 +164,9 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) { rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg); } + +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { + SRpcIpSet ipSet = {0}; + dnodeGetMnodeDnodeIpSet(&ipSet); + rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp); +} diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index cf6b5e38ab..52407dc7a4 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -21,6 +21,7 @@ #include "trpc.h" #include "tglobal.h" #include "http.h" +#include "mnode.h" #include "dnode.h" #include "dnodeInt.h" #include "dnodeVRead.h" @@ -28,7 +29,7 @@ #include "dnodeShell.h" static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg); +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); static void * tsDnodeShellRpc = NULL; static int32_t tsDnodeQueryReqNum = 0; @@ -86,7 +87,7 @@ int32_t dnodeInitShell() { rpcInit.cfp = dnodeProcessMsgFromShell; rpcInit.sessions = TSDB_SESSIONS_PER_DNODE; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = tsShellActivityTimer * 1500; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dnodeRetrieveUserAuthInfo; tsDnodeShellRpc = rpcOpen(&rpcInit); @@ -106,7 +107,7 @@ void dnodeCleanupShell() { } } -void dnodeProcessMsgFromShell(SRpcMsg *pMsg) { +void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { SRpcMsg rpcMsg; rpcMsg.handle = pMsg->handle; rpcMsg.pCont = NULL; @@ -138,7 +139,34 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg) { } static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - return TSDB_CODE_SUCCESS; + int code = mgmtRetriveAuth(user, spi, encrypt, secret, ckey); + if (code != TSDB_CODE_NOT_READY) return code; + + SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); + strcpy(pMsg->user, user); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pMsg; + rpcMsg.contLen = sizeof(SDMAuthMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; + + dTrace("user:%s, send auth msg to mnode", user); + SRpcMsg rpcRsp = {0}; + dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + dError("user:%s, auth msg received from mnode, error:%s", user, tstrerror(rpcRsp.code)); + } else { + dTrace("user:%s, auth msg received from mnode", user); + SDMAuthRsp *pRsp = rpcRsp.pCont; + memcpy(secret, pRsp->secret, TSDB_KEY_LEN); + memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + *spi = pRsp->spi; + *encrypt = pRsp->encrypt; + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; } SDnodeStatisInfo dnodeGetStatisInfo() { diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 5145a46831..9884cf2870 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -52,6 +52,7 @@ int32_t dnodeGetDnodeId(); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); #ifdef __cplusplus } diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 37fec24c20..48b1ac97bd 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -26,8 +26,10 @@ void mgmtCleanUpSystem(); void mgmtStopSystem(); void sdbUpdateSync(); -void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg); -void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg); +int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg); +void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg); + #ifdef __cplusplus } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 571b551368..33a0e4f2c6 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,6 +36,11 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + #define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] @@ -74,10 +79,10 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_DATA_NULL_STR "NULL" #define TSDB_DATA_NULL_STR_L "null" -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 -#define TSDB_ERR -1 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 +#define TSDB_ERR -1 #define TS_PATH_DELIMITER "." diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 8dac73606d..44980b3912 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -100,6 +100,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_TABLE, "config-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_VNODE, "config-vnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_STATUS, "status" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_GRANT, "grant" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) @@ -737,6 +738,14 @@ typedef struct { char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDAlterStreamMsg; +typedef struct { + char user[TSDB_USER_LEN + 1]; + char spi; + char encrypt; + char secret[TSDB_KEY_LEN + 1]; + char ckey[TSDB_KEY_LEN + 1]; +} SDMAuthMsg, SDMAuthRsp; + #pragma pack(pop) #ifdef __cplusplus diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 8b082b65b8..eff210433f 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -66,10 +66,7 @@ typedef struct { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *); - - // call back to process notify the ipSet changes, for client app only - void (*ufp)(void *ahandle, SRpcIpSet *pIpSet); + void (*cfp)(SRpcMsg *, SRpcIpSet *); // call back to retrieve the client auth info, for server app only int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index f4242fbe1f..4468ee4262 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -102,13 +102,15 @@ int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); void tsdbClearTableCfg(STableCfg *config); int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val); -int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name); +char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes); int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid); +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size); + // the TSDB repository info typedef struct STsdbRepoInfo { STsdbCfg tsdbCfg; diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index ba71f9373b..3ac2efb83b 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -85,8 +85,7 @@ typedef struct SSuperTableObj { int32_t numOfTables; int16_t nextColId; SSchema * schema; - int32_t vgLen; - int32_t * vgList; + void * vgHash; } SSuperTableObj; typedef struct { diff --git a/src/mnode/inc/mgmtInt.h b/src/mnode/inc/mgmtInt.h index 17553ade72..a83d5beee0 100644 --- a/src/mnode/inc/mgmtInt.h +++ b/src/mnode/inc/mgmtInt.h @@ -23,13 +23,13 @@ extern "C" { #include "tlog.h" #include "monitor.h" -extern int32_t mdebugFlag; +extern int32_t mDebugFlag; extern int32_t sdbDebugFlag; // mnode log function -#define mError(...) if (mdebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); } -#define mWarn(...) if (mdebugFlag & DEBUG_WARN) { taosPrintLog("WARN MND ", mdebugFlag, __VA_ARGS__); } -#define mTrace(...) if (mdebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mdebugFlag, __VA_ARGS__); } +#define mError(...) if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); } +#define mWarn(...) if (mDebugFlag & DEBUG_WARN) { taosPrintLog("WARN MND ", mDebugFlag, __VA_ARGS__); } +#define mTrace(...) if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); } #define mPrint(...) { taosPrintLog("MND ", 255, __VA_ARGS__); } #define mLError(...) monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 4f54a85baa..2f17df92fa 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -297,8 +297,10 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { if (pDb != NULL) { mgmtDecDbRef(pDb); if (pCreate->ignoreExist) { + mTrace("db:%s, already exist, ignore exist is set", pCreate->db); return TSDB_CODE_SUCCESS; } else { + mError("db:%s, is already exist, ignore exist not set", pCreate->db); return TSDB_CODE_DB_ALREADY_EXIST; } } @@ -751,6 +753,8 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { code = mgmtCreateDb(pMsg->pUser->pAcct, pCreate); if (code == TSDB_CODE_SUCCESS) { mLPrint("db:%s, is created by %s", pCreate->db, pMsg->pUser->user); + } else { + mError("db:%s, failed to create, reason:%s", pCreate->db, tstrerror(code)); } } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 4a18c5e285..c25f4457a9 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -278,7 +278,7 @@ void sdbUpdateSync() { sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort); } - SSyncInfo syncInfo; + SSyncInfo syncInfo = {0}; syncInfo.vgId = 1; syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; diff --git a/src/mnode/src/mgmtServer.c b/src/mnode/src/mgmtServer.c index a6b4362cbf..7810189e34 100644 --- a/src/mnode/src/mgmtServer.c +++ b/src/mnode/src/mgmtServer.c @@ -88,10 +88,12 @@ void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg) { SRpcIpSet ipSet = {0}; dnodeGetMnodeDnodeIpSet(&ipSet); + for (int i = 0; i < ipSet.numOfIps; ++i) + ipSet.port[i] = htons(ipSet.port[i]); mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); for (int32_t i = 0; i < ipSet.numOfIps; ++i) { - mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]); + mTrace("mnode index:%d %s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i])); } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index c869e752d3..c27dc66dd5 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -41,7 +41,6 @@ typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); -//static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg); static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg); static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg); @@ -137,7 +136,7 @@ void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { mgmtGetMnodeIpSet(&ipSet); mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); for (int32_t i = 0; i < ipSet.numOfIps; ++i) { - mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]); + mTrace("mnode index:%d ip:%s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i])); } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); @@ -343,29 +342,6 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { rpcSendResponse(&rpcRsp); } -/* -static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - *spi = 1; - *encrypt = 0; - *ckey = 0; - - if (!sdbIsMaster()) { - *secret = 0; - return TSDB_CODE_NOT_READY; - } - - SUserObj *pUser = mgmtGetUser(user); - if (pUser == NULL) { - *secret = 0; - return TSDB_CODE_INVALID_USER; - } else { - memcpy(secret, pUser->pass, TSDB_KEY_LEN); - mgmtDecUserRef(pUser); - return TSDB_CODE_SUCCESS; - } -} -*/ - static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMConnectMsg *pConnectMsg = pMsg->pCont; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index bfe357cf7c..49311b0112 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -24,6 +24,7 @@ #include "tname.h" #include "tidpool.h" #include "tglobal.h" +#include "hash.h" #include "dnode.h" #include "mgmtDef.h" #include "mgmtInt.h" @@ -363,39 +364,35 @@ static void mgmtCleanUpChildTables() { } static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { - if (pStable->vgLen == 0) { - pStable->vgLen = 8; - pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t)); - } - - bool find = false; - int32_t pos = 0; - for (pos = 0; pos < pStable->vgLen; ++pos) { - if (pStable->vgList[pos] == 0) break; - if (pStable->vgList[pos] == pCtable->vgId) { - find = true; - break; - } - } - - if (!find) { - if (pos >= pStable->vgLen) { - pStable->vgLen *= 2; - pStable->vgList = realloc(pStable->vgList, pStable->vgLen * sizeof(int32_t)); - } - pStable->vgList[pos] = pCtable->vgId; - } - pStable->numOfTables++; + + if (pStable->vgHash == NULL) { + pStable->vgHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + } + + if (pStable->vgHash != NULL) { + taosHashPut(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); + } } static void mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { pStable->numOfTables--; + + if (pStable->vgHash == NULL) return; + + SVgObj *pVgroup = mgmtGetVgroup(pCtable->vgId); + if (pVgroup != NULL) { + taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); + } + mgmtDecVgroupRef(pVgroup); } static void mgmtDestroySuperTable(SSuperTableObj *pStable) { + if (pStable->vgHash != NULL) { + taosHashCleanup(pStable->vgHash); + pStable->vgHash = NULL; + } tfree(pStable->schema); - tfree(pStable->vgList) tfree(pStable); } @@ -434,7 +431,7 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) { void *oldSchema = pTable->schema; memcpy(pTable, pNew, pOper->rowSize); pTable->schema = pNew->schema; - free(pNew->vgList); + free(pNew->vgHash); free(pNew); free(oldSchema); } @@ -797,26 +794,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTables != 0) { - mgmtDropAllChildTablesInStable(pStable); - for (int32_t vg = 0; vg < pStable->vgLen; ++vg) { - int32_t vgId = pStable->vgList[vg]; - if (vgId == 0) break; - - SVgObj *pVgroup = mgmtGetVgroup(vgId); + SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); + while (taosHashIterNext(pIter)) { + int32_t *pVgId = taosHashIterGet(pIter); + SVgObj *pVgroup = mgmtGetVgroup(*pVgId); if (pVgroup == NULL) break; - + SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); pDrop->contLen = htonl(sizeof(SMDDropSTableMsg)); - pDrop->vgId = htonl(vgId); + pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); mgmtExtractTableName(pStable->info.tableId, pDrop->tableId); - mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, vgId); + mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, pVgroup->vgId); SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&ipSet, &rpcMsg); mgmtDecVgroupRef(pVgroup); } + + mgmtDropAllChildTablesInStable(pStable); } SSdbOper oper = { @@ -1243,59 +1240,58 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { SCMSTableVgroupMsg *pInfo = pMsg->pCont; int32_t numOfTable = htonl(pInfo->numOfTables); - - char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg); - SCMSTableVgroupRspMsg *pRsp = NULL; - - // todo set the initial size to be 10, fix me - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable; - - pRsp = rpcMallocCont(contLen); + + // reserve space + int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); + for (int32_t i = 0; i < numOfTable; ++i) { + char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i; + SSuperTableObj *pTable = mgmtGetSuperTable(stableName); + if (pTable->vgHash != NULL) { + contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo)); + } + mgmtDecTableRef(pTable); + } + + SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } - + pRsp->numOfTables = htonl(numOfTable); char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg); - - for(int32_t i = 0; i < numOfTable; ++i) { - SSuperTableObj *pTable = mgmtGetSuperTable(name); - - pMsg->pTable = (STableObj *)pTable; - if (pMsg->pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } - - SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg; - - int32_t vg = 0; - for (; vg < pTable->vgLen; ++vg) { - int32_t vgId = pTable->vgList[vg]; - if (vgId == 0) break; - - SVgObj *vgItem = mgmtGetVgroup(vgId); - if (vgItem == NULL) break; - - pVgroup->vgroups[vg].vgId = htonl(vgId); - for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) { - SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode; + + for (int32_t i = 0; i < numOfTable; ++i) { + char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i; + SSuperTableObj *pTable = mgmtGetSuperTable(stableName); + SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg; + + SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash); + int32_t vgSize = 0; + while (taosHashIterNext(pIter)) { + int32_t *pVgId = taosHashIterGet(pIter); + SVgObj * pVgroup = mgmtGetVgroup(*pVgId); + if (pVgroup == NULL) continue; + + pVgroupInfo->vgroups[vgSize].vgId = htonl(pVgroup->vgId); + for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { + SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; if (pDnode == NULL) break; - - strncpy(pVgroup->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn)); - pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort); - - pVgroup->vgroups[vg].numOfIps++; + + strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn)); + pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort); + + pVgroupInfo->vgroups[vgSize].numOfIps++; } - - mgmtDecVgroupRef(vgItem); + + vgSize++; + mgmtDecVgroupRef(pVgroup); } - - pVgroup->numOfVgroups = htonl(vg); - + + pVgroupInfo->numOfVgroups = htonl(vgSize); + // one table is done, try the next table - msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo); + msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo); } SRpcMsg rpcRsp = {0}; diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 9630ab3d58..350367e0ae 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -37,6 +37,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); +static void mgmtProcessAuthMsg(SRpcMsg *rpcMsg); static int32_t mgmtUserActionDestroy(SSdbOper *pOper) { tfree(pOper->pObj); @@ -140,7 +141,8 @@ int32_t mgmtInitUsers() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_USER, mgmtGetUserMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mgmtRetrieveUsers); - + dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mgmtProcessAuthMsg); + mTrace("table:%s, hash is created", tableDesc.tableName); return 0; } @@ -529,3 +531,40 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); } + +int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (!sdbIsMaster()) { + *secret = 0; + mTrace("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_NOT_READY)); + return TSDB_CODE_NOT_READY; + } + + SUserObj *pUser = mgmtGetUser(user); + if (pUser == NULL) { + *secret = 0; + mError("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_INVALID_USER)); + return TSDB_CODE_INVALID_USER; + } else { + *spi = 1; + *encrypt = 0; + *ckey = 0; + + memcpy(secret, pUser->pass, TSDB_KEY_LEN); + mgmtDecUserRef(pUser); + mTrace("user:%s, auth info is returned", user); + return TSDB_CODE_SUCCESS; + } +} + +static void mgmtProcessAuthMsg(SRpcMsg *rpcMsg) { + SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; + + SDMAuthMsg *pAuthMsg = rpcMsg->pCont; + SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp)); + + rpcRsp.code = mgmtRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); + rpcRsp.pCont = pAuthRsp; + rpcRsp.contLen = sizeof(SDMAuthRsp); + + rpcSendResponse(&rpcRsp); +} diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 9bc36413de..410b2ac9d2 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -50,11 +50,11 @@ typedef struct tQueryInfo { SSchema sch; // schema of tags char* q; __compar_fn_t compare; // filter function - void* param; // STSchema, + void* param; // STSchema } tQueryInfo; typedef struct SExprTraverseSupp { - __result_filter_fn_t fp; + __result_filter_fn_t nodeFilterFn; __do_filter_suppl_fn_t setupInfoFn; void * pExtInfo; } SExprTraverseSupp; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 906dadb317..2088e5a49d 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -110,6 +110,14 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct SWindowResInfo windowResInfo; } STableQueryInfo; +typedef struct SQueryCostSummary { +} SQueryCostSummary; + +typedef struct SGroupItem { + STableId id; + STableQueryInfo* info; +} SGroupItem; + typedef struct SQuery { int16_t numOfCols; int16_t numOfTags; @@ -131,17 +139,15 @@ typedef struct SQuery { SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* defaultVal; - TSKEY lastKey; +// TSKEY lastKey; uint32_t status; // query status SResultRec rec; int32_t pos; SData** sdata; + STableQueryInfo* current; SSingleColumnFilterInfo* pFilterInfo; } SQuery; -typedef struct SQueryCostSummary { -} SQueryCostSummary; - typedef struct SQueryRuntimeEnv { SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index 2c77ee6032..4894d67e04 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -22,20 +22,20 @@ extern "C" { #include "tlog.h" -extern int32_t qdebugFlag; +extern int32_t qDebugFlag; #define qTrace(...) \ - if (qdebugFlag & DEBUG_TRACE) { \ - taosPrintLog("DND QRY ", qdebugFlag, __VA_ARGS__); \ + if (qDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("DND QRY ", qDebugFlag, __VA_ARGS__); \ } #define qError(...) \ - if (qdebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR QRY ", qdebugFlag, __VA_ARGS__); \ + if (qDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR QRY ", qDebugFlag, __VA_ARGS__); \ } #define qWarn(...) \ - if (qdebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN QRY ", qdebugFlag, __VA_ARGS__); \ + if (qDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN QRY ", qDebugFlag, __VA_ARGS__); \ } #ifdef __cplusplus diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index a92f401323..a11c03f2c0 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -104,7 +104,7 @@ extern "C" { enum { MASTER_SCAN = 0x0u, - SUPPLEMENTARY_SCAN = 0x1u, + REVERSE_SCAN = 0x1u, REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan FIRST_STAGE_MERGE = 0x10u, SECONDARY_STAGE_MERGE = 0x20u, diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 98682ce778..f727acb720 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -540,8 +540,11 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; + SQueryCond cond = {0}; - setQueryCond(pQueryInfo, &cond); + if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { + //todo handle error + } if (cond.start != NULL) { iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); @@ -552,18 +555,18 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (cond.start != NULL) { int32_t optr = cond.start->optr; - if (optr == TSDB_RELATION_EQUAL) { + if (optr == TSDB_RELATION_EQUAL) { // equals while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - if (ret == 0) { - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); - } else { + if (ret != 0) { break; } + + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } - } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal bool comp = true; int32_t ret = 0; @@ -582,8 +585,33 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr comp = false; } } - } else if (optr == TSDB_RELATION_NOT_EQUAL) { - assert(0); + } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal + bool comp = true; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + } + + tSkipListDestroyIter(iter); + + comp = true; + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + } + } else { assert(0); } @@ -732,7 +760,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); - return param->fp(pItem, pExpr->_node.info); + return param->nodeFilterFn(pItem, pExpr->_node.info); } /** @@ -773,35 +801,33 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki tSkipListDestroyIter(iter); } - - -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { bool addToResult = false; SSkipListNode *pNode = tSkipListIterGet(iter); - char* pTable = SL_GET_NODE_DATA(pNode); - - //todo refactor: - char* name = (*(STable**) pTable)->name; -// char* name = NULL; -// tsdbGetTableName(tsdb, pTable, &name); + char * pData = SL_GET_NODE_DATA(pNode); + + // todo refactor: + tstr *name = ((STableIndexElem *)pData)->pTable->name; + // char* name = NULL; +// tsdbGetTableName(pQueryInfo->, pTable, &name); // todo speed up by using hash if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { addToResult = !pQueryInfo->compare(name, pQueryInfo->q); } } else { - // TODO: other columns + addToResult = filterFp(pNode, pQueryInfo); } if (addToResult) { - taosArrayPush(result, pTable); + taosArrayPush(res, pData); } } @@ -825,7 +851,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { - tArrayTraverse(pExpr, param->fp, result); + tArrayTraverse(pExpr, param->nodeFilterFn, result); return; } @@ -833,7 +859,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); } return; diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index b8d1189129..9589be86e4 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -820,7 +820,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt = *pDB; pInfo->pDCLInfo->dbOpt.dbname = *pToken; - pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists->z != NULL); + pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 0ffc665e55..fb19d0f50c 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -40,9 +40,9 @@ #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) -#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN) +#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) +#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) @@ -96,11 +96,6 @@ typedef struct { STSCursor cur; } SQueryStatusInfo; -typedef struct SGroupItem { - STableId id; - STableQueryInfo* info; -} SGroupItem; - static void setQueryStatus(SQuery *pQuery, int8_t status); static bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } @@ -121,7 +116,7 @@ static bool hasMainOutput(SQuery *pQuery); static void createTableQueryInfo(SQInfo *pQInfo); static void buildTagQueryResult(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTaleId, STableQueryInfo *pTableQueryInfo); +static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTableId, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQInfo *pQInfo); bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { @@ -428,9 +423,10 @@ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlock return false; } - *pColStatis = NULL; if (pStatis != NULL) { *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); + } else { + *pColStatis = NULL; } if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { @@ -620,8 +616,8 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, } // query completed - if ((lastKey >= pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (lastKey <= pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((lastKey >= pQuery->current->win.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (lastKey <= pQuery->current->win.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { closeAllTimeWindow(pWindowResInfo); pWindowResInfo->curIndex = pWindowResInfo->size - 1; @@ -661,22 +657,22 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, setQueryStatus(pQuery, QUERY_RESBUF_FULL); } - qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n); + qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n); } assert(pWindowResInfo->prevSKey != 0); } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, - bool updateLastKey) { + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; int32_t order = pQuery->order.order; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + STableQueryInfo* item = pQuery->current; + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -684,13 +680,13 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo assert(ekey < pPrimaryColumn[startPos]); } else { if (updateLastKey) { - pQuery->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; } } } else { num = pDataBlockInfo->rows - startPos; if (updateLastKey) { - pQuery->lastKey = pDataBlockInfo->window.ekey + step; + item->lastKey = pDataBlockInfo->window.ekey + step; } } } else { // desc @@ -700,13 +696,13 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo assert(ekey > pPrimaryColumn[startPos]); } else { if (updateLastKey) { - pQuery->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; } } } else { num = startPos + 1; if (updateLastKey) { - pQuery->lastKey = pDataBlockInfo->window.skey + step; + item->lastKey = pDataBlockInfo->window.skey + step; } } } @@ -820,7 +816,21 @@ static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { return ekey; } -char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, +//todo binary search +static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *p = taosArrayGet(pDataBlock, i); + if (colId == p->info.colId) { + return p->pData; + } + } + + return NULL; +} + +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { char *dataBlock = NULL; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -867,20 +877,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { dataBlock = NULL; } else { - /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare - * stage, the remain meter may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding meter schema is reinforced. - */ - int32_t numOfCols = taosArrayGetSize(pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *p = taosArrayGet(pDataBlock, i); - if (pCol->colId == p->info.colId) { - dataBlock = p->pData; - break; - } - } + dataBlock = getDataBlockImpl(pDataBlock, pCol->colId); } } @@ -904,8 +901,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SQuery * pQuery = pRuntimeEnv->pQuery; SColumnInfoData *pColInfo = NULL; - TSKEY * primaryKeyCol = NULL; - + + TSKEY *primaryKeyCol = NULL; if (pDataBlock != NULL) { pColInfo = taosArrayGet(pDataBlock, 0); primaryKeyCol = (TSKEY *)(pColInfo->pData); @@ -919,7 +916,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SDataStatis *tpField = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); @@ -1095,15 +1092,20 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + SQuery* pQuery = pRuntimeEnv->pQuery; + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } + if (functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST) { + return !QUERY_IS_ASC_QUERY(pQuery); + } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) { + return QUERY_IS_ASC_QUERY(pQuery); + } + // in the supplementary scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv) && - !(functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST || - functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { + if (IS_REVERSE_SCAN(pRuntimeEnv)) {// && (functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS)) { return false; } @@ -1115,9 +1117,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo* item = pQuery->current; + TSKEY *primaryKeyCol = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData; - - bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); int16_t type = 0; @@ -1134,7 +1137,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SDataStatis *pColStatis = NULL; bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); - char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, &sasArray[k], pRuntimeEnv->scanFlag); @@ -1143,7 +1146,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - pFilterInfo->pData = getDataBlocks(pRuntimeEnv, &sasArray[k], pFilterInfo->info.colId, pDataBlockInfo->rows, pDataBlock); + pFilterInfo->pData = getDataBlockImpl(pDataBlock, pFilterInfo->info.colId); + assert(pFilterInfo->pData != NULL); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1157,10 +1161,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } int32_t j = 0; - TSKEY lastKey = -1; - + int32_t offset = -1; + for (j = 0; j < pDataBlockInfo->rows; ++j) { - int32_t offset = GET_COL_DATA_POS(pQuery, j, step); + offset = GET_COL_DATA_POS(pQuery, j, step); if (pRuntimeEnv->pTSBuf != NULL) { int32_t r = doTSJoinFilter(pRuntimeEnv, offset); @@ -1194,7 +1198,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); - lastKey = ts; STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1223,17 +1226,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } else { // other queries // decide which group this rows belongs to according to current state value if (groupbyStateValue) { - char *stateVal = groupbyColumnData + bytes * offset; + char *val = groupbyColumnData + bytes * offset; - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, stateVal, type, bytes); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } } - // update the lastKey - lastKey = primaryKeyCol[offset]; - // all startOffset are identical offset -= pCtx[0].startOffset; @@ -1254,7 +1254,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } - pQuery->lastKey = lastKey + step; + item->lastKey = primaryKeyCol[offset] + step; // todo refactor: extract method for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -1269,10 +1269,12 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, - SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + + STableQueryInfo* pTableQInfo = pQuery->current; + SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1280,7 +1282,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl } TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1746,73 +1748,6 @@ static UNUSED_FUNC bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pP } } -// TODO refactor code, the best way to implement the last_row is utilizing the iterator -bool normalizeUnBoundLastRowQuery(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter) { -#if 0 - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - SQuery * pQuery = pRuntimeEnv->pQuery; - SMeterObj *pMeterObj = pRuntimeEnv->pTabObj; - - assert(!QUERY_IS_ASC_QUERY(pQuery) && notHasQueryTimeRange(pQuery)); - __block_search_fn_t searchFn = vnodeSearchKeyFunc[pMeterObj->searchAlgorithm]; - - TSKEY lastKey = -1; - - pQuery->fileId = -1; - vnodeFreeFieldsEx(pRuntimeEnv); - - // keep in-memory cache status in local variables in case that it may be changed by write operation - getBasicCacheInfoSnapshot(pQuery, pMeterObj->pCache, pMeterObj->vnode); - - SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; - if (pCacheInfo != NULL && pCacheInfo->cacheBlocks != NULL && pQuery->numOfBlocks > 0) { - pQuery->fileId = -1; - TSKEY key = pMeterObj->lastKey; - - pQuery->window.skey = key; - pQuery->window.ekey = key; - pQuery->lastKey = pQuery->window.skey; - - /* - * cache block may have been flushed to disk, and no data in cache anymore. - * So, copy cache block to local buffer is required. - */ - lastKey = getQueryStartPositionInCache(pRuntimeEnv, &pQuery->slot, &pQuery->pos, false); - if (lastKey < 0) { // data has been flushed to disk, try again search in file - lastKey = getQueryPositionForCacheInvalid(pRuntimeEnv, searchFn); - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { - return false; - } - } - } else { // no data in cache, try file - TSKEY key = pMeterObj->lastKeyOnFile; - - pQuery->window.skey = key; - pQuery->window.ekey = key; - pQuery->lastKey = pQuery->window.skey; - - bool ret = getQualifiedDataBlock(pMeterObj, pRuntimeEnv, QUERY_RANGE_LESS_EQUAL, searchFn); - if (!ret) { // no data in file, return false; - return false; - } - - lastKey = getTimestampInDiskBlock(pRuntimeEnv, pQuery->pos); - } - - assert(lastKey <= pQuery->window.skey); - - pQuery->window.skey = lastKey; - pQuery->window.ekey = lastKey; - pQuery->lastKey = pQuery->window.skey; - - return getNeighborPoints(pQInfo, pMeterObj, pPointInterpSupporter); -#endif - - return true; -} - static void setScanLimitationByResultBuffer(SQuery *pQuery) { if (isTopBottomQuery(pQuery)) { pQuery->checkBuffer = 0; @@ -2327,7 +2262,7 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow) { pTimeWindow->ekey = pTimeWindow->skey + (pQuery->intervalTime - 1); } -SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo, SDataStatis **pStatis) { +SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis) { SQuery *pQuery = pRuntimeEnv->pQuery; uint32_t r = 0; @@ -2351,16 +2286,16 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl qTrace("QInfo:%p data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { - if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } if (*pStatis == NULL) { - pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); } } else { assert(r == BLK_DATA_ALL_NEEDED); - if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { + if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } @@ -2376,7 +2311,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl // return DISK_DATA_DISCARDED; } - pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); } return pDataBlock; @@ -2447,8 +2382,11 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo* pTableQueryInfo = pQuery->current; + qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", - GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); + GET_QINFO_ADDR(pRuntimeEnv), pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, pTableQueryInfo->lastKey, + pQuery->order.order); TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { @@ -2479,7 +2417,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } } - // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block + // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; @@ -2498,18 +2436,25 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // set the pCtx output buffer position pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; + + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } } - + + qTrace("QInfo: %p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv), + newSize, pRec->capacity, newSize - pRec->rows); + pRec->capacity = newSize; } } SDataStatis *pStatis = NULL; - SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); @@ -2520,7 +2465,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } } - // if the result buffer is not full, set the query completed flag + // if the result buffer is not full, set the query complete if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { setQueryStatus(pQuery, QUERY_COMPLETED); } @@ -2530,8 +2475,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t step = QUERY_IS_ASC_QUERY(pQuery) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP; closeAllTimeWindow(&pRuntimeEnv->windowResInfo); - removeRedundantWindow(&pRuntimeEnv->windowResInfo, pQuery->lastKey - step, step); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; + removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window } else { assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); } @@ -2540,8 +2485,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { return 0; } -static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTableQInfo->lastKey = pQuery->lastKey; } - /* * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer @@ -2554,17 +2497,17 @@ static void doSetTagValueInParam(void *tsdb, STableId* pTableId, int32_t tagColI int16_t type = 0; if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - tsdbGetTableName(tsdb, pTableId, &val); - bytes = strnlen(val, TSDB_TABLE_NAME_LEN); + val = tsdbGetTableName(tsdb, pTableId, &bytes); type = TSDB_DATA_TYPE_BINARY; + tVariantCreateFromBinary(param, varDataVal(val), varDataLen(val), type); } else { tsdbGetTableTagVal(tsdb, pTableId, tagColId, &type, &bytes, &val); - } - - tVariantCreateFromBinary(param, val, bytes, type); - - if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - tfree(val); + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(param, varDataVal(val), varDataLen(val), type); + } else { + tVariantCreateFromBinary(param, val, bytes, type); + } } } @@ -2706,26 +2649,21 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t } } -void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { -#if 0 +void UNUSED_FUNC displayInterResult(SData **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); - SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); - SMeterObj *pMeterObj = pQInfo->pObj; - for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; - int32_t type = 0; - - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].base.colInfo.flag)) { - type = pQuery->pSelectExpr[i].type; - } else { - type = pMeterObj->schema[colIndex].type; - } +// int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; + int32_t type = pQuery->pSelectExpr[i].type; +// } else { +// type = pMeterObj->schema[colIndex].type; +// } printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; @@ -2747,7 +2685,6 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } printf("\n"); } -#endif } typedef struct SCompSupporter { @@ -3015,7 +2952,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -3082,11 +3019,6 @@ void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pRes } } -void setTableDataInfo(STableQueryInfo *pTableQueryInfo, int32_t tableIndex, int32_t groupId) { - pTableQueryInfo->groupIdx = groupId; - pTableQueryInfo->tableIndex = tableIndex; -} - static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { if (pTableQueryInfo == NULL) { return; @@ -3132,19 +3064,6 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow } } } - - int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); - - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - SGroupItem *item = taosArrayGet(group, j); - updateTableQueryInfoForReverseScan(pQuery, item->info); - } - } } void disableFuncInReverseScan(SQInfo *pQInfo) { @@ -3170,6 +3089,18 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { } } } + + int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); + + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + SGroupItem *item = taosArrayGet(group, j); + updateTableQueryInfoForReverseScan(pQuery, item->info); + } + } } void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3345,13 +3276,14 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQueryInfo = pQuery->current; + SQueryStatusInfo info = { - .status = pQuery->status, + .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, - .lastKey = pQuery->lastKey, - .w = pQuery->window, - .curWindow = {.skey = pQuery->lastKey, .ekey = pQuery->window.ekey}, + .lastKey = pTableQueryInfo->lastKey, + .w = pQuery->window, + .curWindow = {.skey = pTableQueryInfo->lastKey, .ekey = pTableQueryInfo->win.ekey}, }; return info; @@ -3376,7 +3308,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI STsdbQueryCond cond = { .twindow = pQuery->window, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; @@ -3395,6 +3327,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo* pTableQueryInfo = pQuery->current; SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); @@ -3408,17 +3341,19 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query // during reverse scan - pQuery->lastKey = pStatus->lastKey; + pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - pQuery->window = pStatus->w; + pTableQueryInfo->win = pStatus->w; } void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { + SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pQuery->current; + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3429,7 +3364,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { if (pRuntimeEnv->scanFlag == MASTER_SCAN) { qstatus.status = pQuery->status; - qstatus.curWindow.ekey = pQuery->lastKey - step; + qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -3443,7 +3378,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { STsdbQueryCond cond = { .twindow = qstatus.curWindow, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; @@ -3549,12 +3484,10 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; - - pQuery->window = pTableQueryInfo->win; - pQuery->lastKey = pTableQueryInfo->lastKey; - - assert(((pQuery->lastKey >= pQuery->window.skey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pQuery->lastKey <= pQuery->window.skey) && !QUERY_IS_ASC_QUERY(pQuery))); + pQuery->current = pTableQueryInfo; + + assert(((pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && !QUERY_IS_ASC_QUERY(pQuery))); } /** @@ -3562,8 +3495,10 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STableId* pTableId, int32_t groupIdx, TSKEY nextKey) { +void setExecutionContext(SQInfo *pQInfo, STableId* pTableId, int32_t groupIdx, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; + SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; int32_t GROUPRESULTID = 1; @@ -3647,15 +3582,16 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, STableId* pTableId, STableQueryInfo *p * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there * is a previous result generated or not. */ -void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSKEY key) { +void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo *pTableQueryInfo = pQuery->current; + if (pTableQueryInfo->queryRangeSet) { - pQuery->lastKey = key; pTableQueryInfo->lastKey = key; } else { - pQuery->window.skey = key; +// pQuery->window.skey = key; + pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; // for too small query range, no data in this interval. @@ -3675,7 +3611,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); - pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp + pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == 0) { if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3687,10 +3623,8 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK } pTableQueryInfo->queryRangeSet = 1; - pTableQueryInfo->lastKey = pQuery->window.skey; - pTableQueryInfo->win.skey = pQuery->window.skey; - - pQuery->lastKey = pQuery->window.skey; + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + pTableQueryInfo->win.skey = pTableQueryInfo->win.skey; } } @@ -3710,7 +3644,9 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { * 2. if there are top/bottom, first_dst/last_dst functions, we need to load timestamp column in any cases; */ STimeWindow *w = &pDataBlockInfo->window; - bool loadPrimaryTS = (pQuery->lastKey >= w->skey && pQuery->lastKey <= w->ekey) || + STableQueryInfo* pTableQueryInfo = pQuery->current; + + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); return loadPrimaryTS; @@ -3793,7 +3729,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde qTrace("QInfo:%p copy data to query buf completed", pQInfo); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, numOfResult); + displayInterResult(pQuery->sdata, pRuntimeEnv, numOfResult); #endif return numOfResult; } @@ -3847,7 +3783,6 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo } updateWindowResNumOfRes(pRuntimeEnv, pTableQueryInfo); - updatelastkey(pQuery, pTableQueryInfo); } bool vnodeHasRemainResults(void *handle) { @@ -4041,10 +3976,12 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo* pTableQueryInfo = pQuery->current; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed - pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; + pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; pQuery->limit.offset = 0; return; } @@ -4064,11 +4001,10 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc TSKEY *keys = (TSKEY *)pColInfoData->pData; // update the offset value - pQuery->lastKey = keys[pQuery->pos]; + pTableQueryInfo->lastKey = keys[pQuery->pos]; pQuery->limit.offset = 0; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes); @@ -4084,6 +4020,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->pos = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + STableQueryInfo* pTableQueryInfo = pQuery->current; TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { @@ -4095,8 +4032,8 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->limit.offset > blockInfo.rows) { pQuery->limit.offset -= blockInfo.rows; - pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; - pQuery->lastKey += step; + pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; + pTableQueryInfo->lastKey += step; qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, pQuery->limit.offset); @@ -4125,6 +4062,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { TSKEY skey1, ekey1; STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + STableQueryInfo *pTableQueryInfo = pQuery->current; while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); @@ -4170,11 +4108,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { // set the abort info pQuery->pos = startPos; - pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; pWindowResInfo->prevSKey = tw.skey; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); @@ -4199,7 +4136,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { // set the abort info pQuery->pos = startPos; - pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; pWindowResInfo->prevSKey = tw.skey; win = tw; } else { @@ -4220,12 +4157,9 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); - // dataInCache requires lastKey value - pQuery->lastKey = pQuery->window.skey; - STsdbQueryCond cond = { .twindow = pQuery->window, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; @@ -4238,6 +4172,9 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); } + + // create the table query support structures + createTableQueryInfo(pQInfo); } pQInfo->tsdb = tsdb; @@ -4331,7 +4268,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool // } // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value - pQuery->lastKey = pQuery->window.skey; return TSDB_CODE_SUCCESS; } @@ -4399,18 +4335,19 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { break; } } - + assert(pTableQueryInfo != NULL); restoreIntervalQueryRange(pRuntimeEnv, pTableQueryInfo); SDataStatis *pStatis = NULL; - SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis); TSKEY nextKey = blockInfo.window.skey; if (!isIntervalQuery(pQuery)) { - setExecutionContext(pQInfo, pTableQueryInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIdx, nextKey); + setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIdx, nextKey); } else { // interval query - setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); + setIntervalQueryRange(pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo); if (ret != TSDB_CODE_SUCCESS) { @@ -4446,6 +4383,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { .numOfCols = pQuery->numOfCols, }; + // todo refactor SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayInit(1, sizeof(STableId)); @@ -4564,6 +4502,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { initCtxOutputBuf(pRuntimeEnv); setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(tx, 0), pQInfo->tsdb); + + // here we simply set the first table as current table + pRuntimeEnv->pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info; scanAllDataBlocks(pRuntimeEnv); int64_t numOfRes = getNumOfResult(pRuntimeEnv); @@ -4580,8 +4521,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } } else { - createTableQueryInfo(pQInfo); - /* * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. @@ -4614,12 +4553,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } SGroupItem *item = taosArrayGet(group, pQInfo->tableIndex); + pQuery->current = item->info; - STableQueryInfo *pInfo = item->info; - if (pInfo->lastKey > 0) { - pQuery->window.skey = pInfo->lastKey; - } - if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { pQInfo->tableIndex++; continue; @@ -4627,7 +4562,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // SPointInterpoSupporter pointInterpSupporter = {0}; - // TODO handle the limit problem + // TODO handle the limit offset problem if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { // skipBlocks(pRuntimeEnv); @@ -4659,8 +4594,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * to ensure that, we can reset the query range once query on a meter is completed. */ pQInfo->tableIndex++; - pInfo->lastKey = pQuery->lastKey; - // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { @@ -4668,7 +4601,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } else { // forward query range - pQuery->window.skey = pQuery->lastKey; + pQuery->window.skey = pQuery->current->lastKey; // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { @@ -4836,7 +4769,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4853,12 +4786,9 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - // create the query support structures - createTableQueryInfo(pQInfo); - // do check all qualified data blocks int64_t el = queryOnDataBlocks(pQInfo); - qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start", pQInfo, el); + qTrace("QInfo:%p master scan completed, elapsed time: %lldms, reverse scan start", pQInfo, el); // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { @@ -4892,7 +4822,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } } else { // not a interval query @@ -4909,10 +4839,12 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcess(SQInfo *pQInfo) { +static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - + + SQuery *pQuery = pRuntimeEnv->pQuery; + pQuery->current = pTableInfo; // set current query table info + scanAllDataBlocks(pRuntimeEnv); finalizeQueryResult(pRuntimeEnv); @@ -4932,10 +4864,12 @@ static void tableFixedOutputProcess(SQInfo *pQInfo) { limitResults(pQInfo); } -static void tableMultiOutputProcess(SQInfo *pQInfo) { +static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - + + SQuery *pQuery = pRuntimeEnv->pQuery; + pQuery->current = pTableInfo; + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -4970,15 +4904,15 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { } qTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->limit.offset, pQuery->lastKey); + pQInfo, pQuery->limit.offset, pQuery->current->lastKey); resetCtxOutputBuf(pRuntimeEnv); } limitResults(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->lastKey, - pQuery->window.ekey); + qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, + pQuery->current->lastKey, pQuery->window.ekey); } if (!isTSCompQuery(pQuery)) { @@ -5018,11 +4952,12 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { } // handle time interval query on table -static void tableIntervalProcess(SQInfo *pQInfo) { +static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); - SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfInterpo = 0; + SQuery *pQuery = pRuntimeEnv->pQuery; + pQuery->current = pTableInfo; // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv); @@ -5131,15 +5066,19 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + + assert(pQInfo->groupInfo.numOfTables == 1); + SArray* g = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + SGroupItem* item = taosArrayGet(g, 0); + // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) - tableIntervalProcess(pQInfo); + tableIntervalProcess(pQInfo, item->info); } else if (isFixedOutputQuery(pQuery)) { - tableFixedOutputProcess(pQInfo); + tableFixedOutputProcess(pQInfo, item->info); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); - tableMultiOutputProcess(pQInfo); + tableMultiOutputProcess(pQInfo, item->info); } // record the total elapsed time @@ -5340,26 +5279,26 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - - pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; + + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; + pColFilter->filterstr = htons(pFilterMsg->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterstr) { - pDestFilterInfo->len = htobe64(pFilterInfo->len); + if (pColFilter->filterstr) { + pColFilter->len = htobe64(pFilterMsg->len); - pDestFilterInfo->pz = (int64_t) calloc(1, pDestFilterInfo->len); - memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len); - pMsg += (pDestFilterInfo->len); + pColFilter->pz = (int64_t) calloc(1, pColFilter->len); + memcpy((void *)pColFilter->pz, pMsg, pColFilter->len); + pMsg += (pColFilter->len + 1); } else { - pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); - pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); + pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); + pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); } - pDestFilterInfo->lowerRelOptr = htons(pFilterInfo->lowerRelOptr); - pDestFilterInfo->upperRelOptr = htons(pFilterInfo->upperRelOptr); + pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); + pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); } } @@ -5756,9 +5695,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; - - SColumnInfo *pColInfo = &pQuery->colList[i]; - pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); + pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); } pQuery->tagColList = pTagCols; @@ -5833,7 +5770,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->pos = -1; pQuery->window = pQueryMsg->window; - pQuery->lastKey = pQuery->window.skey; if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); @@ -5910,9 +5846,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - - // qTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, - // pQInfo->refCount); + return code; _error: @@ -6085,34 +6019,35 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi if (pQueryMsg->numOfTables <= 0) { qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); code = TSDB_CODE_INVALID_QUERY_MSG; - goto _query_over; + goto _over; } if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; - goto _query_over; + goto _over; } SExprInfo *pExprs = NULL; if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - goto _query_over; + goto _over; } SSqlGroupbyExpr *pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code); if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _query_over; + goto _over; } bool isSTableQuery = false; STableGroupInfo groupInfo = {0}; + //todo multitable_query?? if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) { isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY); STableId *id = taosArrayGet(pTableIdList, 0); if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { - goto _query_over; + goto _over; } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; @@ -6129,7 +6064,7 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi numOfGroupByCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; - goto _query_over; + goto _over; } } else { assert(0); @@ -6138,11 +6073,12 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; + goto _over; } code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); -_query_over: +_over: tfree(tagCond); tfree(tbnameCond); taosArrayDestroy(pTableIdList); @@ -6318,12 +6254,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo check the return value, refactor codes if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - tsdbGetTableName(pQInfo->tsdb, &item->id, &data); + data = tsdbGetTableName(pQInfo->tsdb, &item->id, &bytes); char* dst = pQuery->sdata[j]->data + i * (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); - STR_WITH_MAXSIZE_TO_VARSTR(dst, data, TSDB_TABLE_NAME_LEN); - tfree(data); - + memcpy(dst, data, varDataTLen(data)); } else {// todo refactor, return the true length of binary|nchar data tsdbGetTableTagVal(pQInfo->tsdb, &item->id, pExprInfo[j].base.colInfo.colId, &type, &bytes, &data); assert(bytes == pExprInfo[j].bytes && type == pExprInfo[j].type); diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c index 3218b26179..41a888e92d 100644 --- a/src/query/src/queryFilterFunc.c +++ b/src/query/src/queryFilterFunc.c @@ -183,33 +183,33 @@ bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) == 0; } bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) == 0; } //////////////////////////////////////////////////////////////// bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; + return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; } bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; } //////////////////////////////////////////////////////////////// @@ -270,11 +270,11 @@ bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return true; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) != 0; } bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { @@ -282,7 +282,7 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return true; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } //////////////////////////////////////////////////////////////// diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index 0c4a2b6757..51d3286722 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -389,6 +389,7 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) { return 0; } +// todo handle the error static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { char tmpBuf[40] = {0}; @@ -424,7 +425,12 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { pVariant->wpz = (wchar_t *)tmp; } else { - taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL); + size_t output = -1; + taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output); + + if (pDestSize != NULL) { + *pDestSize = output; + } } return 0; @@ -779,7 +785,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) { } case TSDB_DATA_TYPE_NCHAR: { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL; + *(uint32_t *) payload = TSDB_DATA_NCHAR_NULL; } else { if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) { toNchar(pVariant, &payload, &pVariant->nLen); diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index 41156ce8ff..fa2b58a10b 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -76,4 +76,8 @@ TEST(testCase, patternMatchTest) { str = "carzero"; ret = patternMatch("%o", str, strlen(str), &info); EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "19"; + ret = patternMatch("%9", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 685b0f2b86..ca4b211be8 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -55,9 +55,8 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *); + void (*cfp)(SRpcMsg *, SRpcIpSet *); int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); - void (*ufp)(void *ahandle, SRpcIpSet *pIpSet); void *idPool; // handle to ID pool void *tmrCtrl; // handle to timer @@ -222,7 +221,6 @@ void *rpcOpen(const SRpcInit *pInit) { if (pInit->secret) strcpy(pRpc->secret, pInit->secret); if (pInit->ckey) strcpy(pRpc->ckey, pInit->ckey); pRpc->spi = pInit->spi; - pRpc->ufp = pInit->ufp; pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; @@ -793,7 +791,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { pConn->chandle = pRecv->chandle; pConn->peerIp = pRecv->ip; - if (pConn->peerPort == 0) pConn->peerPort = pRecv->port; + pConn->peerPort = pRecv->port; if (pHead->port) pConn->peerPort = htons(pHead->port); terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen); @@ -900,10 +898,11 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); } else { // for asynchronous API - if (pRpc->ufp && (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect)) - (*pRpc->ufp)(pContext->ahandle, &pContext->ipSet); // notify the update of ipSet + SRpcIpSet *pIpSet = NULL; + if (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect) + pIpSet = &pContext->ipSet; - (*pRpc->cfp)(pMsg); + (*pRpc->cfp)(pMsg, pIpSet); } // free the request message @@ -924,7 +923,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if ( rpcIsReq(pHead->msgType) ) { rpcMsg.handle = pConn; taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer); - (*(pRpc->cfp))(&rpcMsg); + (*(pRpc->cfp))(&rpcMsg, NULL); } else { // it's a response SRpcReqContext *pContext = pConn->pContext; @@ -1094,8 +1093,8 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { } else { if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) - tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, + tTrace( "%s %p, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", + pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 2aa1f0e4e9..ea1ebb5974 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -31,22 +31,16 @@ typedef struct { void *pRpc; } SInfo; -static void processResponse(SRpcMsg *pMsg) { +static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { SInfo *pInfo = (SInfo *)pMsg->handle; tTrace("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); + if (pIpSet) pInfo->ipSet = *pIpSet; + rpcFreeCont(pMsg->pCont); - sem_post(&pInfo->rspSem); } -static void processUpdateIpSet(void *handle, SRpcIpSet *pIpSet) { - SInfo *pInfo = (SInfo *)handle; - - tTrace("thread:%d, ip set is changed, index:%d", pInfo->index, pIpSet->inUse); - pInfo->ipSet = *pIpSet; -} - static int tcount = 0; static void *sendRequest(void *param) { @@ -99,7 +93,6 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.cfp = processResponse; - rpcInit.ufp = processUpdateIpSet; rpcInit.sessions = 100; rpcInit.idleTime = tsShellActivityTimer*1000; rpcInit.user = "michael"; diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c index 683cbb590a..3b19d7a9ea 100644 --- a/src/rpc/test/rsclient.c +++ b/src/rpc/test/rsclient.c @@ -32,12 +32,6 @@ typedef struct { void *pRpc; } SInfo; -static void processUpdateIpSet(void *handle, SRpcIpSet *pIpSet) { - SInfo *pInfo = (SInfo *)handle; - - tTrace("thread:%d, ip set is changed, index:%d", pInfo->index, pIpSet->inUse); - pInfo->ipSet = *pIpSet; -} static int tcount = 0; static int terror = 0; @@ -100,8 +94,6 @@ int main(int argc, char *argv[]) { rpcInit.localPort = 0; rpcInit.label = "APP"; rpcInit.numOfThreads = 1; - // rpcInit.cfp = processResponse; - rpcInit.ufp = processUpdateIpSet; rpcInit.sessions = 100; rpcInit.idleTime = tsShellActivityTimer*1000; rpcInit.user = "michael"; diff --git a/src/rpc/test/rserver.c b/src/rpc/test/rserver.c index 9f781ef276..958d099027 100644 --- a/src/rpc/test/rserver.c +++ b/src/rpc/test/rserver.c @@ -113,7 +113,7 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char return ret; } -void processRequestMsg(SRpcMsg *pMsg) { +void processRequestMsg(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 278463f0d2..fbf6e0f272 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -81,7 +81,6 @@ typedef struct { // ---------- TSDB TABLE DEFINITION typedef struct STable { int8_t type; - char * name; STableId tableId; int64_t superUid; // Super table UID int32_t sversion; @@ -96,9 +95,10 @@ typedef struct STable { TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure struct STable *next; // TODO: remove the next struct STable *prev; + tstr * name; // NOTE: there a flexible string here } STable; -#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey) +#define TSDB_GET_TABLE_LAST_KEY(tb) ((tb)->lastKey) void * tsdbEncodeTable(STable *pTable, int *contLen); STable *tsdbDecodeTable(void *cont, int contLen); @@ -121,6 +121,12 @@ typedef struct { int maxCols; } STsdbMeta; +// element put in skiplist for each table +typedef struct STableIndexElem { + STsdbMeta* pMeta; + STable* pTable; +} STableIndexElem; + STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables); int32_t tsdbFreeMeta(STsdbMeta *pMeta); STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable); @@ -145,12 +151,10 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable); STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo); -int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); -int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); -char * getTupleKey(const void *data); +char *getTSTupleKey(const void * data); typedef struct { int blockId; @@ -490,9 +494,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper); int tsdbWriteCompIdx(SRWHelper *pHelper); // --------- Other functions need to further organize -void tsdbFitRetention(STsdbRepo *pRepo); -int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); -void tsdbAdjustCacheBlocks(STsdbCache *pCache); +void tsdbFitRetention(STsdbRepo *pRepo); +int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); +void tsdbAdjustCacheBlocks(STsdbCache *pCache); +int32_t tsdbGetMetaFileName(char *rootDir, char *fname); #ifdef __cplusplus } diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 84f8a81eea..08970eab3e 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -164,7 +164,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { } tsdbUnLockRepo((TsdbRepoT *)pRepo); - tsdbTrace("tsdbId %d: tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); + tsdbTrace("vgId: %d tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); return 0; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 8d570911a9..a353f54353 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -7,6 +7,7 @@ #include "tscompression.h" #include "tchecksum.h" #include "ttime.h" +#include int tsdbDebugFlag = 135; @@ -133,6 +134,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) */ int32_t tsdbDropRepo(TsdbRepoT *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; + int id = pRepo->config.tsdbId; pRepo->state = TSDB_REPO_STATE_CLOSED; @@ -148,6 +150,8 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) { free(pRepo->rootDir); free(pRepo); + tsdbTrace("vgId: %d tsdb repository is dropped!", id); + return 0; } @@ -238,6 +242,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { pRepo->state = TSDB_REPO_STATE_ACTIVE; + tsdbTrace("vgId: %d open tsdb repository successfully!", pRepo->config.tsdbId); return (TsdbRepoT *)pRepo; } @@ -256,6 +261,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { int32_t tsdbCloseRepo(TsdbRepoT *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; if (pRepo == NULL) return 0; + int id = pRepo->config.tsdbId; pRepo->state = TSDB_REPO_STATE_CLOSED; tsdbLockRepo(repo); @@ -289,6 +295,8 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) { tfree(pRepo->rootDir); tfree(pRepo); + tsdbTrace("vgId: %d repository is closed!", id); + return 0; } @@ -349,6 +357,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); + tsdbTrace("vgId: %d start to commit!", pRepo->config.tsdbId); return 0; } @@ -376,11 +385,6 @@ STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) { return NULL; } -int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg); -} - int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) { // TODO return 0; @@ -395,13 +399,6 @@ TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) { return TSDB_GET_TABLE_LAST_KEY(pTable); } -int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { - if (repo == NULL) return -1; - STsdbRepo *pRepo = (STsdbRepo *)repo; - - return tsdbDropTableImpl(pRepo->tsdbMeta, tableId); -} - STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) { // TODO return NULL; @@ -756,12 +753,12 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) { if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1; if (mkdir(dirName, 0755) < 0) { - tsdbError("id %d: failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno)); + tsdbError("vgId: %d failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno)); return -1; } - tsdbError( - "id %d: set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep " + tsdbTrace( + "vgId: %d set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep " "%d, minRowsPerFileBlock %d, maxRowsPerFileBlock %d, precision %d, compression%d", pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); @@ -808,7 +805,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -831,7 +828,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable if (pTable->mem == NULL) { pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTupleKey); + pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); pTable->mem->keyFirst = INT64_MAX; pTable->mem->keyLast = 0; } @@ -842,6 +839,9 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); + tsdbTrace("vgId: %d, tid: %d, uid: " PRId64 "a row is inserted to table! key" PRId64, + pRepo->config.tsdbId, pTable->tableId.tid, pTable->tableId.uid, dataRowKey(row)); + return 0; } @@ -1104,11 +1104,14 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK } static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { + int8_t oldCompRession = pRepo->config.compression; pRepo->config.compression = compression; + tsdbTrace("vgId: %d tsdb compression is changed from %d to %d", oldCompRession, compression); } static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { STsdbCfg *pCfg = &pRepo->config; + int oldKeep = pCfg->keep; int maxFiles = keep / pCfg->maxTables + 3; if (pRepo->config.keep > keep) { @@ -1120,8 +1123,57 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { } pRepo->tsdbFileH->maxFGroups = maxFiles; } + tsdbTrace("vgId: %d keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); } static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { // TODO + int oldMaxTables = pRepo->config.maxTables; + tsdbTrace("vgId: %d tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); +} + +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) { + // TODO: need to refactor this function + + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char fname[256] = "\0"; + + struct stat fState; + char *spath = strdup(pRepo->rootDir); + char *prefixDir = dirname(spath); + + if (name[0] == 0) { + // Map index to the file name + int fid = (*index) / 3; + + if (fid > pFileH->numOfFGroups) { + // return meta data file + if ((*index) % 3 > 0) { // it is finished + tfree(spath); + return 0; + } else { + tsdbGetMetaFileName(pRepo->rootDir, fname); + } + } else { + // return data file name + strcpy(fname, pFileH->fGroup[fid].files[(*index) % 3].fname); + } + strcpy(name, fname + strlen(spath)); + } else { + // Name is provided, need to get the file info + sprintf(fname, "%s/%s", prefixDir, name); + } + + if (stat(fname, &fState) < 0) { + tfree(spath); + return 0; + } + + *size = fState.st_size; + magic = *size; + + return magic; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 0828db7f00..d5ba605e1a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1,6 +1,4 @@ #include - -// #include "taosdef.h" #include "tskiplist.h" #include "tsdb.h" #include "taosdef.h" @@ -10,6 +8,8 @@ #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here #define TSDB_META_FILE_NAME "META" +const int32_t DEFAULT_TAG_INDEX_COLUMN = 0; // skip list built based on the first column of tags + static int tsdbFreeTable(STable *pTable); static int32_t tsdbCheckTableCfg(STableCfg *pCfg); static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx); @@ -39,11 +39,12 @@ void *tsdbEncodeTable(STable *pTable, int *contLen) { void *ptr = ret; T_APPEND_MEMBER(ptr, pTable, STable, type); - // Encode name - *(int *)ptr = strlen(pTable->name); + // Encode name, todo refactor + *(int *)ptr = varDataLen(pTable->name); ptr = (char *)ptr + sizeof(int); - memcpy(ptr, pTable->name, strlen(pTable->name)); - ptr = (char *)ptr + strlen(pTable->name); + memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name)); + ptr = (char *)ptr + varDataLen(pTable->name); + T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid); T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid); T_APPEND_MEMBER(ptr, pTable, STable, superUid); @@ -79,9 +80,12 @@ STable *tsdbDecodeTable(void *cont, int contLen) { T_READ_MEMBER(ptr, int8_t, pTable->type); int len = *(int *)ptr; ptr = (char *)ptr + sizeof(int); - pTable->name = calloc(1, len + 1); + pTable->name = calloc(1, len + VARSTR_HEADER_SIZE); if (pTable->name == NULL) return NULL; - memcpy(pTable->name, ptr, len); + + varDataSetLen(pTable->name, len); + memcpy(pTable->name->data, ptr, len); + ptr = (char *)ptr + len; T_READ_MEMBER(ptr, int64_t, pTable->tableId.uid); T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid); @@ -105,8 +109,13 @@ void tsdbFreeEncode(void *cont) { } static char* getTagIndexKey(const void* pData) { - STable* table = *(STable**) pData; - return getTupleKey(table->tagVal); + STableIndexElem* elem = (STableIndexElem*) pData; + + SDataRow row = elem->pTable->tagVal; + STSchema* pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable); + STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; + + return tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); } int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { @@ -230,31 +239,23 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t STable* pTable = tsdbGetTableByUid(pMeta, id->uid); STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); - STColumn* pCol = NULL; - int32_t offset = 0; + + // todo binary search for(int32_t col = 0; col < schemaNCols(pSchema); ++col) { STColumn* p = schemaColAt(pSchema, col); if (p->colId == colId) { pCol = p; break; } - - if (p->type == TSDB_DATA_TYPE_BINARY || p->type == TSDB_DATA_TYPE_NCHAR) { - offset += sizeof(int32_t); - } else { - offset += p->bytes; - } } if (pCol == NULL) { return -1; // No matched tags. Maybe the modification of tags has not been done yet. } - assert(pCol != NULL); - SDataRow row = (SDataRow)pTable->tagVal; - char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + offset); + char* d = tdGetRowDataOfCol(row, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); *val = d; *type = pCol->type; @@ -263,19 +264,29 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t return TSDB_CODE_SUCCESS; } -int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name) { +char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes) { STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); - *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN); - if (*name == NULL) { - return -1; + if (pTable == NULL) { + if (bytes != NULL) { + *bytes = 0; + } + + return NULL; } else { - return 0; + if (bytes != NULL) { + *bytes = varDataLen(pTable->name); + } + + return (char*) pTable->name; } } -int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { +int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + if (tsdbCheckTableCfg(pCfg) < 0) return -1; STable *super = NULL; @@ -296,7 +307,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { super->schema = tdDupSchema(pCfg->schema); super->tagSchema = tdDupSchema(pCfg->tagSchema); super->tagVal = NULL; - super->name = strdup(pCfg->sname); + + // todo refactor extract method + size_t size = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN); + super->name = malloc(size + VARSTR_HEADER_SIZE); + STR_WITH_SIZE_TO_VARSTR(super->name, pCfg->sname, size); // index the first tag column STColumn* pColSchema = schemaColAt(super->tagSchema, 0); @@ -322,7 +337,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { } table->tableId = pCfg->tableId; - table->name = strdup(pCfg->name); + + size_t size = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN); + table->name = malloc(size + VARSTR_HEADER_SIZE); + STR_WITH_SIZE_TO_VARSTR(table->name, pCfg->name, size); + table->lastKey = 0; if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE table->type = TSDB_CHILD_TABLE; @@ -335,8 +354,14 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { } // Register to meta - if (newSuper) tsdbAddTableToMeta(pMeta, super, true); + if (newSuper) { + tsdbAddTableToMeta(pMeta, super, true); + tsdbTrace("vgId: %d super table is created! uid " PRId64, pRepo->config.tsdbId, + super->tableId.uid); + } tsdbAddTableToMeta(pMeta, table, true); + tsdbTrace("vgId: %d table is created! tid %d, uid " PRId64, pRepo->config.tsdbId, table->tableId.tid, + table->tableId.uid); // Write to meta file int bufLen = 0; @@ -369,13 +394,24 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId) { return pTable; } -int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { +// int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { +int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + if (pRepo == NULL) return -1; + + STsdbMeta *pMeta = pRepo->tsdbMeta; if (pMeta == NULL) return -1; STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); - if (pTable == NULL) return -1; + if (pTable == NULL) { + tsdbError("vgId %d: failed to drop table since table not exists! tid %d, uid " PRId64, pRepo->config.tsdbId, + tableId.tid, tableId.uid); + return -1; + } + tsdbTrace("vgId: %d table is dropped! tid %d, uid " PRId64, pRepo->config.tsdbId, tableId.tid, tableId.uid); if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1; + return 0; } @@ -467,7 +503,9 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) { if (pTable->type == TSDB_SUPER_TABLE) { SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STableIndexElem *pEle = (STableIndexElem *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = pEle->pTable; + ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE); pMeta->tables[tTable->tableId.tid] = NULL; @@ -513,11 +551,14 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function - SSkipListNode* pNode = calloc(1, headSize + POINTER_BYTES); + SSkipListNode* pNode = calloc(1, headSize + sizeof(STableIndexElem)); pNode->level = level; SSkipList* list = pSTable->pIndex; - memcpy(SL_GET_NODE_DATA(pNode), &pTable, POINTER_BYTES); + STableIndexElem* elem = (STableIndexElem*) (SL_GET_NODE_DATA(pNode)); + + elem->pTable = pTable; + elem->pMeta = pMeta; tSkipListPut(list, pNode); return 0; @@ -529,7 +570,10 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); assert(pSTable != NULL); - char* key = dataRowTuple(pTable->tagVal); // key + STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); + STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; + + char* key = tdGetRowDataOfCol(pTable->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); bool ret = tSkipListRemove(pSTable->pIndex, key); assert(ret); @@ -539,7 +583,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { static int tsdbEstimateTableEncodeSize(STable *pTable) { int size = 0; size += T_MEMBER_SIZE(STable, type); - size += sizeof(int) + strlen(pTable->name); + size += sizeof(int) + varDataLen(pTable->name); size += T_MEMBER_SIZE(STable, tableId); size += T_MEMBER_SIZE(STable, superUid); size += T_MEMBER_SIZE(STable, sversion); @@ -556,8 +600,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) { return size; } -char *getTupleKey(const void * data) { +char *getTSTupleKey(const void * data) { SDataRow row = (SDataRow)data; - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c index 6821fc2d98..f9d10ec579 100644 --- a/src/tsdb/src/tsdbMetaFile.c +++ b/src/tsdb/src/tsdbMetaFile.c @@ -28,7 +28,7 @@ typedef struct { int64_t uid; } SRecordInfo; -static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); +// static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); // static int32_t tsdbCheckMetaHeader(int fd); static int32_t tsdbWriteMetaHeader(int fd); static int tsdbCreateMetaFile(char *fname); @@ -180,7 +180,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) { tfree(mfh); } -static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { +int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { if (rootDir == NULL) return -1; sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME); return 0; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 008ad223ed..84615ff8c2 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -620,9 +620,13 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf if (pCol->info.type != TSDB_DATA_TYPE_BINARY && pCol->info.type != TSDB_DATA_TYPE_NCHAR) { memmove(pCol->pData, src->pData + bytes * start, bytes * pQueryHandle->realNumOfRows); } else { // handle the var-string + char* dst = pCol->pData; + + // todo refactor, only copy one-by-one for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) { char* p = tdGetColDataOfRow(src, k); - memcpy(pCol->pData + k * bytes, p, varDataTLen(p)); // todo refactor + memcpy(dst, p, varDataTLen(p)); + dst += bytes; } } @@ -1076,17 +1080,13 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY } assert(offset != -1); // todo handle error + void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - void *value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset); memcpy(pData, value, varDataTLen(value)); - - offset += sizeof(int32_t); } else { - memcpy(pData, dataRowTuple(row) + offset, pColInfo->info.bytes); - offset += pColInfo->info.bytes; + memcpy(pData, value, pColInfo->info.bytes); } - } numOfRows++; @@ -1225,8 +1225,8 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) { while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable* t = *(STable**)SL_GET_NODE_DATA(pNode); - taosArrayPush(list, &t->tableId); + STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*) pNode)); + taosArrayPush(list, &elem->pTable->tableId); } tSkipListDestroyIter(iter); @@ -1235,6 +1235,7 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) { /** * convert the result pointer to table id instead of table object pointer + * todo remove it by using callback function to change the final result in-time. * @param pRes */ static void convertQueryResult(SArray* pRes, SArray* pTableList) { @@ -1244,8 +1245,8 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) { size_t size = taosArrayGetSize(pTableList); for (int32_t i = 0; i < size; ++i) { // todo speedup by using reserve space. - STable* pTable = taosArrayGetP(pTableList, i); - taosArrayPush(pRes, &pTable->tableId); + STableIndexElem* elem = taosArrayGet(pTableList, i); + taosArrayPush(pRes, &elem->pTable->tableId); } } @@ -1309,7 +1310,12 @@ void filterPrepare(void* expr, void* param) { pInfo->q = (char*) pCond->arr; } else { pInfo->q = calloc(1, pSchema->bytes); - tVariantDump(pCond, pInfo->q, pSchema->type); + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + tVariantDump(pCond, varDataVal(pInfo->q), pSchema->type); + varDataSetLen(pInfo->q, pCond->nLen); // the length may be changed after dump, so assign its value after dump + } else { + tVariantDump(pCond, pInfo->q, pSchema->type); + } } } @@ -1341,16 +1347,16 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { int32_t bytes = 0; if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { - f1 = pTable1->name; - f2 = pTable2->name; + f1 = (char*) pTable1->name; + f2 = (char*) pTable2->name; type = TSDB_DATA_TYPE_BINARY; - bytes = TSDB_TABLE_NAME_LEN; + bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; } else { - f1 = dataRowTuple(pTable1->tagVal); - f2 = dataRowTuple(pTable2->tagVal); - - type = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->type; - bytes = schemaColAt(pTableGroupSupp->pTagSchema, colIndex)->bytes; + STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); + bytes = pCol->bytes; + + f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); } int32_t ret = doCompare(f1, f2, type, bytes); @@ -1426,26 +1432,22 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -bool tSkipListNodeFilterCallback(const void* pNode, void* param) { +bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - - STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + + STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; int8_t type = pInfo->sch.type; if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { - val = pTable->name; + val = (char*) elem->pTable->name; type = TSDB_DATA_TYPE_BINARY; } else { STSchema* pTSchema = (STSchema*) pInfo->param; // todo table schema is identical to stable schema?? int32_t offset = pTSchema->columns[pInfo->colIndex].offset; - if (pInfo->sch.type == TSDB_DATA_TYPE_BINARY || pInfo->sch.type == TSDB_DATA_TYPE_NCHAR) { - val = tdGetRowDataOfCol(pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset); - } else { - val = dataRowTuple(pTable->tagVal) + offset; - } + val = tdGetRowDataOfCol(elem->pTable->tagVal, pInfo->sch.type, TD_DATA_ROW_HEAD_SIZE + offset); } int32_t ret = 0; @@ -1456,8 +1458,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { ret = pInfo->compare(val, pInfo->q); } } else { -// tVariant t = {0}; -// tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); ret = pInfo->compare(val, pInfo->q); } @@ -1497,12 +1497,12 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .fp = (__result_filter_fn_t) tSkipListNodeFilterCallback, + .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; - SArray* pTableList = taosArrayInit(8, POINTER_BYTES); + SArray* pTableList = taosArrayInit(8, sizeof(STableIndexElem)); tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp); tExprTreeDestroy(&pExpr, destroyHelper); diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 32978453fc..f343912cde 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -144,11 +144,11 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const } if (leftPartEnd > start) { - tqsortImpl(src, size, start, leftPartEnd, param, comparFn, buf); + tqsortImpl(src, start, leftPartEnd, size, param, comparFn, buf); } if (rightPartStart < end) { - tqsortImpl(src, size, rightPartStart, end, param, comparFn, buf); + tqsortImpl(src, rightPartStart, end, size, param, comparFn, buf); } } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 2d6ae13f97..4bac669dc0 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -51,42 +51,54 @@ int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { } } +int32_t compareFloatVal(const void *pLeft, const void *pRight) { + float ret = GET_FLOAT_VAL(pLeft) - GET_FLOAT_VAL(pRight); + if (fabs(ret) < FLT_EPSILON) { + return 0; + } else { + return ret > 0? 1 : -1; + } +} + int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); if (fabs(ret) < FLT_EPSILON) { return 0; } else { - return ret > 0 ? 1 : -1; + return ret > 0? 1 : -1; } } -int32_t compareStrVal(const void *pLeft, const void *pRight) { - return (int32_t)strcmp(pLeft, pRight); +int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { + int32_t len1 = varDataLen(pLeft); + int32_t len2 = varDataLen(pRight); + + if (len1 != len2) { + return len1 > len2? 1:-1; + } else { + int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1:-1; + } + } } -int32_t compareWStrVal(const void *pLeft, const void *pRight) { - // SSkipListKey *pL = (SSkipListKey *)pLeft; - // SSkipListKey *pR = (SSkipListKey *)pRight; - // - // if (pL->nLen == 0 && pR->nLen == 0) { - // return 0; - // } - // - // // handle only one-side bound compare situation, there is only lower bound or only upper bound - // if (pL->nLen == -1) { - // return 1; // no lower bound, lower bound is minimum, always return -1; - // } else if (pR->nLen == -1) { - // return -1; // no upper bound, upper bound is maximum situation, always return 1; - // } - // - // int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz); - // - // if (ret == 0) { - // return 0; - // } else { - // return ret > 0 ? 1 : -1; - // } - return 0; +int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { + int32_t len1 = varDataLen(pLeft); + int32_t len2 = varDataLen(pRight); + + if (len1 != len2) { + return len1 > len2? 1:-1; + } else { + int32_t ret = wcsncmp(varDataVal(pLeft), varDataVal(pRight), len1); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1 : -1; + } + } } /* @@ -124,7 +136,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat size_t n = strcspn(str, next); str += n; - if (str[0] == 0 || (n >= size - 1)) { + if (str[0] == 0 || (n >= size)) { break; } @@ -172,10 +184,10 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcsspn(str, accept); + size_t n = wcscspn(str, accept); str += n; - if (str[0] == 0 || (n >= size - 1)) { + if (str[0] == 0 || (n >= size)) { break; } @@ -254,7 +266,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_FLOAT: { - comparFn = compareDoubleVal; break; + comparFn = compareFloatVal; break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -267,7 +279,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } else if (optr == TSDB_RELATION_IN) { comparFn = compareFindStrInArray; } else { /* normal relational comparFn */ - comparFn = compareStrVal; + comparFn = compareLenPrefixedStr; } break; @@ -277,7 +289,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { if (optr == TSDB_RELATION_LIKE) { comparFn = compareWStrPatternComp; } else { - comparFn = compareWStrVal; + comparFn = compareLenPrefixedWStr; } break; @@ -296,6 +308,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { switch (keyType) { case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: comparFn = compareInt8Val; break; case TSDB_DATA_TYPE_SMALLINT: @@ -308,21 +321,19 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_BOOL: - comparFn = compareInt32Val; - break; - case TSDB_DATA_TYPE_FLOAT: + comparFn = compareFloatVal; + break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: - comparFn = compareStrVal; + comparFn = compareLenPrefixedStr; break; case TSDB_DATA_TYPE_NCHAR: - comparFn = compareWStrVal; + comparFn = compareLenPrefixedWStr; break; default: @@ -349,13 +360,20 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { } return (ret < 0) ? -1 : 1; } - default: { - int32_t ret = strncmp(f1, f2, (size_t)size); - if (ret == 0) { - return ret; - } + default: { // todo refactor + tstr* t1 = (tstr*) f1; + tstr* t2 = (tstr*) f2; - return (ret < 0) ? -1 : 1; + if (t1->len != t2->len) { + return t1->len > t2->len? 1:-1; + } else { + int32_t ret = strncmp(t1->data, t2->data, t1->len); + if (ret == 0) { + return 0; + } else { + return ret < 0? -1:1; + } + } } } } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index dfbdc5e35d..61ae647bf4 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -236,7 +236,7 @@ void taosReadGlobalLogCfg() { int olen, vlen; char fileName[PATH_MAX] = {0}; - mdebugFlag = 135; + mDebugFlag = 135; sdbDebugFlag = 135; wordexp_t full_path; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 16845812bf..c7e7c11506 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -267,7 +267,7 @@ void vnodeRelease(void *pVnodeRaw) { assert(refCount >= 0); if (refCount > 0) { - vTrace("vgId:%d, release vnode, refCount:%d", pVnode, vgId, refCount); + vTrace("vgId:%d, release vnode, refCount:%d", vgId, refCount); return; } @@ -383,9 +383,8 @@ static int vnodeWalCallback(void *arg) { } static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) { - // SVnodeObj *pVnode = ahandle; - //tsdbGetFileInfo(pVnode->tsdb, name, index, size); - return 0; + SVnodeObj *pVnode = ahandle; + return tsdbGetFileInfo(pVnode->tsdb, name, index, size); } static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { @@ -405,9 +404,9 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { static void vnodeNotifyFileSynced(void *ahandle) { SVnodeObj *pVnode = ahandle; - vTrace("pVnode:%p vgId:%d, data file is synced", pVnode, pVnode->vgId); + vTrace("vgId:%d, data file is synced", pVnode->vgId); - // clsoe tsdb, then open tsdb + // close tsdb, then open tsdb } static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { @@ -693,8 +692,9 @@ static bool vnodeReadVersion(SVnodeObj *pVnode) { sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(versionFile, "r"); if (!fp) { - vTrace("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, - versionFile, strerror(errno)); + if (errno != ENOENT) { + vError("vgId:%d, failed to open version file:%s error:%s", pVnode->vgId, versionFile, strerror(errno)); + } return false; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index d2dd1811cf..03407a5aa2 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -65,7 +65,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont pRet->len = sizeof(SQueryTableRsp); pRet->rsp = pRsp; - vTrace("vgId:%d QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); } else { pQInfo = pCont; code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -83,7 +83,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont int32_t code = TSDB_CODE_SUCCESS; - vTrace("vgId:%d QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo); pRet->code = qRetrieveQueryResultInfo(pQInfo); if (pRet->code != TSDB_CODE_SUCCESS) { @@ -104,6 +104,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont } } - vTrace("vgId:%d QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); return code; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index f560c648e1..ec0a3b2f0b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -51,10 +51,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status != TAOS_VN_STATUS_READY && qtype == TAOS_QTYPE_RPC) - return TSDB_CODE_NOT_ACTIVE_VNODE; - - if (pHead->version == 0) { // from client + if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; @@ -64,12 +61,10 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // assign version pVnode->version++; pHead->version = pVnode->version; - } else { + } else { // from wal or forward // for data from WAL or forward, version may be smaller if (pHead->version <= pVnode->version) return 0; } - - // more status and role checking here pVnode->version = pHead->version; @@ -77,9 +72,13 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { code = walWrite(pVnode->wal, pHead); if (code < 0) return code; - int32_t syncCode = syncForwardToPeer(pVnode->sync, pHead, item); + // forward to peers if data is from RPC or CQ + int32_t syncCode = 0; + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_CQ) + syncCode = syncForwardToPeer(pVnode->sync, pHead, item); if (syncCode < 0) return syncCode; + // write data locally code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); if (code < 0) return code; @@ -239,7 +238,7 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet code = tsdbDropTable(pVnode->tsdb, stableId); - vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode, pTable->tableId, tstrerror(code)); + vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code)); return code; } diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py index 73a412fb80..b5e53d862e 100644 --- a/tests/pytest/import_merge/importDataH2.py +++ b/tests/pytest/import_merge/importDataH2.py @@ -46,16 +46,16 @@ class TDTestCase: self.maxrows) tdLog.info("================= step2") - tdLog.info("import %d sequential data" % (self.maxrows / 2)) + tdLog.info("import %d sequential data" % (self.maxrows // 2)) startTime = self.startTime sqlcmd = ['import into tb1 values'] - for rid in range(1, self.maxrows / 2 + 1): + for rid in range(1, self.maxrows // 2 + 1): sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) tdSql.execute(" ".join(sqlcmd)) tdLog.info("================= step3") tdSql.query('select * from tb1') - tdSql.checkRows(self.maxrows / 2) + tdSql.checkRows(self.maxrows // 2) tdLog.info("================= step4") tdDnodes.stop(1) @@ -70,7 +70,7 @@ class TDTestCase: tdLog.info("================= step7") tdSql.execute('reset query cache') tdSql.query('select * from tb1 order by ts desc') - tdSql.checkRows(self.maxrows / 2 + 1) + tdSql.checkRows(self.maxrows // 2 + 1) tdLog.info("================= step8") tdLog.info("import 10 data in batch before") @@ -83,7 +83,7 @@ class TDTestCase: tdLog.info("================= step9") tdSql.execute('reset query cache') tdSql.query('select * from tb1 order by ts desc') - tdSql.checkRows(self.maxrows / 2 + 11) + tdSql.checkRows(self.maxrows // 2 + 11) def stop(self): tdSql.close() diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py index 2359ca214f..e946a254c2 100644 --- a/tests/pytest/import_merge/importDataSub.py +++ b/tests/pytest/import_merge/importDataSub.py @@ -46,16 +46,17 @@ class TDTestCase: self.maxrows) tdLog.info("================= step2") - tdLog.info("import %d sequential data" % (self.maxrows / 2)) + tdLog.info("import %d sequential data" % (self.maxrows // 2)) startTime = self.startTime sqlcmd = ['import into tb1 values'] - for rid in range(1, self.maxrows / 2 + 1): + for rid in range(1, self.maxrows // 2 + 1): sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) tdLog.info("================= step3") tdSql.query('select * from tb1') - tdSql.checkRows(self.maxrows / 2) + tdSql.checkRows(self.maxrows // 2) tdLog.info("================= step4") tdDnodes.stop(1) @@ -73,7 +74,7 @@ class TDTestCase: tdLog.info("================= step9") tdSql.execute('reset query cache') tdSql.query('select * from tb1 order by ts desc') - tdSql.checkRows(self.maxrows / 2) + tdSql.checkRows(self.maxrows // 2) def stop(self): tdSql.close() diff --git a/tests/pytest/import_merge/importToCommit.py b/tests/pytest/import_merge/importToCommit.py index 7a408bcdce..3684dde049 100644 --- a/tests/pytest/import_merge/importToCommit.py +++ b/tests/pytest/import_merge/importToCommit.py @@ -33,7 +33,7 @@ class TDTestCase: tdDnodes.start(1) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') - tdSql.execute('create database db cache 512 tables 10') + tdSql.execute('create database db cache 512 maxtables 10') tdSql.execute('use db') tdLog.info("================= step1") diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index e51e8792ec..853ebe1d76 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -29,88 +29,17 @@ python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f table/db_table.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importDataLastTO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importDataLastT.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importDataTO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importDataT.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHeadOverlap.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHORestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHPORestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHRestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importLastSub.py -python3 ./test.py $1 -s && sleep 1 - -python3 ./test.py $1 -f import_merge/importBlock1HO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1HPO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1H.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1S.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1Sub.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1TO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1TPO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock1T.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2HO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2HPO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2H.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2S.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2Sub.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2TO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2TPO.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlock2T.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importBlockbetween.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importCacheFileSub.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importCacheFileTO.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importCacheFileT.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importDataLastSub.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importHead.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importLastTO.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importLastT.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importSpan.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importSRestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importSubRestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTailOverlap.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importTail.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTORestart.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTPORestart.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importTRestart.py python3 ./test.py $1 -s && sleep 1 diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index e492a35c20..23adab2c47 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -201,8 +201,8 @@ class TDDnode: self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - tdLog.debug("wait 2 seconds for the dnode:%d to start." % (self.index)) - time.sleep(2) + tdLog.debug("wait 4 seconds for the dnode:%d to start." % (self.index)) + time.sleep(4) def stop(self): if self.valgrind == 0: diff --git a/tests/pytest/valgrind-test.sh b/tests/pytest/valgrind-test.sh new file mode 100755 index 0000000000..853ebe1d76 --- /dev/null +++ b/tests/pytest/valgrind-test.sh @@ -0,0 +1,45 @@ +#!/bin/bash +python3 ./test.py $1 -f insert/basic.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/int.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/float.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/bool.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/double.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/smallint.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/tinyint.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/binary.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/date.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/nchar.py +python3 ./test.py $1 -s && sleep 1 + +python3 ./test.py $1 -f table/column_name.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f table/column_num.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f table/db_table.py +python3 ./test.py $1 -s && sleep 1 + +python3 ./test.py $1 -f import_merge/importCacheFileT.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importDataLastSub.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importHead.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importLastT.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importSpan.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importTail.py +python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f import_merge/importTRestart.py +python3 ./test.py $1 -s && sleep 1 diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index c8b21c6856..d0bb58ec20 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -1,27 +1,22 @@ -################################# run general/cache/new_metrics.sim run general/compress/compress.sim run general/compute/avg.sim -run general/compute/bottom.sim run general/compute/count.sim run general/db/len.sim run general/db/basic4.sim run general/http/restful_insert.sim run general/import/basic.sim run general/import/commit.sim -run general/insert/basic.sim run general/insert/query_file_memory.sim run general/parser/binary_escapeCharacter.sim run general/parser/columnValue_bigint.sim run general/parser/select_from_cache_disk.sim run general/table/autocreate.sim -run general/table/basic3.sim run general/table/column_name.sim run general/table/int.sim run general/table/vgroup.sim run general/user/basic1.sim -run general/user/user_len.sim +run general/user/pass_alter.sim run general/vector/single.sim run general/connection/connection.sim run general/user/authority.sim -################################## diff --git a/tests/script/general/cache/restart_metrics.sim b/tests/script/general/cache/restart_metrics.sim index dbd15b945f..c85a66869a 100644 --- a/tests/script/general/cache/restart_metrics.sim +++ b/tests/script/general/cache/restart_metrics.sim @@ -27,11 +27,15 @@ sql create table $tb using $mt tags( "1" ) sql insert into $tb values (now, 1) sql select * from $tb -print ===>rows $rows, data $data01 +#print ===>rows $rows, data $data01 + if $rows != 1 then + print expect 1, actual: $rows return -1 -endi +endi + if $data01 != 1 then + print expect 1 actual: $data01 return -1 endi @@ -66,8 +70,10 @@ sql select * from $tb print ===>rows $rows, data $data01 if $rows != 1 then return -1 -endi +endi + if $data01 != 1 then + print expect 1, actual $data01 return -1 endi @@ -80,6 +86,7 @@ if $data01 != 1 then return -1 endi if $data02 != 3 then + print expect 3 actual: $data02 return -1 endi diff --git a/tests/script/general/compute/diff2.sim b/tests/script/general/compute/diff2.sim index 53b508ca6f..0be907c16a 100644 --- a/tests/script/general/compute/diff2.sim +++ b/tests/script/general/compute/diff2.sim @@ -81,7 +81,7 @@ sql_error select diff(c8) from $tb sql_error select diff(c9) from $tb sql_error select diff(ts) from $tb sql_error select diff(c1), diff(c2) from $tb -sql_error select 2+diff(c1) from $tb +#sql_error select 2+diff(c1) from $tb sql_error select diff(c1+2) from $tb sql_error select diff(c1) from $tb where ts > 0 and ts < now + 100m interval(10m) sql_error select diff(c1) from $mt diff --git a/tests/script/general/db/delete_reuse2.sim b/tests/script/general/db/delete_reuse2.sim index 73a9dae442..d2fcaad838 100644 --- a/tests/script/general/db/delete_reuse2.sim +++ b/tests/script/general/db/delete_reuse2.sim @@ -26,6 +26,7 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 print ========= start dnodes +sleep 2000 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect diff --git a/tests/script/general/db/delete_reusevnode2.sim b/tests/script/general/db/delete_reusevnode2.sim index 12357b9e44..ff9361260c 100644 --- a/tests/script/general/db/delete_reusevnode2.sim +++ b/tests/script/general/db/delete_reusevnode2.sim @@ -1,7 +1,5 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 - system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 10 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10 @@ -36,7 +34,7 @@ while $i < 10 sql create table $tb1 using st tags(1) sql insert into $tb1 values(now, 1); - $tb1 = $tb . 5 + $tb1 = $tb . 4 sql create table $tb1 using st tags(1) sql insert into $tb1 values(now, 1); @@ -58,7 +56,7 @@ while $i < 10 sql drop table st - sleep 1000 + sleep 2000 print times $i $i = $i + 1 diff --git a/tests/script/general/db/testSuite.sim b/tests/script/general/db/testSuite.sim index 2cac8b8fa8..87863001a3 100644 --- a/tests/script/general/db/testSuite.sim +++ b/tests/script/general/db/testSuite.sim @@ -1,4 +1,4 @@ -run general//db/basic.sim +run general/db/basic.sim run general/db/basic1.sim run general/db/basic2.sim run general/db/basic3.sim @@ -13,3 +13,4 @@ run general/db/delete_writing2.sim run general/db/len.sim run general/db/repeat.sim run general/db/tables.sim +run general/db/vnodes.sim diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim new file mode 100644 index 0000000000..684910884b --- /dev/null +++ b/tests/script/general/db/vnodes.sim @@ -0,0 +1,45 @@ +system sh/stop_dnodes.sh + +$totalVnodes = 10 +$maxTables = 4 +$totalRows = $totalVnodes * $maxTables + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $maxTables +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes +system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 +system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000 +system sh/cfg.sh -n dnode1 -c maxShellConns -v 100000 +system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 100000 + +print ========== prepare data +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect +sql create database db blocks 2 cache 1 maxTables $maxTables +sql use db + +print ========== step1 +sql create table mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$x = 0 +while $x < $totalRows + $tb = t . $x + sql create table $tb using mt tags( $x ) + sql insert into $tb values (now, $x ) + $x = $x + 1 +endw + +print ========== step2 +sql select * from mt +if $rows != $totalRows then + return -1 +endi + +sql select count(*) from mt +if $data00 != $totalRows then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/field/bigint.sim b/tests/script/general/field/bigint.sim index 5cca73595c..10060f7422 100644 --- a/tests/script/general/field/bigint.sim +++ b/tests/script/general/field/bigint.sim @@ -132,7 +132,9 @@ endi print =============== step6 sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = 1 group by tgcol print $data00 $data01 $data02 $data03 $data04 $data05 $data06 -if $data00 != 100 then +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 +if $data00 != 100 then + print expect 100, actual $data00 return -1 endi diff --git a/tests/script/general/parser/auto_create_tb.sim b/tests/script/general/parser/auto_create_tb.sim index 8fb0ddae54..54e028a200 100644 --- a/tests/script/general/parser/auto_create_tb.sim +++ b/tests/script/general/parser/auto_create_tb.sim @@ -96,25 +96,25 @@ $ts1 = $ts0 + 1000 $ts2 = $ts0 + 2000 sql insert into tb_1 using $stb tags (-1) values ( $ts1 , 1,1,1,1,'bin',1,1,1,'涛思数据') ( $ts2 , 2,2,2,2,'binar', 1,1,1,'nchar') sql select * from $stb -if $rows != 3 then +if $rows != 5 then return -1 endi -if $data19 != 涛思数据 then +if $data09 != 涛思数据 then return -1 endi -if $data11 != 1 then +if $data01 != 1 then return -1 endi -if $data22 != 2 then +if $data42 != 2 then return -2 endi -if $data23 != 2.00000 then +if $data43 != 2.00000 then return -1 endi -if $data25 != binar then +if $data45 != binar then return -1 endi -if $data29 != nchar then +if $data49 != nchar then return -1 endi sql drop table tb_1 @@ -127,22 +127,22 @@ sql select * from $stb if $rows != 5 then return -1 endi -if $data19 != 涛思数据 then +if $data09 != 涛思数据 then return -1 endi -if $data11 != 1 then +if $data01 != 1 then return -1 endi -if $data22 != 2 then +if $data42 != 2 then return -2 endi -if $data23 != 2.00000 then +if $data43 != 2.00000 then return -1 endi -if $data25 != binar then +if $data45 != binar then return -1 endi -if $data29 != nchar then +if $data49 != nchar then return -1 endi @@ -154,13 +154,13 @@ sql show tables if $rows != 3 then return -1 endi -if $data00 != tb3 then +if $data00 != tb1 then return -1 endi if $data10 != tb2 then return -1 endi -if $data20 != tb1 then +if $data20 != tb3 then return -1 endi diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index a5e003fb65..e43cc15173 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -43,9 +43,11 @@ if $data07 != 1 then return -1 endi if $data08 != BINARY then + print expect BINARY actual: $data08 return -1 endi if $data09 != NCHAR then + print expect NCHAR actual: $data09 return -1 endi diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index fbdc12446a..2a6bc83b41 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -194,8 +194,10 @@ sql reset query cache sql select * from $mt where tgcol = '1' step2: if $rows != 100 then + print expect 100, actual: $rows return -1 endi + sql select * from $mt where tgcol > '0' #print rows = $rows if $rows != 100 then diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aaf8979337..fb56868f1f 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,113 +1,100 @@ -# run general/parser/alter.sim -# sleep 2000 -# run general/parser/alter1.sim -# sleep 2000 -# run general/parser/alter_stable.sim -# sleep 2000 -# run general/parser/auto_create_tb.sim -# sleep 2000 -# run general/parser/auto_create_tb_drop_tb.sim -# sleep 2000 +#run general/parser/alter.sim +#sleep 2000 +#run general/parser/alter1.sim +#sleep 2000 +#run general/parser/alter_stable.sim +#sleep 2000 +#run general/parser/auto_create_tb.sim +#sleep 2000 +#run general/parser/auto_create_tb_drop_tb.sim -run general/parser/binary_escapeCharacter.sim -sleep 2000 -run general/parser/bug.sim sleep 2000 run general/parser/col_arithmetic_operation.sim sleep 2000 -run general/parser/columnValue_bigint.sim +run general/parser/columnValue.sim sleep 2000 -run general/parser/columnValue_bool.sim +run general/parser/commit.sim sleep 2000 -run general/parser/columnValue_double.sim +run general/parser/create_db.sim sleep 2000 -run general/parser/columnValue_float.sim +run general/parser/create_mt.sim sleep 2000 -run general/parser/columnValue_int.sim +run general/parser/create_tb.sim sleep 2000 +run general/parser/dbtbnameValidate.sim +sleep 2000 +run general/parser/import_commit1.sim +sleep 2000 +run general/parser/import_commit2.sim +sleep 2000 +run general/parser/import_commit3.sim +sleep 2000 +run general/parser/insert_tb.sim +sleep 2000 +run general/parser/first_last.sim +sleep 2000 +run general/parser/import_file.sim +sleep 2000 +run general/parser/lastrow.sim +sleep 2000 +run general/parser/nchar.sim -# sleep 2000 -# run general/parser/col_arithmetic_operation.sim -# sleep 2000 -# run general/parser/columnValue.sim -# sleep 2000 -# run general/parser/commit.sim -# run general/parser/create_db.sim -# sleep 2000 -# run general/parser/create_mt.sim -# sleep 2000 -# run general/parser/create_tb.sim -# sleep 2000 -# run general/parser/dbtbnameValidate.sim -# sleep 2000 -# run general/parser/fill.sim -# sleep 2000 -# run general/parser/fill_stb.sim -# sleep 2000 -# run general/parser/first_last.sim -# sleep 2000 -# run general/parser/import_commit1.sim -# sleep 2000 -# run general/parser/import_commit2.sim -# sleep 2000 -# run general/parser/import_commit3.sim -# sleep 2000 -# run general/parser/import_file.sim -# sleep 2000 -# run general/parser/insert_tb.sim -# sleep 2000 -# run general/parser/tags_dynamically_specifiy.sim -# sleep 2000 -# run general/parser/interp.sim -# run general/parser/lastrow.sim -# sleep 2000 -# run general/parser/limit.sim -# sleep 2000 -# run general/parser/limit1.sim -# sleep 2000 -# run general/parser/limit1_tblocks100.sim -# sleep 2000 -# run general/parser/limit2.sim -# sleep 2000 -# run general/parser/mixed_blocks.sim -# sleep 2000 -# run general/parser/nchar.sim -# sleep 2000 -# run general/parser/null_char.sim -# sleep 2000 -# run general/parser/selectResNum.sim -# sleep 2000 -# run general/parser/select_across_vnodes.sim -# sleep 2000 +sleep 2000 +run general/parser/null_char.sim +sleep 2000 +run general/parser/single_row_in_tb.sim +sleep 2000 run general/parser/select_from_cache_disk.sim sleep 2000 -# run general/parser/set_tag_vals.sim -# sleep 2000 -# run general/parser/single_row_in_tb.sim -# sleep 2000 -# run general/parser/slimit.sim -# sleep 2000 +run general/parser/limit.sim + +sleep 2000 +run general/parser/fill.sim +sleep 2000 +run general/parser/fill_stb.sim +sleep 2000 +run general/parser/tags_dynamically_specifiy.sim +sleep 2000 +run general/parser/interp.sim +sleep 2000 +run general/parser/limit1.sim +sleep 2000 +run general/parser/limit1_tblocks100.sim +sleep 2000 +run general/parser/limit2.sim +sleep 2000 +run general/parser/mixed_blocks.sim +sleep 2000 +run general/parser/selectResNum.sim +sleep 2000 +run general/parser/select_across_vnodes.sim +sleep 2000 +run general/parser/set_tag_vals.sim +sleep 2000 +run general/parser/slimit.sim +sleep 2000 run general/parser/slimit1.sim sleep 2000 -run general/parser/slimit1_query.sim +run general/parser/slimit_alter_tags.sim sleep 2000 -# run general/parser/slimit_alter_tags.sim -# sleep 2000 -# run general/parser/stream_on_sys.sim -# sleep 2000 -# run general/parser/stream.sim -# sleep 2000 -# run general/parser/tbnameIn.sim -# sleep 2000 -# run general/parser/where.sim -# sleep 2000 -# #run general/parser/repeatAlter.sim -# sleep 2000 -# #run general/parser/repeatStream.sim -# sleep 2000 -# run general/parser/join.sim -# run general/parser/join_multivnode.sim -# run general/parser/projection_limit_offset.sim -# sleep 2000 -# run general/parser/select_with_tags.sim -# run general/parser/groupby.sim +run general/parser/stream_on_sys.sim +sleep 2000 +run general/parser/stream.sim +sleep 2000 +run general/parser/tbnameIn.sim +sleep 2000 +run general/parser/where.sim +sleep 2000 +#run general/parser/repeatAlter.sim +sleep 2000 +#run general/parser/repeatStream.sim +sleep 2000 +run general/parser/join.sim +sleep 2000 +run general/parser/join_multivnode.sim +sleep 2000 +run general/parser/projection_limit_offset.sim +sleep 2000 +run general/parser/select_with_tags.sim +sleep 2000 +run general/parser/groupby.sim diff --git a/tests/script/general/table/fill.sim b/tests/script/general/table/fill.sim index 00048eb025..333573e577 100644 --- a/tests/script/general/table/fill.sim +++ b/tests/script/general/table/fill.sim @@ -42,7 +42,7 @@ sql select count(*), last(ts), min(k), max(k), avg(k) from db.mt where a=0 and t print =================== step2 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 10000 +sleep 5000 system sh/exec.sh -n dnode1 -s start sleep 3000 diff --git a/tests/script/general/tag/bool.sim b/tests/script/general/tag/bool.sim index 540eaf1238..349cb738bf 100644 --- a/tests/script/general/tag/bool.sim +++ b/tests/script/general/tag/bool.sim @@ -122,7 +122,8 @@ if $rows != 100 then return -1 endi sql select * from $mt where tgcol = 1 -if $rows != 100 then +if $rows != 100 then + print expect 100, actual:$rows return -1 endi sql select * from $mt where tgcol <> 1 diff --git a/tests/script/general/tag/create.sim b/tests/script/general/tag/create.sim index c0cdbe87dc..3e16810f91 100644 --- a/tests/script/general/tag/create.sim +++ b/tests/script/general/tag/create.sim @@ -134,7 +134,8 @@ if $data01 != 1 then return -1 endi sql select * from $mt where tgcol = 0 -if $rows != 0 then +if $rows != 0 then + print expect 0, actual: $rows return -1 endi @@ -184,7 +185,8 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool, tgcol2 bool) sql create table $tb using $mt tags( 1, 2 ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = 2 -if $rows != 1 then +if $rows != 1 then + print expect 1, actual: $rows return -1 endi if $data01 != 1 then diff --git a/tests/script/general/tag/filter.sim b/tests/script/general/tag/filter.sim index 4388f029f9..75a6ed00da 100644 --- a/tests/script/general/tag/filter.sim +++ b/tests/script/general/tag/filter.sim @@ -122,7 +122,8 @@ endi print =============== step14 sql select count(tbcol) as c from $mt where ts > 1000 group by tgcol print $data00 $data01 $data02 $data03 $data04 $data05 $data06 -if $data00 != 100 then +if $data00 != 100 then + print expect 100, actual $data00 return -1 endi diff --git a/tests/script/general/user/pass_alter.sim b/tests/script/general/user/pass_alter.sim index 034e704a87..857d658db1 100644 --- a/tests/script/general/user/pass_alter.sim +++ b/tests/script/general/user/pass_alter.sim @@ -19,28 +19,47 @@ endi print ============= step2 sql close +sleep 2500 +print user read login sql connect read -sleep 2000 sql alter user read pass 'taosdata' sql alter user write pass 'taosdata1' -x step2 return -1 step2: +sql_error create user read pass 'taosdata1' +sql_error create user write pass 'taosdata1' + +sql show users +if $rows != 5 then + return -1 +endi print ============= step3 sql close +sleep 2500 +print user write login sql connect write -sleep 2000 + +sql_error create user read pass 'taosdata1' +sql_error create user write pass 'taosdata1' sql alter user write pass 'taosdata' sql alter user read pass 'taosdata' -x step3 return -1 step3: +sql show users +if $rows != 5 then + return -1 +endi + print ============= step4 sql close +sleep 2500 +print root write login sql connect -sleep 2000 sql create user oroot pass 'taosdata' + sql show users if $rows != 6 then return -1 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 48617c149a..62bae1c2a5 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,9 +1,6 @@ cd ../../debug; cmake .. -#cd ../../debug; make clean cd ../../debug; make - cd ../../../debug; cmake .. -#cd ../../../debug; make clean cd ../../../debug; make #./test.sh -f general/alter/cached_schema_after_alter.sim @@ -15,12 +12,12 @@ cd ../../../debug; make #./test.sh -f general/alter/table.sim ./test.sh -f general/cache/new_metrics.sim -#./test.sh -f general/cache/restart_metrics.sim +./test.sh -f general/cache/restart_metrics.sim ./test.sh -f general/cache/restart_table.sim -#./test.sh -f general/column/commit.sim -#./test.sh -f general/column/metrics.sim -#./test.sh -f general/column/table.sim +#hongze ./test.sh -f general/column/commit.sim +#hongze ./test.sh -f general/column/metrics.sim +#hongze ./test.sh -f general/column/table.sim ./test.sh -f general/compress/commitlog.sim ./test.sh -f general/compress/compress.sim @@ -31,11 +28,11 @@ cd ../../../debug; make ./test.sh -f general/compute/bottom.sim ./test.sh -f general/compute/count.sim ./test.sh -f general/compute/diff.sim -#./test.sh -f general/compute/diff2.sim +# liao./test.sh -f general/compute/diff2.sim ./test.sh -f general/compute/first.sim -#./test.sh -f general/compute/interval.sim -#./test.sh -f general/compute/last.sim -#./test.sh -f general/compute/leastsquare.sim +# liao./test.sh -f general/compute/interval.sim +# liao./test.sh -f general/compute/last.sim +# liao./test.sh -f general/compute/leastsquare.sim ./test.sh -f general/compute/max.sim ./test.sh -f general/compute/min.sim ./test.sh -f general/compute/null.sim @@ -53,41 +50,42 @@ cd ../../../debug; make ./test.sh -f general/db/delete_reuse1.sim ./test.sh -f general/db/delete_reuse2.sim ./test.sh -f general/db/delete_reusevnode.sim -#./test.sh -f general/db/delete_reusevnode2.sim +./test.sh -f general/db/delete_reusevnode2.sim ./test.sh -f general/db/delete_writing1.sim ./test.sh -f general/db/delete_writing2.sim ./test.sh -f general/db/len.sim +#./test.sh -u -f general/db/vnodes.sim ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim -#./test.sh -f general/field/2.sim +./test.sh -f general/field/2.sim #./test.sh -f general/field/3.sim #./test.sh -f general/field/4.sim #./test.sh -f general/field/5.sim #./test.sh -f general/field/6.sim -##./test.sh -f general/field/bigint.sim -#./test.sh -f general/field/binary.sim -##./test.sh -f general/field/bool.sim -#./test.sh -f general/field/single.sim -##./test.sh -f general/field/smallint.sim -##./test.sh -f general/field/tinyint.sim +./test.sh -f general/field/bigint.sim +# liao./test.sh -f general/field/binary.sim +./test.sh -f general/field/bool.sim +./test.sh -f general/field/single.sim +./test.sh -f general/field/smallint.sim +./test.sh -f general/field/tinyint.sim -##./test.sh -f general/http/restful.sim +# jeff ./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim -##./test.sh -f general/http/restful_full.sim +# jeff ./test.sh -f general/http/restful_full.sim ./test.sh -f general/http/prepare.sim ./test.sh -f general/http/telegraf.sim ./test.sh -f general/http/grafana_bug.sim -#./test.sh -f general/http/grafana.sim +# jeff ./test.sh -f general/http/grafana.sim ./test.sh -f general/import/basic.sim ./test.sh -f general/import/commit.sim ./test.sh -f general/import/large.sim -#./test.sh -f general/import/replica1.sim +#hongze ./test.sh -f general/import/replica1.sim ./test.sh -f general/insert/basic.sim -#./test.sh -f general/insert/insert_drop.sim +#hongze ./test.sh -f general/insert/insert_drop.sim ./test.sh -f general/insert/query_block1_memory.sim ./test.sh -f general/insert/query_block2_memory.sim ./test.sh -f general/insert/query_block1_file.sim @@ -96,8 +94,6 @@ cd ../../../debug; make ./test.sh -f general/insert/query_multi_file.sim ./test.sh -f general/insert/tcp.sim -#parser - # ./test.sh -f general/parser/alter.sim # ./test.sh -f general/parser/alter1.sim # ./test.sh -f general/parser/alter_stable.sim @@ -112,8 +108,8 @@ cd ../../../debug; make ./test.sh -f general/parser/columnValue_float.sim ./test.sh -f general/parser/columnValue_int.sim # ./test.sh -f general/parser/col_arithmetic_operation.sim -# ./test.sh -f general/parser/columnValue.sim -# ./test.sh -f general/parser/commit.sim +./test.sh -f general/parser/columnValue.sim +./test.sh -f general/parser/commit.sim # ./test.sh -f general/parser/create_db.sim # ./test.sh -f general/parser/create_mt.sim # ./test.sh -f general/parser/create_tb.sim @@ -121,9 +117,9 @@ cd ../../../debug; make # ./test.sh -f general/parser/fill.sim # ./test.sh -f general/parser/fill_stb.sim # ./test.sh -f general/parser/first_last.sim -# ./test.sh -f general/parser/import_commit1.sim -# ./test.sh -f general/parser/import_commit2.sim -# ./test.sh -f general/parser/import_commit3.sim +./test.sh -f general/parser/import_commit1.sim +./test.sh -f general/parser/import_commit2.sim +./test.sh -f general/parser/import_commit3.sim # ./test.sh -f general/parser/import_file.sim # ./test.sh -f general/parser/insert_tb.sim # ./test.sh -f general/parser/tags_dynamically_specifiy.sim @@ -160,14 +156,12 @@ cd ../../../debug; make #./test.sh -f general/stable/disk.sim #./test.sh -f general/stable/metrics.sim #./test.sh -f general/stable/values.sim -#./test.sh -f general/stable/vnode3.sim - -#stream +./test.sh -f general/stable/vnode3.sim ./test.sh -f general/table/autocreate.sim ./test.sh -f general/table/basic1.sim ./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim +#hongze ./test.sh -f general/table/basic3.sim ./test.sh -f general/table/bigint.sim ./test.sh -f general/table/binary.sim ./test.sh -f general/table/bool.sim @@ -177,17 +171,17 @@ cd ../../../debug; make ./test.sh -f general/table/column2.sim ./test.sh -f general/table/date.sim ./test.sh -f general/table/db.table.sim -#./test.sh -f general/table/delete_reuse1.sim -#./test.sh -f general/table/delete_reuse2.sim -#./test.sh -f general/table/delete_writing.sim -#./test.sh -f general/table/describe.sim +./test.sh -f general/table/delete_reuse1.sim +./test.sh -f general/table/delete_reuse2.sim +#hongze ./test.sh -f general/table/delete_writing.sim +./test.sh -f general/table/describe.sim ./test.sh -f general/table/double.sim -#./test.sh -f general/table/fill.sim +./test.sh -f general/table/fill.sim ./test.sh -f general/table/float.sim ./test.sh -f general/table/int.sim ./test.sh -f general/table/limit.sim ./test.sh -f general/table/smallint.sim -#./test.sh -f general/table/table_len.sim +./test.sh -f general/table/table_len.sim ./test.sh -f general/table/table.sim ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim @@ -197,63 +191,61 @@ cd ../../../debug; make #./test.sh -f general/tag/5.sim #./test.sh -f general/tag/6.sim #./test.sh -f general/tag/add.sim -#./test.sh -f general/tag/bigint.sim +./test.sh -f general/tag/bigint.sim #./test.sh -f general/tag/binary_binary.sim #./test.sh -f general/tag/binary.sim #./test.sh -f general/tag/bool_binary.sim #./test.sh -f general/tag/bool_int.sim -#./test.sh -f general/tag/bool.sim +./test.sh -f general/tag/bool.sim #./test.sh -f general/tag/change.sim -#./test.sh -f general/tag/column.sim +#liao ./test.sh -f general/tag/column.sim #./test.sh -f general/tag/commit.sim -#./test.sh -f general/tag/create.sim +#liao ./test.sh -f general/tag/create.sim #./test.sh -f general/tag/delete.sim #./test.sh -f general/tag/double.sim -#./test.sh -f general/tag/filter.sim +./test.sh -f general/tag/filter.sim #./test.sh -f general/tag/float.sim #./test.sh -f general/tag/int_binary.sim #./test.sh -f general/tag/int_float.sim -#./test.sh -f general/tag/int.sim +./test.sh -f general/tag/int.sim #./test.sh -f general/tag/set.sim -#./test.sh -f general/tag/smallint.sim -#./test.sh -f general/tag/tinyint.sim +./test.sh -f general/tag/smallint.sim +./test.sh -f general/tag/tinyint.sim ./test.sh -f general/user/basic1.sim -#./test.sh -f general/user/monitor.sim -##./test.sh -f general/user/pass_alter.sim +./test.sh -f general/user/monitor.sim +./test.sh -f general/user/pass_alter.sim ./test.sh -f general/user/pass_len.sim -##./test.sh -f general/user/user_create.sim +./test.sh -f general/user/user_create.sim ./test.sh -f general/user/user_len.sim -#./test.sh -f general/vector/metrics_field.sim -#./test.sh -f general/vector/metrics_mix.sim -#./test.sh -f general/vector/metrics_query.sim -#./test.sh -f general/vector/metrics_tag.sim -#./test.sh -f general/vector/metrics_time.sim -#./test.sh -f general/vector/multi.sim +./test.sh -f general/vector/metrics_field.sim +./test.sh -f general/vector/metrics_mix.sim +./test.sh -f general/vector/metrics_query.sim +./test.sh -f general/vector/metrics_tag.sim +./test.sh -f general/vector/metrics_time.sim +#liao ./test.sh -f general/vector/multi.sim ./test.sh -f general/vector/single.sim -#./test.sh -f general/vector/table_field.sim -#./test.sh -f general/vector/table_mix.sim -#./test.sh -f general/vector/table_query.sim -#./test.sh -f general/vector/table_time.sim - -################################# +./test.sh -f general/vector/table_field.sim +./test.sh -f general/vector/table_mix.sim +./test.sh -f general/vector/table_query.sim +./test.sh -f general/vector/table_time.sim ./test.sh -u -f unique/account/account_create.sim -##./test.sh -u -f unique/account/account_delete.sim +./test.sh -u -f unique/account/account_delete.sim ./test.sh -u -f unique/account/account_len.sim -##./test.sh -u -f unique/account/authority.sim +./test.sh -u -f unique/account/authority.sim ./test.sh -u -f unique/account/basic.sim ./test.sh -u -f unique/account/paras.sim -##./test.sh -u -f unique/account/pass_alter.sim +./test.sh -u -f unique/account/pass_alter.sim ./test.sh -u -f unique/account/pass_len.sim ./test.sh -u -f unique/account/usage.sim -##./test.sh -u -f unique/account/user_create.sim +./test.sh -u -f unique/account/user_create.sim ./test.sh -u -f unique/account/user_len.sim #./test.sh -u -f unique/big/balance.sim -#./test.sh -u -f unique/big/maxvnodes.sim -#./test.sh -u -f unique/big/tcp.sim +#slguan ./test.sh -u -f unique/big/maxvnodes.sim +./test.sh -u -f unique/big/tcp.sim ##./test.sh -u -f unique/cluster/balance1.sim ##./test.sh -u -f unique/cluster/balance2.sim @@ -298,27 +290,22 @@ cd ../../../debug; make #./test.sh -u -f unique/metrics/replica3_dnode6.sim #./test.sh -u -f unique/metrics/replica3_vnode3.sim -##./test.sh -u -f unique/mnode/mgmt22.sim -##./test.sh -u -f unique/mnode/mgmt23.sim -##./test.sh -u -f unique/mnode/mgmt24.sim -##./test.sh -u -f unique/mnode/mgmt25.sim -##./test.sh -u -f unique/mnode/mgmt26.sim -##./test.sh -u -f unique/mnode/mgmt33.sim -##./test.sh -u -f unique/mnode/mgmt34.sim +./test.sh -u -f unique/mnode/mgmt22.sim +./test.sh -u -f unique/mnode/mgmt23.sim +./test.sh -u -f unique/mnode/mgmt24.sim +./test.sh -u -f unique/mnode/mgmt25.sim +./test.sh -u -f unique/mnode/mgmt26.sim +./test.sh -u -f unique/mnode/mgmt33.sim +./test.sh -u -f unique/mnode/mgmt34.sim #./test.sh -u -f unique/mnode/mgmtr2.sim #./test.sh -u -f unique/mnode/secondIp.sim -#stream - ##./test.sh -u -f unique/table/delete_part.sim -##./test.sh -u -f unique/vnode/replica2_basic2.sim -##./test.sh -u -f unique/vnode/replica3_basic.sim - #./test.sh -u -f unique/vnode/commit.sim #./test.sh -u -f unique/vnode/many.sim #./test.sh -u -f unique/vnode/replica2_basic.sim -##./test.sh -u -f unique/vnode/replica2_basic2.sim +./test.sh -u -f unique/vnode/replica2_basic2.sim #./test.sh -u -f unique/vnode/replica2_repeat.sim ##./test.sh -u -f unique/vnode/replica3_basic.sim #./test.sh -u -f unique/vnode/replica3_repeat.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 5587e3bb8c..bc18479896 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -99,7 +99,7 @@ echo "logDir $LOG_DIR" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 151" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG diff --git a/tests/script/test.sh b/tests/script/test.sh index 743597eabd..8e3959a680 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -99,7 +99,8 @@ echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG echo "monitorDebugFlag 135" >> $TAOS_CFG echo "udebugFlag 135" >> $TAOS_CFG -echo "clog 0" >> $TAOS_CFG +echo "tablemetakeeptimer 5" >> $TAOS_CFG +echo "wal 0" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG echo " " >> $TAOS_CFG diff --git a/tests/script/unique/account/account_create.sim b/tests/script/unique/account/account_create.sim index d1d5ebece0..1966e1a7ce 100644 --- a/tests/script/unique/account/account_create.sim +++ b/tests/script/unique/account/account_create.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect print ============================ dnode1 start diff --git a/tests/script/unique/account/account_delete.sim b/tests/script/unique/account/account_delete.sim index 71a75a7b35..40075dc1a6 100644 --- a/tests/script/unique/account/account_delete.sim +++ b/tests/script/unique/account/account_delete.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect print ============= step1 sql create account oroot pass 'taosdata' diff --git a/tests/script/unique/account/account_len.sim b/tests/script/unique/account/account_len.sim index ae2db26d58..81d0f0bfb1 100644 --- a/tests/script/unique/account/account_len.sim +++ b/tests/script/unique/account/account_len.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect $i = 0 $dbPrefix = aldb diff --git a/tests/script/unique/account/authority.sim b/tests/script/unique/account/authority.sim index fe978a5827..2bb61cb0f2 100644 --- a/tests/script/unique/account/authority.sim +++ b/tests/script/unique/account/authority.sim @@ -4,8 +4,8 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect print ============= step1 diff --git a/tests/script/unique/account/basic.sim b/tests/script/unique/account/basic.sim index d06f7a15d9..adbc84d766 100644 --- a/tests/script/unique/account/basic.sim +++ b/tests/script/unique/account/basic.sim @@ -1,6 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/paras.sim b/tests/script/unique/account/paras.sim index b2e540400c..70a423f70f 100644 --- a/tests/script/unique/account/paras.sim +++ b/tests/script/unique/account/paras.sim @@ -1,6 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/pass_alter.sim b/tests/script/unique/account/pass_alter.sim index 936611833a..644548be0b 100644 --- a/tests/script/unique/account/pass_alter.sim +++ b/tests/script/unique/account/pass_alter.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect print ============= step1 sql create user read pass 'taosdata1' diff --git a/tests/script/unique/account/pass_len.sim b/tests/script/unique/account/pass_len.sim index f4f9c04964..370ea94200 100644 --- a/tests/script/unique/account/pass_len.sim +++ b/tests/script/unique/account/pass_len.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect $i = 0 $dbPrefix = apdb diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index f022380ac4..63c3d49f43 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -1,6 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/user_create.sim b/tests/script/unique/account/user_create.sim index 789035cfcc..2a64786225 100644 --- a/tests/script/unique/account/user_create.sim +++ b/tests/script/unique/account/user_create.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect print =============== step1 sql show users diff --git a/tests/script/unique/account/user_len.sim b/tests/script/unique/account/user_len.sim index 918551fa66..1163ac2e4b 100644 --- a/tests/script/unique/account/user_len.sim +++ b/tests/script/unique/account/user_len.sim @@ -3,8 +3,8 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect $i = 0 $dbPrefix = lm_us_db diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index 52199f046e..4b0a6cd0bf 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -20,7 +20,7 @@ system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 1000 print =============== prepare data -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sleep 3000 sql connect @@ -87,7 +87,7 @@ endi print ========== step1 sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start $x = 0 show1: @@ -98,12 +98,12 @@ show1: endi sql show dnodes -print dnode1 freeVnodes $data3_192.168.0.1 -print dnode2 freeVnodes $data3_192.168.0.2 -if $data3_192.168.0.1 != 2 then +print dnode1 openvnodes $data2_1 +print dnode2 openvnodes $data2_2 +if $data2_1 != 2 then goto show1 endi -if $data3_192.168.0.2 != 2 then +if $data2_2 != 2 then goto show1 endi @@ -139,7 +139,7 @@ endi print ========== step2 sql create dnode $hostname3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 10000 print ========== step3 @@ -154,20 +154,20 @@ show3: endi sql show dnodes -print dnode1 freeVnodes $data3_192.168.0.1 -print dnode2 freeVnodes $data3_192.168.0.2 -print dnode3 freeVnodes $data3_192.168.0.3 -if $data3_192.168.0.1 != 2 then +print dnode1 freeVnodes $data2_1 +print dnode2 freeVnodes $data2_2 +print dnode3 freeVnodes $data2_3 +if $data2_1 != 2 then goto show3 endi -if $data3_192.168.0.2 != NULL then +if $data2_2 != NULL then goto show3 endi -if $data3_192.168.0.3 != 2 then +if $data2_3 != 2 then goto show3 endi -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -211,16 +211,16 @@ show4: endi sql show dnodes -print dnode1 freeVnodes $data3_192.168.0.1 -print dnode3 freeVnodes $data3_192.168.0.3 -if $data3_192.168.0.1 != 0 then +print dnode1 freeVnodes $data2_1 +print dnode3 freeVnodes $data2_3 +if $data2_1 != 0 then goto show4 endi -if $data3_192.168.0.3 != NULL then +if $data2_3 != NULL then goto show4 endi -system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -255,7 +255,7 @@ endi print ========== step5 sql alter database db replica 2 sql create dnode $hostname4 -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode4 -s start $x = 0 show5: @@ -266,12 +266,12 @@ show5: endi sql show dnodes -print dnode1 freeVnodes $data3_192.168.0.1 -print dnode4 freeVnodes $data3_192.168.0.4 -if $data3_192.168.0.1 != 0 then +print dnode1 freeVnodes $data2_1 +print dnode4 freeVnodes $data2_4 +if $data2_1 != 0 then goto show5 endi -if $data3_192.168.0.4 != 0 then +if $data2_4 != 0 then goto show5 endi diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index 00995ba32c..3015d07b6d 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -1,25 +1,32 @@ system sh/stop_dnodes.sh +$totalVnodes = 100 +$minVnodes = 48 +$maxVnodes = 52 +$maxTables = 4 +$totalRows = $totalVnodes * $maxTables + system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 100 -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v $maxTables +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v $totalVnodes system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 100000 system sh/cfg.sh -n dnode1 -c maxShellConns -v 100000 system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 100000 print ========== prepare data -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect -sql create database db +sql create database db blocks 2 cache 1 maxTables $maxTables sql use db print ========== step1 sql create table mt (ts timestamp, tbcol int) TAGS(tgcol int) $x = 0 -while $x < 25600 +while $x < $totalRows $tb = t . $x sql create table $tb using mt tags( $x ) sql insert into $tb values (now, $x ) @@ -28,16 +35,15 @@ endw print ========== step2 sql select * from mt -if $rows != 25600 then +if $rows != $totalRows then return -1 endi sql select count(*) from mt -if $data00 != 25600 then +if $data00 != $totalRows then return -1 endi - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c walLevel -v 0 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 100 @@ -49,7 +55,7 @@ system sh/cfg.sh -n dnode2 -c maxMgmtConnections -v 100000 print ========== step2 sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start $x = 0 show3: @@ -60,23 +66,23 @@ show3: endi sql show dnodes -print dnode1 freeVnodes $data3_192.168.0.1 -print dnode2 freeVnodes $data3_192.168.0.2 -if $data3_192.168.0.1 != 126 then +print dnode1 openvnodes $data2_1 +print dnode2 openvnodes $data2_2 +if $data2_1 != $minVnodes then goto show3 endi -if $data3_192.168.0.2 != 130 then +if $data2_2 != $maxVnodes then goto show3 endi print ========== step4 sql select * from mt -if $rows != 25600 then +if $rows != $totalRows then return -1 endi sql select count(*) from mt -if $data00 != 25600 then +if $data00 != $totalRows then return -1 endi diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 1fb414ec74..590035c0f8 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -14,8 +14,8 @@ system sh/cfg.sh -n dnode1 -c adminDebugFlag -v 131 system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c debugFlag -v 131 -system sh/exec.sh -n dnode1 -s start - +system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect print ======================== dnode1 start @@ -52,7 +52,7 @@ while $i < $tbNum endw $i = 0 -while $i < 100 +while $i < 10 print =============== step3 $i sql select count(*) from $mt print ===> $data00 $data01 @@ -83,7 +83,7 @@ while $i < $tbNum endw $i = 0 -while $i < 100 +while $i < 10 print =============== step5 $i sql select count(*) from $mt where tgcol < 20200 print ===> $data00 $data01 diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index 50692524cf..e4e1235e66 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -41,14 +41,21 @@ if $data2_2 != slave then goto show2 endi +system sh/exec_up.sh -n dnode1 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT +system sh/exec_up.sh -n dnode4 -s stop -x SIGINT +system sh/exec_up.sh -n dnode5 -s stop -x SIGINT +system sh/exec_up.sh -n dnode6 -s stop -x SIGINT +system sh/exec_up.sh -n dnode7 -s stop -x SIGINT +system sh/exec_up.sh -n dnode8 -s stop -x SIGINT + +return + print ============== step3 sql_error drop dnode $hostname1 -x error1 print should not drop master -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -return - print ============== step4 system sh/exec_up.sh -n dnode1 -s stop -x SIGINT sql_error show mnodes @@ -83,7 +90,7 @@ if $data2_2 != slave then endi print ============== step7 -system sh/exec_up.sh -n dnode3 -s start -t +system sh/exec_up.sh -n dnode3 -s start sql create dnode $hostname3 sleep 5000 @@ -105,7 +112,7 @@ endi if $data2_2 != slave then goto show7 endi -if $data3_3 != NULL then +if $data3_3 != null then goto show7 endi diff --git a/tests/script/unique/mnode/mgmt23.sim b/tests/script/unique/mnode/mgmt23.sim index 99d9159573..8c144dd0c4 100644 --- a/tests/script/unique/mnode/mgmt23.sim +++ b/tests/script/unique/mnode/mgmt23.sim @@ -59,7 +59,7 @@ endi if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi @@ -78,7 +78,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -106,7 +106,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then diff --git a/tests/script/unique/mnode/mgmt25.sim b/tests/script/unique/mnode/mgmt25.sim index ba545f9479..8c8eeba466 100644 --- a/tests/script/unique/mnode/mgmt25.sim +++ b/tests/script/unique/mnode/mgmt25.sim @@ -59,7 +59,7 @@ endi if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi @@ -78,7 +78,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then diff --git a/tests/script/unique/mnode/mgmt26.sim b/tests/script/unique/mnode/mgmt26.sim index e5ead5bb39..373a0abaed 100644 --- a/tests/script/unique/mnode/mgmt26.sim +++ b/tests/script/unique/mnode/mgmt26.sim @@ -59,7 +59,7 @@ endi if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi @@ -79,7 +79,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -106,7 +106,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 36b7ba8139..e3a62a2d22 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -18,10 +18,10 @@ print dnode3 ==> $data3_3 if $data2_1 != master then return -1 endi -if $data3_2 != NULL then +if $data3_2 != null then return -1 endi -if $data3_3 != NULL then +if $data3_3 != null then return -1 endi @@ -44,7 +44,7 @@ endi if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi @@ -86,7 +86,7 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -156,7 +156,7 @@ print dnode1 ==> $dnode1Role print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role -if $dnode1Role != NULL then +if $dnode1Role != null then return -1 endi #if $dnode2Role != master then diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index 1139daea1c..4409c825c8 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -22,10 +22,10 @@ print dnode3 ==> $data3_3 if $data2_1 != master then return -1 endi -if $data3_2 != NULL then +if $data3_2 != null then return -1 endi -if $data3_3 != NULL then +if $data3_3 != null then return -1 endi @@ -50,10 +50,10 @@ endi if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi -if $dnode4Role != NULL then +if $dnode4Role != null then return -1 endi @@ -81,7 +81,7 @@ endi if $dnode3Role != slave then return -1 endi -if $dnode4Role != NULL then +if $dnode4Role != null then return -1 endi @@ -110,7 +110,7 @@ endi if $dnode3Role != slave then return -1 endi -if $dnode4Role != NULL then +if $dnode4Role != null then return -1 endi @@ -131,7 +131,7 @@ print dnode4 ==> $dnode4Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -165,7 +165,7 @@ print dnode4 ==> $dnode4Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -207,7 +207,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role -if $dnode1Role != NULL then +if $dnode1Role != null then return -1 endi if $dnode2Role != slave then diff --git a/tests/script/unique/vnode/replica2_basic2.sim b/tests/script/unique/vnode/replica2_basic2.sim index a86071faaf..041db8249a 100644 --- a/tests/script/unique/vnode/replica2_basic2.sim +++ b/tests/script/unique/vnode/replica2_basic2.sim @@ -27,6 +27,7 @@ system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 print ========= start dnodes system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect sql create dnode $hostname2 system sh/exec_up.sh -n dnode2 -s start @@ -89,6 +90,19 @@ if $data2_3 != 4 then return -1 endi +if $data4_1 != ready then + print dnode1 status should ready but is $data4_1 + return -1 +endi + +if $data4_2 != ready then + return -1 +endi + +if $data4_3 != ready then + return -1 +endi + print ========= step2 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) diff --git a/tests/test-all.sh b/tests/test-all.sh index 93bfa7a426..e58a6f5132 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -21,11 +21,12 @@ fi echo -e "${GREEN} ### Total $totalSuccess TSIM case(s) succeed! ### ${NC}" totalFailed=`grep 'failed\|fault' out.txt | wc -l` -echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" +# echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" if [ "$totalFailed" -ne "0" ]; then -# echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" - exit $totalFailed + echo -e "${RED} ### Total $totalFailed TSIM case(s) failed! ### ${NC}" + +# exit $totalFailed fi echo "### run Python script ###" @@ -46,6 +47,7 @@ fi totalPyFailed=`grep 'failed\|fault' pytest-out.txt | wc -l` if [ "$totalPyFailed" -ne "0" ]; then echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" - exit $totalPyFailed +# exit $totalPyFailed fi +exit $(($totalFailed + $totalPyFailed)) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index d6ab2d5b92..5ea1cb4a1a 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -415,7 +415,7 @@ void simCloseNativeConnect(SScript *script) { simTrace("script:%s, taos:%p closed", script->fileName, script->taos); taos_close(script->taos); - taosMsleep(1000); + taosMsleep(1200); script->taos = NULL; }