diff --git a/.gitignore b/.gitignore index e91d6739a5..d0af51d31b 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ Target/ *.failed *.sql sim/ +*DS_Store diff --git a/.travis.yml b/.travis.yml index 39fddc20c9..7df3a7d7fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,10 +46,10 @@ matrix: pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ cd ${TRAVIS_BUILD_DIR}/tests - ./test-all.sh || travis_terminate $? + ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $? cd ${TRAVIS_BUILD_DIR}/tests/pytest - ./simpletest.sh -g 2>&1 | tee mem-error-out.txt + ./smoketest.sh -g 2>&1 | tee mem-error-out.txt sleep 1 # Color setting @@ -86,13 +86,12 @@ matrix: addons: coverity_scan: - # GitHub project metadata # ** specific to your project ** project: name: TDengine version: 2.x - description: taosdata/TDengine + description: TDengine # Where email notification of build analysis results will be sent notification_email: sdsang@taosdata.com diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index ecbef79ae4..ac1894369d 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -48,7 +48,7 @@ ELSEIF (TD_WINDOWS_64) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () - TARGET_LINK_LIBRARIES(taos trpc) + TARGET_LINK_LIBRARIES(taos trpc tutil) ELSEIF (TD_DARWIN_64) SET(CMAKE_MACOSX_RPATH 1) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2181b718e2..e6d15c8736 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -25,6 +25,7 @@ extern "C" { */ #include "os.h" #include "tbuffer.h" +#include "exception.h" #include "qextbuffer.h" #include "taosdef.h" #include "tscSecondaryMerge.h" @@ -177,7 +178,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf); +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw); void tscTagCondCopy(STagCond* dest, const STagCond* src); void tscTagCondRelease(STagCond* pCond); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 50acbb1a49..b349a4e2dd 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -45,8 +45,6 @@ enum { DATA_FROM_DATA_FILE = 2, }; -typedef SCMSTableVgroupRspMsg SVgroupsInfo; - typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 32fde5f7fa..83a30b512d 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -47,7 +47,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e7e37d880d..41a5f5177d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -385,7 +385,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pMsg += sizeof(SMgmtHead); SCMCfgDnodeMsg* pCfg = (SCMCfgDnodeMsg*)pMsg; - strncpy(pCfg->ip, pDCL->a[0].z, pDCL->a[0].n); + strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n); @@ -1185,10 +1185,18 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel return invalidSqlErrMsg(pQueryInfo->msg, "invalid arithmetic expression in select clause"); } - SBuffer buf = exprTreeToBinary(pNode); + SBufferWriter bw = tbufInitWriter(NULL, false); + + TRY(0) { + exprTreeToBinary(&bw, pNode); + } CATCH(code) { + tbufCloseWriter(&bw); + UNUSED(code); + // TODO: other error handling + } END_TRY - size_t len = tbufTell(&buf); - char* c = tbufGetData(&buf, true); + size_t len = tbufTell(&bw); + char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); @@ -3294,7 +3302,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S // set join query condition if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); pCondExpr->tsJoin = true; /* @@ -3714,14 +3722,14 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - -// int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); tscColumnListInsert(pTableMetaInfo->tagColList, &index); getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - -// columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); tscColumnListInsert(pTableMetaInfo->tagColList, &index); } } @@ -3739,7 +3747,15 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, SArray* colList = taosArrayInit(10, sizeof(SColIndex)); ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, colList); - SBuffer buf = exprTreeToBinary(p); + SBufferWriter bw = tbufInitWriter(NULL, false); + + TRY(0) { + exprTreeToBinary(&bw, p); + } CATCH(code) { + tbufCloseWriter(&bw); + UNUSED(code); + // TODO: more error handling + } END_TRY // add to source column list STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); @@ -3753,7 +3769,7 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tscColumnListInsert(pTableMetaInfo->tagColList, &index); } - tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf); + tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -4654,17 +4670,17 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* return TSDB_CODE_SUCCESS; } + // todo refactor if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - bool queryOnTags = tscQueryTags(pQueryInfo); - - if (queryOnTags != true) { // local handle the super table tag query + if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } } @@ -4785,16 +4801,15 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea } static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { - pMsg->blocksPerTable = htons(pCreateDb->numOfBlocksPerTable); - pMsg->compression = pCreateDb->compressionLevel; - - pMsg->commitLog = (char)pCreateDb->commitLog; - pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); - pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock); + pMsg->cacheBlockSize = htonl(-1); + pMsg->totalBlocks = htonl(-1); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl(pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(-1); + pMsg->maxRowsPerFileBlock = htonl(-1); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->replications = pCreateDb->replica; pMsg->ignoreExist = pCreateDb->ignoreExists; } @@ -5327,29 +5342,22 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { char msg[512] = {0}; - if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 2)) { + if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->replications != -1 && - (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { + (pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->rowsInFileBlock); - if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { - snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5361,9 +5369,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { } val = htonl(pCreate->maxSessions); - if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { + if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + TSDB_MIN_TABLES, TSDB_MAX_TABLES); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5373,24 +5381,17 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", - pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0e5907f992..4aee787271 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -49,11 +49,11 @@ static void tscSetDnodeIpList(SSqlObj* pSql, STableMeta* pTableMeta) { SRpcIpSet* pIpList = &pSql->ipList; pIpList->numOfIps = pTableMeta->vgroupInfo.numOfIps; - pIpList->port = tsDnodeShellPort; pIpList->inUse = 0; for(int32_t i = 0; i < pTableMeta->vgroupInfo.numOfIps; ++i) { - pIpList->ip[i] = pTableMeta->vgroupInfo.ipAddr[i].ip; + strcpy(pIpList->fqdn[i], pTableMeta->vgroupInfo.ipAddr[i].fqdn); + pIpList->port[i] = pTableMeta->vgroupInfo.ipAddr[i].port; } } @@ -62,7 +62,7 @@ void tscPrintMgmtIp() { tscError("invalid mgmt IP list:%d", tscMgmtIpSet.numOfIps); } else { for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscTrace("mgmt index:%d ip:%d", i, tscMgmtIpSet.ip[i]); + tscTrace("mgmt index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); } } } @@ -70,9 +70,8 @@ void tscPrintMgmtIp() { void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) { tscMgmtIpSet.numOfIps = pIpList->numOfIps; tscMgmtIpSet.inUse = pIpList->inUse; - tscMgmtIpSet.port = htons(pIpList->port); for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscMgmtIpSet.ip[i] = htonl(pIpList->ip[i]); + tscMgmtIpSet.port[i] = htons(pIpList->port[i]); } } @@ -80,8 +79,7 @@ void tscSetMgmtIpListFromEdge() { if (tscMgmtIpSet.numOfIps != 1) { tscMgmtIpSet.numOfIps = 1; tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.port = tsMnodeShellPort; - tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp); + taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); tscTrace("edge mgmt IP list:"); tscPrintMgmtIp(); } @@ -213,9 +211,6 @@ int tscSendMsgToServer(SSqlObj *pSql) { rpcSendRequest(pVnodeConn, &pSql->ipList, &rpcMsg); } else { pSql->ipList = tscMgmtIpSet; - pSql->ipList.port = tsMnodeShellPort; - - tscTrace("%p msg:%s is sent to server %d", pSql, taosMsg[pSql->cmd.msgType], pSql->ipList.port); memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); SRpcMsg rpcMsg = { .msgType = pSql->cmd.msgType, @@ -224,6 +219,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; + tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]); rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg); } @@ -294,11 +290,6 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { } } } - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry); - pSql->retry = 0; - } pRes->rspLen = 0; @@ -308,6 +299,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { tscTrace("%p query is cancelled, code:%d", pSql, tstrerror(pRes->code)); } + if (pRes->code == TSDB_CODE_SUCCESS) { + tscTrace("%p reset retry counter to be 0 due to success rsp, old:%d", pSql, pSql->retry); + pSql->retry = 0; + } + if (pRes->code != TSDB_CODE_QUERY_CANCELLED) { assert(rpcMsg->msgType == pCmd->msgType + 1); pRes->code = rpcMsg->code; @@ -565,7 +561,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; tscSetDnodeIpList(pSql, pTableMeta); - tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes)); + tscTrace("%p build submit msg, vgId:%d numOfVgroup:%d numberOfIP:%d", pSql, vgId, htonl(pMsgDesc->numOfVnodes), pSql->ipList.numOfIps); return TSDB_CODE_SUCCESS; } @@ -660,11 +656,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMVgroupInfo* pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; pSql->ipList.numOfIps = pVgroupInfo->numOfIps; // todo fix me - pSql->ipList.port = tsDnodeShellPort; pSql->ipList.inUse = 0; for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip; + strcpy(pSql->ipList.fqdn[i], pVgroupInfo->ipAddr[i].fqdn); + pSql->ipList.port[i] = pVgroupInfo->ipAddr[i].port; } tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index); @@ -931,7 +927,8 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCMCreateDnodeMsg *pCreate = (SCMCreateDnodeMsg *)pCmd->payload; - strncpy(pCreate->ip, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE; return TSDB_CODE_SUCCESS; @@ -1074,7 +1071,7 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMDropDnodeMsg *pDrop = (SCMDropDnodeMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - strcpy(pDrop->ip, pTableMetaInfo->name); + strcpy(pDrop->ep, pTableMetaInfo->name); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; return TSDB_CODE_SUCCESS; @@ -1319,11 +1316,11 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); pAlterTableMsg->type = htons(pAlterInfo->type); - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo); memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN); SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; @@ -1348,11 +1345,6 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCMAlterDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); strcpy(pAlterDbMsg->db, pTableMetaInfo->name); @@ -1585,32 +1577,32 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return pCmd->payloadLen; } -static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { - const int32_t defaultSize = - minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - int32_t n = 0; - size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); - for (int32_t i = 0; i < size; ++i) { - assert(0); -// n += strlen(pQueryInfo->tagCond.cond[i].cond); - } - - int32_t tagLen = n * TSDB_NCHAR_SIZE; - if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { - tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; - } - - int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; - int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; - - int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); - - int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; - - return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); -} +//static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { +//// const int32_t defaultSize = +//// minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; +//// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); +//// +//// int32_t n = 0; +//// size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); +//// for (int32_t i = 0; i < size; ++i) { +//// assert(0); +////// n += strlen(pQueryInfo->tagCond.cond[i].cond); +//// } +//// +//// int32_t tagLen = n * TSDB_NCHAR_SIZE; +//// if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { +//// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; +//// } +//// +//// int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; +//// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; +//// +//// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); +//// +//// int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; +//// +//// return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); +//} int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { @@ -1758,14 +1750,22 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { #endif SSqlCmd *pCmd = &pSql->cmd; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pCmd->payload; - strncpy(pStableVgroupMsg->tableId, pTableMetaInfo->name, tListLen(pStableVgroupMsg->tableId)); + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + + SCMSTableVgroupMsg *pStableVgroupMsg = (SCMSTableVgroupMsg *) pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SCMSTableVgroupMsg); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + strncpy(pMsg, pTableMetaInfo->name, TSDB_TABLE_ID_LEN); + pMsg += TSDB_TABLE_ID_LEN; + } pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; - pCmd->payloadLen = sizeof(SCMSTableVgroupMsg); + pCmd->payloadLen = (pMsg - pCmd->payload); return TSDB_CODE_SUCCESS; } @@ -1856,10 +1856,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) { - pMetaMsg->vgroup.ipAddr[i].ip = htonl(pMetaMsg->vgroup.ipAddr[i].ip); pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port); - - assert(pMetaMsg->vgroup.ipAddr[i].ip != 0); } SSchema* pSchema = pMetaMsg->schema; @@ -2123,28 +2120,37 @@ _error_clean: #endif SSqlRes* pRes = &pSql->res; + // NOTE: the order of several table must be preserved. SCMSTableVgroupRspMsg *pStableVgroup = (SCMSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfVgroups = htonl(pStableVgroup->numOfVgroups); + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char* pMsg = pRes->pRsp + sizeof(SCMSTableVgroupRspMsg); // master sqlObj locates in param SSqlObj* parent = pSql->param; assert(parent != NULL); SSqlCmd* pCmd = &parent->cmd; - STableMetaInfo* pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - - pInfo->vgroupList = malloc(pRes->rspLen); - memcpy(pInfo->vgroupList, pStableVgroup, pRes->rspLen); - - for(int32_t i = 0; i < pInfo->vgroupList->numOfVgroups; ++i) { - SCMVgroupInfo* pVgroups = &pInfo->vgroupList->vgroups[i]; - - pVgroups->vgId = htonl(pVgroups->vgId); - assert(pVgroups->numOfIps >= 1); - - for(int32_t j = 0; j < pVgroups->numOfIps; ++j) { - pVgroups->ipAddr[j].ip = htonl(pVgroups->ipAddr[j].ip); - pVgroups->ipAddr[j].port = htons(pVgroups->ipAddr[j].port); + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); + SVgroupsInfo * pVgroupInfo = (SVgroupsInfo *)pMsg; + pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups); + + size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo); + pInfo->vgroupList = calloc(1, size); + assert(pInfo->vgroupList != NULL); + + memcpy(pInfo->vgroupList, pVgroupInfo, size); + for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { + SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + + pVgroups->vgId = htonl(pVgroups->vgId); + assert(pVgroups->numOfIps >= 1); + + for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { + pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); + } + + pMsg += size; } } @@ -2277,7 +2283,6 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { if (pTableMetaInfo->pTableMeta) { taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); -// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); } return 0; @@ -2503,42 +2508,27 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { return code; } +static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->vgroupList == NULL) { + return false; + } + } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} + int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { int code = TSDB_CODE_NETWORK_UNAVAIL; SSqlCmd *pCmd = &pSql->cmd; - - //the query condition is serialized into pCmd->payload, we need to rebuild key for stable meta info in cache. -// bool required = false; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - if (pQueryInfo->pTableMetaInfo[0]->vgroupList != NULL) { + + if (allVgroupInfoRetrieved(pCmd, clauseIndex)) { return TSDB_CODE_SUCCESS; } - -#if 0 - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - char tagstr[TSDB_MAX_TAGS_LEN + 1] = {0}; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - tscGetMetricMetaCacheKey(pQueryInfo, tagstr, pTableMetaInfo->pTableMeta->uid); - -// taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), false); - - SSuperTableMeta *ppMeta = (SSuperTableMeta *)taosCacheAcquireByName(tscCacheHandle, tagstr); - if (ppMeta == NULL) { - required = true; - break; - } else { -// pTableMetaInfo->pMetricMeta = ppMeta; - } - } - - // all metricmeta for one clause are retrieved from cache, no need to retrieve metricmeta from management node - if (!required) { - return TSDB_CODE_SUCCESS; - } -#endif - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; @@ -2550,11 +2540,11 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { return code; } + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); - - STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->tagColList); + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -2562,25 +2552,9 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { return code; } - tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); - - pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + tscTrace("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables); - pNewQueryInfo->slimit = pQueryInfo->slimit; - pNewQueryInfo->order = pQueryInfo->order; - - STagCond* pTagCond = &pNewQueryInfo->tagCond; - tscTrace("%p new sqlobj:%p info, numOfTables:%d, slimit:%" PRId64 ", soffset:%" PRId64 ", order:%d, tbname cond:%s", - pSql, pNew, pNewQueryInfo->numOfTables, pNewQueryInfo->slimit.limit, pNewQueryInfo->slimit.offset, - pNewQueryInfo->order.order, pTagCond->tbnameCond.cond) - -// if (pSql->fp != NULL && pSql->pStream == NULL) { -// pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); -// tscFreeQueryInfo(pCmd); -// } - - tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; code = tscProcessSql(pNew); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 3b75739e5e..f73052ec38 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -72,24 +72,24 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } + tscMgmtIpSet.numOfIps = 0; + if (ip && ip[0]) { tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.port = tsMnodeShellPort; tscMgmtIpSet.numOfIps = 1; - tscMgmtIpSet.ip[0] = inet_addr(ip); - - if (tsMasterIp[0] && strcmp(ip, tsMasterIp) != 0) { - tscMgmtIpSet.numOfIps = 2; - tscMgmtIpSet.ip[1] = inet_addr(tsMasterIp); + strcpy(tscMgmtIpSet.fqdn[0], ip); + tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort; + } else { + if (tsMaster[0] != 0) { + taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; } - if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) { - tscMgmtIpSet.numOfIps = 3; - tscMgmtIpSet.ip[2] = inet_addr(tsSecondIp); + if (tsSecond[0] != 0) { + taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; } } - - tscMgmtIpSet.port = port ? port : tsMnodeShellPort; STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { @@ -131,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; tsem_init(&pSql->rspSem, 0, 0); @@ -167,10 +167,6 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - if (ip == NULL || (ip != NULL && (strcmp("127.0.0.1", ip) == 0 || strcasecmp("localhost", ip) == 0))) { - ip = tsMasterIp; - } - tscTrace("try to create a connection to %s", ip); STscObj *pObj = taosConnectImpl(ip, user, pass, db, port, NULL, NULL, NULL); @@ -716,7 +712,7 @@ char *taos_errstr(TAOS *taos) { void taos_config(int debug, char *log_path) { uDebugFlag = debug; - strcpy(logDir, log_path); + strcpy(tsLogDir, log_path); } char *taos_get_server_info(TAOS *taos) { diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index d938bb0576..7b40d69f1b 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -220,7 +220,7 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { static int tscLoadSubscriptionProgress(SSub* pSub) { char buf[TSDB_MAX_SQL_LEN]; - sprintf(buf, "%s/subscribe/%s", dataDir, pSub->topic); + sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); FILE* fp = fopen(buf, "r"); if (fp == NULL) { @@ -281,12 +281,12 @@ void tscSaveSubscriptionProgress(void* sub) { SSub* pSub = (SSub*)sub; char path[256]; - sprintf(path, "%s/subscribe", dataDir); + sprintf(path, "%s/subscribe", tsDataDir); if (access(path, 0) != 0) { mkdir(path, 0777); } - sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic); + sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); FILE* fp = fopen(path, "w+"); if (fp == NULL) { tscError("failed to create progress file for subscription: %s", pSub->topic); @@ -416,7 +416,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { tscSaveSubscriptionProgress(pSub); } else { char path[256]; - sprintf(path, "%s/subscribe/%s", dataDir, pSub->topic); + sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); remove(path); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d8a2fa366d..8475b02aba 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -928,7 +928,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); pState->numOfTotal = pQueryInfo->numOfTables; - tscTrace("%p start launched subquery, total:%d", pSql, pQueryInfo->numOfTables); + tscTrace("%p start launch subquery, total:%d", pSql, pQueryInfo->numOfTables); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { SJoinSubquerySupporter *pSupporter = tscCreateJoinSupporter(pSql, pState, i); @@ -1231,7 +1231,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // data in from current vnode is stored in cache and disk uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->numOfElems; tscTrace("%p sub:%p all data retrieved from ip:%u,vgId:%d, numOfRows:%d, orderOfSub:%d", pPObj, pSql, - pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].ip, pTableMetaInfo->vgroupList->vgroups[0].vgId, + pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, numOfRowsFromSubquery, idx); tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); @@ -1459,12 +1459,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { if (pState->code != TSDB_CODE_SUCCESS) { // at least one peer subquery failed, abort current query tscTrace("%p sub:%p query failed,ip:%u,vgId:%d,orderOfSub:%d,global code:%d", pParentSql, pSql, - pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex, pState->code); + pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex, pState->code); tscHandleSubqueryError(param, tres, pState->code); } else { // success, proceed to retrieve data from dnode tscTrace("%p sub:%p query complete, ip:%u, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSqlObj, pSql, - pVgroup->ipAddr[0].ip, pVgroup->vgId, trsupport->subqueryIndex); + pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode tscRetrieveFromDnodeCallBack(param, pSql, 0); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 75ab9fc720..897c63f011 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -55,7 +55,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) { if (pVnodeConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = tsLocalIp; rpcInit.localPort = 0; rpcInit.label = "TSC-vnode"; rpcInit.numOfThreads = tscNumOfThreads; @@ -76,7 +75,6 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) { if (*pMgmtConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = tsLocalIp; rpcInit.localPort = 0; rpcInit.label = "TSC-mgmt"; rpcInit.numOfThreads = 1; @@ -109,23 +107,17 @@ void taos_init_imp() { deltaToUtcInitOnce(); if (tscEmbedded == 0) { - /* - * set localIp = 0 - * means unset tsLocalIp in client - * except read from config file - */ - strcpy(tsLocalIp, "0.0.0.0"); // Read global configuration. taosInitGlobalCfg(); taosReadGlobalLogCfg(); // For log directory - if (stat(logDir, &dirstat) < 0) mkdir(logDir, 0755); + if (stat(tsLogDir, &dirstat) < 0) mkdir(tsLogDir, 0755); - sprintf(temp, "%s/taoslog", logDir); + sprintf(temp, "%s/taoslog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 10) < 0) { - printf("failed to open log file in directory:%s\n", logDir); + printf("failed to open log file in directory:%s\n", tsLogDir); } taosReadGlobalCfg(); @@ -133,7 +125,7 @@ void taos_init_imp() { taosPrintGlobalCfg(); tscTrace("starting to initialize TAOS client ..."); - tscTrace("Local IP address is:%s", tsLocalIp); + tscTrace("Local End Point is:%s", tsLocalEp); } taosSetCoreDump(); @@ -143,13 +135,12 @@ void taos_init_imp() { } tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.port = tsMnodeShellPort; tscMgmtIpSet.numOfIps = 1; - tscMgmtIpSet.ip[0] = inet_addr(tsMasterIp); + taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); - if (tsSecondIp[0] && strcmp(tsSecondIp, tsMasterIp) != 0) { + if (tsSecond[0] && strcmp(tsSecond, tsMaster) != 0) { tscMgmtIpSet.numOfIps = 2; - tscMgmtIpSet.ip[1] = inet_addr(tsSecondIp); + taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]); } tscInitMsgsFp(); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3dab478cf7..4656ee71b4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -47,18 +47,18 @@ SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { return NULL; } -void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { - if (tbufTell(pBuf) == 0) { +void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) { + if (tbufTell(bw) == 0) { return; } SCond cond = { .uid = uid, - .len = tbufTell(pBuf), + .len = tbufTell(bw), .cond = NULL, }; - cond.cond = tbufGetData(pBuf, true); + cond.cond = tbufGetData(bw, true); if (pTagCond->pCond == NULL) { pTagCond->pCond = taosArrayInit(3, sizeof(SCond)); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index f770c8d38f..de07ae35e5 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -24,10 +24,10 @@ extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; -extern char dataDir[]; -extern char logDir[]; -extern char scriptDir[]; -extern char osName[]; +extern char tsDataDir[]; +extern char tsLogDir[]; +extern char tsScriptDir[]; +extern char tsOsName[]; // system info extern int64_t tsPageSize; @@ -51,8 +51,10 @@ extern int32_t tsVersion; extern int32_t tscEmbedded; extern int64_t tsMsPerDay[2]; -extern char tsMasterIp[]; -extern char tsSecondIp[]; +extern char tsMaster[]; +extern char tsSecond[]; +extern char tsLocalEp[]; +extern uint16_t tsServerPort; extern uint16_t tsMnodeDnodePort; extern uint16_t tsMnodeShellPort; extern uint16_t tsDnodeShellPort; @@ -74,11 +76,13 @@ extern int16_t tsNumOfVnodesPerCore; extern int16_t tsNumOfTotalVnodes; extern uint32_t tsPublicIpInt; -extern int32_t tsMaxCacheSize; -extern int32_t tsSessionsPerVnode; +extern int32_t tsCacheBlockSize; +extern int32_t tsTotalBlocks; +extern int32_t tsTablesPerVnode; extern int16_t tsDaysPerFile; extern int32_t tsDaysToKeep; -extern int32_t tsRowsInFileBlock; +extern int32_t tsMinRowsInFileBlock; +extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; extern int16_t tsCompression; @@ -176,7 +180,8 @@ void taosInitGlobalCfg(); bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); - +int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port); + #ifdef __cplusplus } #endif diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 5aa50677c9..2749d96ff6 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -30,9 +30,9 @@ char configDir[TSDB_FILENAME_LEN] = "/etc/taos"; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; -char dataDir[TSDB_FILENAME_LEN] = "/var/lib/taos"; -char scriptDir[TSDB_FILENAME_LEN] = "/etc/taos"; -char osName[10] = "Linux"; +char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos"; +char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos"; +char tsOsName[10] = "Linux"; // system info, not configurable int64_t tsPageSize; @@ -60,8 +60,10 @@ int32_t tscEmbedded = 0; */ int64_t tsMsPerDay[] = {86400000L, 86400000000L}; -char tsMasterIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsSecondIp[TSDB_IPv4ADDR_LEN] = {0}; +char tsMaster[TSDB_FQDN_LEN] = {0}; +char tsSecond[TSDB_FQDN_LEN] = {0}; +char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port +uint16_t tsServerPort = 6030; uint16_t tsMnodeShellPort = 6030; // udp[6030-6034] tcp[6030] uint16_t tsDnodeShellPort = 6035; // udp[6035-6039] tcp[6035] uint16_t tsMnodeDnodePort = 6040; // udp/tcp @@ -70,8 +72,6 @@ uint16_t tsSyncPort = 6050; int32_t tsStatusInterval = 1; // second int32_t tsShellActivityTimer = 3; // second -int32_t tsVnodePeerHBTimer = 1; // second -int32_t tsMgmtPeerHBTimer = 1; // second int32_t tsMeterMetaKeepTimer = 7200; // second int32_t tsMetricMetaKeepTimer = 600; // second int32_t tsRpcTimer = 300; @@ -79,26 +79,26 @@ int32_t tsRpcMaxTime = 600; // seconds; float tsNumOfThreadsPerCore = 1.0; float tsRatioOfQueryThreads = 0.5; -char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; #ifdef _TD_ARM_32_ -int32_t tsSessionsPerVnode = 100; +int32_t tsTablesPerVnode = 100; #else -int32_t tsSessionsPerVnode = 1000; +int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES; #endif -int32_t tsMaxCacheSize = 64; //64M -int16_t tsDaysPerFile = 10; -int32_t tsDaysToKeep = 3650; -int32_t tsRowsInFileBlock = 4096; -int16_t tsCommitTime = 3600; // seconds -int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; -int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; -int16_t tsCommitLog = 1; -int32_t tsReplications = TSDB_REPLICA_MIN_NUM; +int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; +int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; +int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; +int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; +int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; +int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; +int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds +int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; +int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL; +int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; /** * Change the meaning of affected rows: @@ -110,7 +110,6 @@ int32_t tsNumOfMPeers = 3; int32_t tsMaxShellConns = 2000; int32_t tsMaxTables = 100000; -char tsLocalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsDefaultDB[TSDB_DB_NAME_LEN] = {0}; char tsDefaultUser[64] = "root"; char tsDefaultPass[64] = "taosdata"; @@ -181,7 +180,6 @@ float tsStreamComputDelayRatio = 0.1; int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance -char tsHttpIp[TSDB_IPv4ADDR_LEN] = "0.0.0.0"; uint16_t tsHttpPort = 6020; // only tcp, range tcp[6020] // uint16_t tsNginxPort = 6060; //only tcp, range tcp[6060] int32_t tsHttpCacheSessions = 100; @@ -192,7 +190,6 @@ int32_t tsHttpEnableRecordSql = 0; int32_t tsTelegrafUseFieldNum = 0; int32_t tsTscEnableRecordSql = 0; -int32_t tsAnyIp = 1; uint32_t tsPublicIpInt = 0; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; @@ -272,69 +269,29 @@ static void doInitGlobalConfig() { SGlobalCfg cfg = {0}; // ip address - cfg.option = "masterIp"; - cfg.ptr = tsMasterIp; + cfg.option = "master"; + cfg.ptr = tsMaster; cfg.valType = TAOS_CFG_VTYPE_IPSTR; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; + cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "secondIp"; - cfg.ptr = tsSecondIp; + cfg.option = "second"; + cfg.ptr = tsSecond; cfg.valType = TAOS_CFG_VTYPE_IPSTR; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "publicIp"; - cfg.ptr = tsPublicIp; - cfg.valType = TAOS_CFG_VTYPE_IPSTR; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "privateIp"; - cfg.ptr = tsPrivateIp; - cfg.valType = TAOS_CFG_VTYPE_IPSTR; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "localIp"; - cfg.ptr = tsLocalIp; - cfg.valType = TAOS_CFG_VTYPE_IPSTR; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpIp"; - cfg.ptr = tsHttpIp; - cfg.valType = TAOS_CFG_VTYPE_IPSTR; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_IPv4ADDR_LEN; + cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); // port - cfg.option = "mnodeShellPort"; - cfg.ptr = &tsMnodeShellPort; + cfg.option = "serverPort"; + cfg.ptr = &tsServerPort; cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 1; @@ -343,56 +300,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "dnodeShellPort"; - cfg.ptr = &tsDnodeShellPort; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65535; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "mnodeDnodePort"; - cfg.ptr = &tsMnodeDnodePort; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 65535; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "dnodeMnodePort"; - cfg.ptr = &tsDnodeMnodePort; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 65535; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - // cfg.option = "syncPort"; - // cfg.ptr = &syncPort; - // cfg.valType = TAOS_CFG_VTYPE_INT16; - // cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - // cfg.minValue = 1; - // cfg.maxValue = 65535; - // cfg.ptrLength = 0; - // cfg.unitType = TAOS_CFG_UTYPE_NONE; - // taosInitConfigOption(cfg); - - cfg.option = "httpPort"; - cfg.ptr = &tsHttpPort; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 65535; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - // directory cfg.option = "configDir"; cfg.ptr = configDir; @@ -405,7 +312,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "logDir"; - cfg.ptr = logDir; + cfg.ptr = tsLogDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; cfg.minValue = 0; @@ -415,7 +322,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "scriptDir"; - cfg.ptr = scriptDir; + cfg.ptr = tsScriptDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; @@ -425,7 +332,7 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "dataDir"; - cfg.ptr = dataDir; + cfg.ptr = tsDataDir; cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; @@ -567,16 +474,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "ctime"; - cfg.ptr = &tsCommitTime; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 30; - cfg.maxValue = 40960; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); - cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -678,32 +575,42 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); // database configs - cfg.option = "clog"; - cfg.ptr = &tsCommitLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "tables"; + cfg.ptr = &tsTablesPerVnode; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_TABLES; + cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "cache"; + cfg.ptr = &tsCacheBlockSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; + cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + + cfg.option = "blocks"; + cfg.ptr = &tsTotalBlocks; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; + cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365; + cfg.minValue = TSDB_MIN_DAYS_PER_FILE; + cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -712,8 +619,58 @@ static void doInitGlobalConfig() { cfg.ptr = &tsDaysToKeep; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365000; + cfg.minValue = TSDB_MIN_KEEP; + cfg.maxValue = TSDB_MAX_KEEP; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "minRows"; + cfg.ptr = &tsMinRowsInFileBlock; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "maxRows"; + cfg.ptr = &tsMaxRowsInFileBlock; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "ctime"; + cfg.ptr = &tsCommitTime; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_COMMIT_TIME; + cfg.maxValue = TSDB_MAX_COMMIT_TIME; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_SECOND; + taosInitConfigOption(cfg); + + cfg.option = "comp"; + cfg.ptr = &tsCompression; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_COMP_LEVEL; + cfg.maxValue = TSDB_MAX_COMP_LEVEL; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "clog"; + cfg.ptr = &tsCommitLog; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_CLOG_LEVEL; + cfg.maxValue = TSDB_MAX_CLOG_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -722,38 +679,8 @@ static void doInitGlobalConfig() { cfg.ptr = &tsReplications; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "tables"; - cfg.ptr = &tsSessionsPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES_PER_VNODE; - cfg.maxValue = TSDB_MAX_TABLES_PER_VNODE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "cache"; - cfg.ptr = &tsMaxCacheSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); - - cfg.option = "rows"; - cfg.ptr = &tsRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 200; - cfg.maxValue = 1048576; + cfg.minValue = TSDB_MIN_REPLICA_NUM; + cfg.maxValue = TSDB_MAX_REPLICA_NUM; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -1205,16 +1132,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "anyIp"; - cfg.ptr = &tsAnyIp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - // version info cfg.option = "gitinfo"; cfg.ptr = gitinfo; @@ -1262,25 +1179,15 @@ void taosInitGlobalCfg() { } bool taosCheckGlobalCfg() { - if (tsPrivateIp[0] == 0) { - taosGetPrivateIp(tsPrivateIp); + taosGetFqdn(tsLocalEp); + sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort); + + if (tsMaster[0] == 0) { + strcpy(tsMaster, tsLocalEp); } - if (tsPublicIp[0] == 0) { - strcpy(tsPublicIp, tsPrivateIp); - } - tsPublicIpInt = inet_addr(tsPublicIp); - - if (tsLocalIp[0] == 0) { - strcpy(tsLocalIp, tsPrivateIp); - } - - if (tsMasterIp[0] == 0) { - strcpy(tsMasterIp, tsPrivateIp); - } - - if (tsSecondIp[0] == 0) { - strcpy(tsSecondIp, tsMasterIp); + if (tsSecond[0] == 0) { + strcpy(tsSecond, tsLocalEp); } taosGetSystemInfo(); @@ -1302,15 +1209,6 @@ bool taosCheckGlobalCfg() { tsNumOfTotalVnodes = tsNumOfTotalVnodes < TSDB_MIN_VNODES ? TSDB_MIN_VNODES : tsNumOfTotalVnodes; } - if (strlen(tsPrivateIp) == 0) { - uError("privateIp is null"); - return false; - } - - if (tscEmbedded) { - strcpy(tsLocalIp, tsPrivateIp); - } - // todo refactor tsVersion = 0; for (int i = 0; i < 10; i++) { @@ -1323,5 +1221,26 @@ bool taosCheckGlobalCfg() { tsVersion = 10 * tsVersion; + tsMnodeShellPort = tsServerPort + TSDB_PORT_MNODESHELL; // udp[6030-6034] tcp[6030] + tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035] + tsMnodeDnodePort = tsServerPort + TSDB_PORT_MNODEDNODE; // udp/tcp + tsDnodeMnodePort = tsServerPort + TSDB_PORT_DNODEMNODE; // udp/tcp + tsSyncPort = tsServerPort + TSDB_PORT_SYNC; + return true; } + +int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp+1); + } + + if (*port == 0) *port = tsServerPort; + + return 0; +} diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index 151d44922d..8b3f21b103 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -59,16 +59,13 @@ void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { tsMnodeIpSet = *pIpSet; } -void dnodeGetMnodeIpSet(void *ipSetRaw, bool usePublicIp) { +void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) { SRpcIpSet *ipSet = ipSetRaw; ipSet->numOfIps = tsMnodeInfos.nodeNum; ipSet->inUse = tsMnodeInfos.inUse; for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) { - if (usePublicIp) { - ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; - } else { - ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; - } + taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]); + ipSet->port[i] += TSDB_PORT_MNODEDNODE; } } @@ -85,26 +82,23 @@ int32_t dnodeInitMClient() { if (!dnodeReadMnodeInfos()) { memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet)); memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); - tsMnodeIpSet.port = tsMnodeDnodePort; tsMnodeIpSet.numOfIps = 1; - tsMnodeIpSet.ip[0] = inet_addr(tsMasterIp); - if (strcmp(tsSecondIp, tsMasterIp) != 0) { + taosGetFqdnPortFromEp(tsMaster, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]); + tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE; + if (strcmp(tsSecond, tsMaster) != 0) { tsMnodeIpSet.numOfIps = 2; - tsMnodeIpSet.ip[1] = inet_addr(tsSecondIp); + taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]); } } else { tsMnodeIpSet.inUse = tsMnodeInfos.inUse; tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum; - tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort; for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { - tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]); } } SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; - rpcInit.localPort = 0; rpcInit.label = "DND-MC"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessRspFromMnode; @@ -182,9 +176,6 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; pMnodeInfo->nodeId = htonl(pMnodeInfo->nodeId); - pMnodeInfo->nodeIp = htonl(pMnodeInfo->nodeIp); - pMnodeInfo->nodePort = htons(pMnodeInfo->nodePort); - pMnodeInfo->syncPort = htons(pMnodeInfo->syncPort); } SDMVgroupAccess *pVgAcccess = pStatusRsp->vgAccess; @@ -207,15 +198,14 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { tsMnodeIpSet.inUse = tsMnodeInfos.inUse; tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum; - tsMnodeIpSet.port = tsMnodeInfos.nodeInfos[0].nodePort; for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { - tsMnodeIpSet.ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]); + tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE; } dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { - dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), - tsMnodeInfos.nodeInfos[i].nodePort, tsMnodeInfos.nodeInfos[i].nodeName); + dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp); } dnodeSaveMnodeInfos(); @@ -291,42 +281,19 @@ static bool dnodeReadMnodeInfos() { } tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; - cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); - if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { - dError("failed to read mnode mgmtIpList.json, nodeIp not found"); - goto PARSE_OVER; - } - tsMnodeInfos.nodeInfos[i].nodeIp = inet_addr(nodeIp->valuestring); - - cJSON *nodePort = cJSON_GetObjectItem(nodeInfo, "nodePort"); - if (!nodePort || nodePort->type != cJSON_Number) { - dError("failed to read mnode mgmtIpList.json, nodePort not found"); - goto PARSE_OVER; - } - tsMnodeInfos.nodeInfos[i].nodePort = (uint16_t)nodePort->valueint; - - cJSON *syncPort = cJSON_GetObjectItem(nodeInfo, "syncPort"); - if (!syncPort || syncPort->type != cJSON_Number) { - dError("failed to read mnode mgmtIpList.json, syncPort not found"); - goto PARSE_OVER; - } - tsMnodeInfos.nodeInfos[i].syncPort = (uint16_t)syncPort->valueint; - - cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); - if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { dError("failed to read mnode mgmtIpList.json, nodeName not found"); goto PARSE_OVER; } - strncpy(tsMnodeInfos.nodeInfos[i].nodeName, nodeName->valuestring, TSDB_NODE_NAME_LEN); - } + strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_FQDN_LEN); + } ret = true; dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse); for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { - dPrint("mnode:%d, ip:%s:%u name:%s", tsMnodeInfos.nodeInfos[i].nodeId, - taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp), tsMnodeInfos.nodeInfos[i].nodePort, - tsMnodeInfos.nodeInfos[i].nodeName); + dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp); } PARSE_OVER: @@ -352,10 +319,7 @@ static void dnodeSaveMnodeInfos() { len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) { len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId); - len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", taosIpStr(tsMnodeInfos.nodeInfos[i].nodeIp)); - len += snprintf(content + len, maxLen - len, " \"nodePort\": %u,\n", tsMnodeInfos.nodeInfos[i].nodePort); - len += snprintf(content + len, maxLen - len, " \"syncPort\": %u,\n", tsMnodeInfos.nodeInfos[i].syncPort); - len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeName); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp); if (i < tsMnodeInfos.nodeNum -1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { @@ -371,8 +335,8 @@ static void dnodeSaveMnodeInfos() { dPrint("save mnode iplist successed"); } -uint32_t dnodeGetMnodeMasteIp() { - return tsMnodeIpSet.ip[tsMnodeIpSet.inUse]; +char *dnodeGetMnodeMasterEp() { + return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp; } void* dnodeGetMnodeInfos() { @@ -402,8 +366,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { //strcpy(pStatus->dnodeName, tsDnodeName); pStatus->version = htonl(tsVersion); pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId); - pStatus->privateIp = htonl(inet_addr(tsPrivateIp)); - pStatus->publicIp = htonl(inet_addr(tsPublicIp)); + strcpy(pStatus->dnodeEp, tsLocalEp); pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfTotalVnodes = htons((uint16_t) tsNumOfTotalVnodes); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); @@ -500,4 +463,4 @@ void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) { int32_t dnodeGetDnodeId() { return tsDnodeCfg.dnodeId; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index b8ac9d1d4a..43b16ae6ea 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -144,12 +144,12 @@ static int32_t dnodeInitSystem() { signal(SIGPIPE, SIG_IGN); struct stat dirstat; - if (stat(logDir, &dirstat) < 0) { - mkdir(logDir, 0755); + if (stat(tsLogDir, &dirstat) < 0) { + mkdir(tsLogDir, 0755); } char temp[TSDB_FILENAME_LEN]; - sprintf(temp, "%s/taosdlog", logDir); + sprintf(temp, "%s/taosdlog", tsLogDir); if (taosInitLog(temp, tsNumOfLogLines, 1) < 0) { printf("failed to init log file\n"); } @@ -161,8 +161,7 @@ static int32_t dnodeInitSystem() { } taosPrintGlobalCfg(); - dPrint("Server IP address is:%s", tsPrivateIp); - dPrint("starting to initialize TDengine ..."); + dPrint("start to initialize TDengine on %s", tsLocalEp); if (dnodeInitStorage() != 0) return -1; if (dnodeInitRead() != 0) return -1; @@ -218,13 +217,13 @@ static void dnodeCheckDataDirOpenned(char *dir) { static int32_t dnodeInitStorage() { struct stat dirstat; - if (stat(dataDir, &dirstat) < 0) { - mkdir(dataDir, 0755); + if (stat(tsDataDir, &dirstat) < 0) { + mkdir(tsDataDir, 0755); } - sprintf(tsMnodeDir, "%s/mnode", dataDir); - sprintf(tsVnodeDir, "%s/vnode", dataDir); - sprintf(tsDnodeDir, "%s/dnode", dataDir); + sprintf(tsMnodeDir, "%s/mnode", tsDataDir); + sprintf(tsVnodeDir, "%s/vnode", tsDataDir); + sprintf(tsDnodeDir, "%s/dnode", tsDataDir); mkdir(tsVnodeDir, 0755); mkdir(tsDnodeDir, 0755); @@ -237,5 +236,5 @@ static int32_t dnodeInitStorage() { static void dnodeCleanupStorage() {} bool dnodeIsFirstDeploy() { - return strcmp(tsMasterIp, tsPrivateIp) == 0; -} \ No newline at end of file + return strcmp(tsMaster, tsLocalEp) == 0; +} diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 97768ca743..2fa7762c44 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -128,20 +128,20 @@ static void dnodeCloseVnodes() { static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); + pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.maxCacheSize = htobe64(pCreate->cfg.maxCacheSize); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); - pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp); for (int32_t j = 0; j < pCreate->cfg.replications; ++j) { pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); - pCreate->nodes[j].nodeIp = htonl(pCreate->nodes[j].nodeIp); } void *pVnode = vnodeAccquireVnode(pCreate->cfg.vgId); diff --git a/src/dnode/src/dnodeMnode.c b/src/dnode/src/dnodeMnode.c index ac477c4de1..75c09d43ba 100644 --- a/src/dnode/src/dnodeMnode.c +++ b/src/dnode/src/dnodeMnode.c @@ -38,7 +38,6 @@ int32_t dnodeInitMnode() { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; rpcInit.localPort = tsDnodeMnodePort; rpcInit.label = "DND-MS"; rpcInit.numOfThreads = 1; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 8a846b8a50..544d443bc0 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -47,7 +47,6 @@ int32_t dnodeInitShell() { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; rpcInit.localPort = tsDnodeShellPort; rpcInit.label = "DND-shell"; rpcInit.numOfThreads = numOfThreads; diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 6fbfa7ffa9..c4b893ab86 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -42,8 +42,8 @@ void dnodeFreeRqueue(void *rqueue); void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code); bool dnodeIsFirstDeploy(); -uint32_t dnodeGetMnodeMasteIp(); -void dnodeGetMnodeIpSet(void *ipSet, bool usePublicIp); +char *dnodeGetMnodeMasterEp(); +void dnodeGetMnodeDnodeIpSet(void *ipSet); void * dnodeGetMnodeInfos(); int32_t dnodeGetDnodeId(); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 89a3cbb3a9..a712b14c8f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -187,11 +187,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_CITY_LEN 20 #define TSDB_STATE_LEN 20 #define TSDB_COUNTRY_LEN 20 -#define TSDB_VNODES_SUPPORT 6 -#define TSDB_MGMT_SUPPORT 4 #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 +#define TSDB_FQDN_LEN 64 #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_METER_VNODE_BITS 20 @@ -218,47 +217,58 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_DNODE_ROLE_MGMT 1 #define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_MAX_MPEERS 5 -#define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) - -#define TSDB_REPLICA_MIN_NUM 1 -#define TSDB_REPLICA_MAX_NUM 3 +#define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta -//default value == 10 -#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days -#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days - -#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. -#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years - -#define TSDB_MIN_COMPRESSION_LEVEL 0 -#define TSDB_MAX_COMPRESSION_LEVEL 2 - -#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 -#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 - -#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 -#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 - #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_CACHE_BLOCKS 100 -#define TSDB_MAX_CACHE_BLOCKS 409600 +#define TSDB_MIN_TOTAL_BLOCKS 2 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 2 -#define TSDB_MIN_AVG_BLOCKS 2 -#define TSDB_MAX_AVG_BLOCKS 2048 -#define TSDB_DEFAULT_AVG_BLOCKS 4 +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 200000 +#define TSDB_DEFAULT_TABLES 1000 -/* - * There is a bug in function taosAllocateId. - * When "create database tables 1" is executed, the wrong sid is assigned, so the minimum value is set to 2. - */ -#define TSDB_MIN_TABLES_PER_VNODE 2 -#define TSDB_MAX_TABLES_PER_VNODE 220000 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 + +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years + +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 + +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 + +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 + +#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_CLOG_LEVEL 0 +#define TSDB_MAX_CLOG_LEVEL 2 +#define TSDB_DEFAULT_CLOG_LEVEL 2 + +#define TSDB_MIN_REPLICA_NUM 1 +#define TSDB_MAX_REPLICA_NUM 3 +#define TSDB_DEFAULT_REPLICA_NUM 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_UNION_CLAUSE 5 @@ -269,24 +279,25 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_RPC_THREADS 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00U // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01U // free qhandle at vnode +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02U -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04U // two-stage subquery for super table +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08U // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10U // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20U // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40U // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80U // join sub query at the second stage +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage -#define TSDB_QUERY_TYPE_INSERT 0x100U // insert type -#define TSDB_QUERY_TYPE_IMPORT 0x200U // import data +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_IMPORT 0x200u // import data #define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) #define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) @@ -308,6 +319,12 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_NORMAL_TABLES 1000 #define TSDB_MAX_CHILD_TABLES 100000 +#define TSDB_PORT_MNODESHELL 0 +#define TSDB_PORT_DNODESHELL 5 +#define TSDB_PORT_DNODEMNODE 10 +#define TSDB_PORT_MNODEDNODE 15 +#define TSDB_PORT_SYNC 20 + typedef enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5084199ac8..c9071f4cb0 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -97,6 +97,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table n TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table") TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_ALREAY_EXIST, 0, 207, "tag already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_NOT_EXIST, 0, 208, "tag not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_ALREAY_EXIST, 0, 209, "field already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_NOT_EXIST, 0, 210, "field not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 211, "column name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TAGS, 0, 211, "too many tags") + // dnode & mnode TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 250, "no enough dnodes") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e9842cc922..a7e509f75c 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -187,7 +187,7 @@ extern char *taosMsg[]; #pragma pack(push, 1) typedef struct { - uint32_t ip; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; } SIpAddr; @@ -238,7 +238,7 @@ typedef struct { typedef struct SSchema { uint8_t type; - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN + 1]; int16_t colId; int16_t bytes; } SSchema; @@ -256,15 +256,16 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char superTableId[TSDB_TABLE_ID_LEN + 1]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int8_t igExists; + int8_t getMeta; int16_t numOfTags; int16_t numOfColumns; int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string @@ -274,13 +275,13 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int16_t type; /* operation type */ char tagVal[TSDB_MAX_BYTES_PER_ROW]; int8_t numOfCols; /* number of schema */ @@ -345,7 +346,6 @@ typedef struct { } SMDDropTableMsg; typedef struct { - int32_t contLen; int32_t vgId; int64_t uid; char tableId[TSDB_TABLE_ID_LEN + 1]; @@ -492,6 +492,7 @@ typedef struct SRetrieveTableRsp { typedef struct { int32_t vgId; + int32_t cfgVersion; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; @@ -502,29 +503,23 @@ typedef struct { } SVnodeLoad; typedef struct { - char acct[TSDB_USER_LEN]; - char db[TSDB_DB_NAME_LEN]; - uint32_t vgId; + char acct[TSDB_USER_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int32_t maxSessions; - int32_t cacheBlockSize; - union { - int32_t totalBlocks; - float fraction; - } cacheNumOfBlocks; - int32_t daysPerFile; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t daysToKeep; - int32_t commitTime; - int32_t rowsInFileBlock; - int16_t blocksPerTable; - int8_t compression; - int8_t commitLog; - int8_t replications; - int8_t repStrategy; - int8_t loadLatest; // load into mem or not - uint8_t precision; // time resolution - int8_t ignoreExist; + int32_t cacheBlockSize; //MB + int32_t totalBlocks; + int32_t daysPerFile; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t daysToKeep; + int32_t commitTime; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int8_t compression; + int8_t commitLog; + int8_t replications; + uint8_t precision; // time resolution + int8_t ignoreExist; } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { @@ -555,24 +550,19 @@ typedef struct { typedef struct { int32_t nodeId; - uint32_t nodeIp; - uint16_t nodePort; - uint16_t syncPort; - char nodeName[TSDB_NODE_NAME_LEN + 1]; + char nodeEp[TSDB_FQDN_LEN]; } SDMMnodeInfo; typedef struct { int8_t inUse; int8_t nodeNum; - SDMMnodeInfo nodeInfos[TSDB_MAX_MPEERS]; + SDMMnodeInfo nodeInfos[TSDB_MAX_REPLICA]; } SDMMnodeInfos; typedef struct { uint32_t version; int32_t dnodeId; - char dnodeName[TSDB_NODE_NAME_LEN + 1]; - uint32_t privateIp; - uint32_t publicIp; + char dnodeEp[TSDB_FQDN_LEN]; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot uint16_t numOfTotalVnodes; // from config file @@ -592,34 +582,34 @@ typedef struct { typedef struct { uint32_t vgId; + int32_t cfgVersion; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; - int64_t maxCacheSize; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; - uint8_t precision; // time resolution + int8_t precision; int8_t compression; - int8_t wals; int8_t commitLog; int8_t replications; + int8_t wals; int8_t quorum; - uint32_t arbitratorIp; int8_t reserved[16]; } SMDVnodeCfg; typedef struct { int32_t nodeId; - uint32_t nodeIp; - char nodeName[TSDB_NODE_NAME_LEN + 1]; + char nodeEp[TSDB_FQDN_LEN]; } SMDVnodeDesc; typedef struct { SMDVnodeCfg cfg; - SMDVnodeDesc nodes[TSDB_MAX_MPEERS]; + SMDVnodeDesc nodes[TSDB_MAX_REPLICA]; } SMDCreateVnodeMsg; typedef struct { @@ -634,48 +624,54 @@ typedef struct { } SCMMultiTableInfoMsg; typedef struct SCMSTableVgroupMsg { - char tableId[TSDB_TABLE_ID_LEN]; -} SCMSTableVgroupMsg; + int32_t numOfTables; +} SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { int32_t vgId; int8_t numOfIps; - SIpAddr ipAddr[TSDB_REPLICA_MAX_NUM]; + SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; } SCMVgroupInfo; typedef struct { - int32_t numOfVgroups; + int32_t numOfVgroups; SCMVgroupInfo vgroups[]; -} SCMSTableVgroupRspMsg; +} SVgroupsInfo; -typedef struct { - int16_t elemLen; +//typedef struct { +// int32_t numOfTables; +//// int32_t numOfVgroups; +//// SCMVgroupInfo vgroups[]; +//} SCMSTableVgroupRspMsg; - char tableId[TSDB_TABLE_ID_LEN + 1]; - int16_t orderIndex; - int16_t orderType; // used in group by xx order by xxx - - int16_t rel; // denotes the relation between condition and table list - - int32_t tableCond; // offset value of table name condition - int32_t tableCondLen; - - int32_t cond; // offset of column query condition - int32_t condLen; - - int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name - int16_t numOfTags; // required number of tags - - int16_t numOfGroupCols; // num of group by columns - int32_t groupbyTagColumnList; -} SSuperTableMetaElemMsg; - -typedef struct { - int32_t numOfTables; - int32_t join; - int32_t joinCondLen; // for join condition - int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM]; -} SSuperTableMetaMsg; +//typedef struct { +// int16_t elemLen; +// +// char tableId[TSDB_TABLE_ID_LEN + 1]; +// int16_t orderIndex; +// int16_t orderType; // used in group by xx order by xxx +// +// int16_t rel; // denotes the relation between condition and table list +// +// int32_t tableCond; // offset value of table name condition +// int32_t tableCondLen; +// +// int32_t cond; // offset of column query condition +// int32_t condLen; +// +// int16_t tagCols[TSDB_MAX_TAGS + 1]; // required tag columns, plus one is for table name +// int16_t numOfTags; // required number of tags +// +// int16_t numOfGroupCols; // num of group by columns +// int32_t groupbyTagColumnList; +//} SSuperTableMetaElemMsg; +// +//typedef struct { +// int32_t numOfTables; +// int32_t join; +// int32_t joinCondLen; // for join condition +// int32_t metaElem[TSDB_MAX_JOIN_TABLE_NUM]; +//} SSuperTableMetaMsg; typedef struct { int32_t nodeId; @@ -684,24 +680,17 @@ typedef struct { } SVnodeDesc; typedef struct { - SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM]; + SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM]; int16_t index; // used locally int32_t vgId; int32_t numOfSids; int32_t pSidExtInfoList[]; // offset value of STableIdInfo } SVnodeSidList; -typedef struct { - int32_t numOfTables; - int32_t numOfVnodes; - uint16_t tagLen; /* tag value length */ - int32_t list[]; /* offset of SVnodeSidList, compared to the SSuperTableMeta struct */ -} SSuperTableMeta; - typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id - char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + char tableId[TSDB_TABLE_ID_LEN + 1]; // table id + char stableId[TSDB_TABLE_ID_LEN + 1]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -742,7 +731,7 @@ typedef struct SCMShowRsp { } SCMShowRsp; typedef struct { - char ip[32]; + char ep[TSDB_FQDN_LEN]; // end point, hostname:port } SCMCreateDnodeMsg, SCMDropDnodeMsg; typedef struct { @@ -757,7 +746,7 @@ typedef struct { } SDMConfigVnodeMsg; typedef struct { - char ip[32]; + char ep[TSDB_FQDN_LEN]; // end point, hostname:port char config[64]; } SMDCfgDnodeMsg, SCMCfgDnodeMsg; diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 5845823b38..8b082b65b8 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -31,8 +31,8 @@ extern int tsRpcHeadSize; typedef struct { int8_t inUse; int8_t numOfIps; - uint16_t port; - uint32_t ip[TSDB_MAX_MPEERS]; + uint16_t port[TSDB_MAX_REPLICA]; + char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SRpcIpSet; typedef struct { @@ -51,7 +51,6 @@ typedef struct { } SRpcMsg; typedef struct { - char *localIp; // local IP used uint16_t localPort; // local port char *label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 7d760bf289..9259da85e3 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -45,15 +45,19 @@ typedef struct { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { - int8_t precision; - int8_t compression; int32_t tsdbId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy + int32_t keep; // day of data to keep + int32_t keep1; + int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t keep; // day of data to keep - int64_t maxCacheSize; // maximum cache size this TSDB can use + int32_t commitTime; + int8_t precision; + int8_t compression; } STsdbCfg; void tsdbSetDefaultCfg(STsdbCfg *pCfg); @@ -137,15 +141,10 @@ int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg); typedef void *TsdbQueryHandleT; // Use void to hide implementation details -typedef struct STableGroupList { // qualified table object list in group - SArray *pGroupList; - int32_t numOfTables; -} STableGroupList; - // query condition to build vnode iterator typedef struct STsdbQueryCond { STimeWindow twindow; - int32_t order; // desc/asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo *colList; } STsdbQueryCond; @@ -178,14 +177,29 @@ typedef void *TsdbPosT; /** * Get the data block iterator, starting from position according to the query condition - * @param pCond query condition, only includes the filter on primary time stamp - * @param pTableList table sid list + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param groupInfo tableId list in the form of set, seperated into different groups according to group by condition * @return */ TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); /** - * move to next block + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param groupInfo tableId list. + * @return + */ +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo); + +/** + * move to next block if exists + * * @param pQueryHandle * @return */ @@ -212,26 +226,16 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle); int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); /** + * * The query condition with primary timestamp is passed to iterator during its constructor function, * the returned data block must be satisfied with the time window condition in any cases, * which means the SData data block is not actually the completed disk data blocks. * - * @param pQueryHandle + * @param pQueryHandle query handle + * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pIdList); - -/** - * todo remove the parameter of position, and order type - * - * Reset to the start(end) position of current query, from which the iterator starts. - * - * @param pQueryHandle - * @param position set the iterator traverses position - * @param order ascending order or descending order - * @return - */ -int32_t tsdbResetQuery(TsdbQueryHandleT *pQueryHandle, STimeWindow *window, TsdbPosT position, int16_t order); +SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); /** * todo remove this function later @@ -268,20 +272,19 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * Get the qualified table id for a super table according to the tag query expression. * @param stableid. super table sid * @param pTagCond. tag query condition - * */ -int32_t tsdbQuerySTableByTagCond( - TsdbRepoT *tsdb, - int64_t uid, - const char *pTagCond, - size_t len, - int16_t tagNameRelType, - const char* tbnameCond, - STableGroupInfo *pGroupList, - SColIndex *pColIndex, - int32_t numOfCols - ); +int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, + int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); /** diff --git a/src/inc/tsync.h b/src/inc/tsync.h index c7a05c0a2d..4b766664f3 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -38,14 +38,15 @@ typedef enum _TAOS_SYNC_STATUS { typedef struct { uint32_t nodeId; // node ID assigned by TDengine - uint32_t nodeIp; // node IP address - char name[TSDB_FILENAME_LEN]; // external node name + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN } SNodeInfo; typedef struct { - uint32_t arbitratorIp; // arbitrator IP address int8_t quorum; // number of confirms required, >=1 int8_t replica; // number of replications, >=1 + uint16_t arbitratorPort; // arbitrator port + char arbitratorFqdn[TSDB_FQDN_LEN]; // arbitrator IP address SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; } SSyncCfg; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index c85ed47023..eeaeec83f2 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -40,7 +40,7 @@ History history; */ TAOS *shellInit(struct arguments *args) { printf("\n"); - printf(CLIENT_VERSION, osName, taos_get_client_info()); + printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); fflush(stdout); // set options before initializing diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 6cbfc72e62..c0fc3ea3d5 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -31,13 +31,9 @@ struct SMnodeObj; typedef struct SDnodeObj { int32_t dnodeId; - uint32_t privateIp; - uint32_t publicIp; - uint16_t mnodeShellPort; - uint16_t mnodeDnodePort; - uint16_t dnodeShellPort; - uint16_t dnodeMnodePort; - uint16_t syncPort; + uint16_t dnodePort; + char dnodeFqdn[TSDB_FQDN_LEN]; + char dnodeEp[TSDB_FQDN_LEN]; int64_t createdTime; uint32_t lastAccess; int32_t openVnodes; @@ -47,7 +43,6 @@ typedef struct SDnodeObj { int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t status; // set in balance function int8_t isMgmt; - char dnodeName[TSDB_NODE_NAME_LEN + 1]; int8_t reserved[15]; int8_t updateEnd[1]; int32_t refCount; @@ -123,7 +118,7 @@ typedef struct SVgObj { uint32_t vgId; char dbName[TSDB_DB_NAME_LEN + 1]; int64_t createdTime; - SVnodeGid vnodeGid[TSDB_VNODES_SUPPORT]; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; int32_t numOfVnodes; int32_t lbDnodeId; int32_t lbTime; @@ -143,14 +138,15 @@ typedef struct SVgObj { } SVgObj; typedef struct { - int64_t maxCacheSize; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; // minimum rows per file block - int32_t maxRowsPerFileBlock; // maximum rows per file block + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; int8_t precision; int8_t compression; @@ -163,6 +159,7 @@ typedef struct SDbObj { char name[TSDB_DB_NAME_LEN + 1]; char acct[TSDB_USER_LEN + 1]; int64_t createdTime; + int32_t cfgVersion; SDbCfg cfg; int8_t status; int8_t reserved[14]; diff --git a/src/mnode/inc/mgmtDnode.h b/src/mnode/inc/mgmtDnode.h index 8acd4e9117..1b5199e727 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/mnode/inc/mgmtDnode.h @@ -38,7 +38,7 @@ void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); void mgmtIncDnodeRef(SDnodeObj *pDnode); void mgmtDecDnodeRef(SDnodeObj *pDnode); void * mgmtGetDnode(int32_t dnodeId); -void * mgmtGetDnodeByIp(uint32_t ip); +void * mgmtGetDnodeByIp(char *ep); void mgmtUpdateDnode(SDnodeObj *pDnode); int32_t mgmtDropDnode(SDnodeObj *pDnode); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index 4fb57d7151..cebbc061f6 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -40,7 +40,7 @@ void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode); void mgmtReleaseMnode(struct SMnodeObj *pMnode); char * mgmtGetMnodeRoleStr(); -void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp); +void mgmtGetMnodeIpSet(SRpcIpSet *ipSet); void mgmtGetMnodeInfos(void *mnodes); #ifdef __cplusplus diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 058ed06f84..3f8927fbd0 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -50,7 +50,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup); -SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip); +SRpcIpSet mgmtGetIpSetFromIp(char *ep); #ifdef __cplusplus } diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c index 7b85dc08e3..c6c10e0da0 100644 --- a/src/mnode/src/mgmtBalance.c +++ b/src/mnode/src/mgmtBalance.c @@ -17,6 +17,7 @@ #include "os.h" #include "trpc.h" #include "tbalance.h" +#include "tglobal.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtMnode.h" @@ -41,7 +42,10 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { if (pDnode == NULL) break; if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) { - float usage = (float)pDnode->openVnodes / pDnode->totalVnodes; + float openVnodes = pDnode->openVnodes; + if (pDnode->isMgmt) openVnodes += tsMgmtEqualVnodeNum; + + float usage = openVnodes / pDnode->totalVnodes; if (usage <= vnodeUsage) { pSelDnode = pDnode; vnodeUsage = usage; diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index 759d400eb8..229964e1d6 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -38,7 +38,6 @@ static void *tsMgmtDClientRpc = NULL; int32_t mgmtInitDClient() { SRpcInit rpcInit = {0}; - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; rpcInit.localPort = 0; rpcInit.label = "MND-DC"; rpcInit.numOfThreads = 1; diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 9f65cd5d4a..9fd928e7da 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -43,7 +43,6 @@ static void *tsMgmtDServerQhandle = NULL; int32_t mgmtInitDServer() { SRpcInit rpcInit = {0}; - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;; rpcInit.localPort = tsMnodeDnodePort; rpcInit.label = "MND-DS"; rpcInit.numOfThreads = 1; @@ -105,14 +104,12 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) { if (!sdbIsMaster()) { SRpcConnInfo connInfo; rpcGetConnInfo(rpcMsg->handle, &connInfo); - bool usePublicIp = false; SRpcIpSet ipSet = {0}; - ipSet.port = tsMnodeDnodePort; - dnodeGetMnodeIpSet(&ipSet, usePublicIp); + dnodeGetMnodeDnodeIpSet(&ipSet); mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); for (int32_t i = 0; i < ipSet.numOfIps; ++i) { - mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]); } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index b3424ffd63..a78fba2208 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -22,6 +22,7 @@ #include "tglobal.h" #include "ttime.h" #include "tname.h" +#include "tbalance.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtAcct.h" @@ -177,78 +178,81 @@ SDbObj *mgmtGetDbByTableId(char *tableId) { } static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - mError("invalid db option maxCacheSize:%d valid range: [%d, %d]", pCfg->maxCacheSize, TSDB_MIN_CACHE_BLOCK_SIZE, + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); } - if (pCfg->maxTables < TSDB_MIN_TABLES_PER_VNODE || pCfg->maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS, + TSDB_MAX_TOTAL_BLOCKS); + } + + if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { - mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, - TSDB_FILE_MAX_PARTITION_RANGE); + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysToKeep1 < pCfg->daysPerFile) { - mError("invalid db option daystokeep:%d", pCfg->daysToKeep); + if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) { + mError("invalid db option daysToKeep:%d", pCfg->daysToKeep); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep2 > pCfg->daysToKeep || pCfg->daysToKeep2 < pCfg->daysToKeep1) { - mError("invalid db option daystokeep1:%d, daystokeep2:%d, daystokeep:%d", pCfg->daysToKeep1, - pCfg->daysToKeep2, pCfg->daysToKeep); + if (pCfg->daysToKeep < pCfg->daysPerFile) { + mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile); return TSDB_CODE_INVALID_OPTION; } - - if (pCfg->minRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < pCfg->minRowsPerFileBlock) { + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCfg->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { - mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, - TSDB_MAX_COMMIT_TIME_INTERVAL); + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->precision != TSDB_TIME_PRECISION_MILLI && pCfg->precision != TSDB_TIME_PRECISION_MICRO) { - mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { + mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION, + TSDB_MAX_PRECISION); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->compression < TSDB_MIN_COMPRESSION_LEVEL || pCfg->compression > TSDB_MAX_COMPRESSION_LEVEL) { - mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMP_LEVEL, + TSDB_MAX_COMP_LEVEL); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitLog < 0 || pCfg->commitLog > 2) { + if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) { mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->replications < TSDB_REPLICA_MIN_NUM || pCfg->replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); + if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, + TSDB_MAX_REPLICA_NUM); return TSDB_CODE_INVALID_OPTION; } @@ -256,14 +260,15 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { } static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < 0) pCfg->maxCacheSize = tsMaxCacheSize; - if (pCfg->maxTables < 0) pCfg->maxTables = tsSessionsPerVnode; + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks; + if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile; if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; - if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsRowsInFileBlock; - if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = pCfg->minRowsPerFileBlock * 2; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock; if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; if (pCfg->compression < 0) pCfg->compression = tsCompression; @@ -293,14 +298,15 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); pDb->createdTime = taosGetTimestampMs(); pDb->cfg = (SDbCfg) { - .maxCacheSize = 64,//(int64_t)pCreate->cacheBlockSize * pCreate->cacheNumOfBlocks.totalBlocks, + .cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, .maxTables = pCreate->maxSessions, .daysPerFile = pCreate->daysPerFile, .daysToKeep = pCreate->daysToKeep, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRowsPerFileBlock = pCreate->rowsInFileBlock * 1, - .maxRowsPerFileBlock = pCreate->rowsInFileBlock * 2, + .minRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, .commitTime = pCreate->commitTime, .precision = pCreate->precision, .compression = pCreate->compression, @@ -459,13 +465,25 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "rows"); + strcpy(pSchema[cols].name, "cache(MB)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "blocks"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(Mb)"); + strcpy(pSchema[cols].name, "maxrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -492,7 +510,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 3; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "time precision"); + strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -583,14 +601,22 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * *(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1 cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxCacheSize; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; cols++; - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.commitTime; cols++; @@ -664,14 +690,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); pCreate->daysToKeep = htonl(pCreate->daysToKeep); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->blocksPerTable = htons(pCreate->blocksPerTable); - pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); - + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + int32_t code; if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_GRANT_EXPIRED; @@ -688,38 +715,70 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { - SDbCfg newCfg = pDb->cfg; - int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxTables = htonl(pAlter->maxSessions); + SDbCfg newCfg = pDb->cfg; + int32_t cacheBlockSize = htonl(pAlter->daysToKeep); + int32_t totalBlocks = htonl(pAlter->totalBlocks); + int32_t maxTables = htonl(pAlter->maxSessions); + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t daysToKeep1 = htonl(pAlter->daysToKeep1); + int32_t daysToKeep2 = htonl(pAlter->daysToKeep2); + int8_t compression = pAlter->compression; int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; + if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) { + mTrace("db:%s, cache:%d change to %d", pDb->name, pDb->cfg.cacheBlockSize, cacheBlockSize); + newCfg.cacheBlockSize = cacheBlockSize; + } + + if (totalBlocks > 0 && totalBlocks != pDb->cfg.totalBlocks) { + mTrace("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks); + newCfg.totalBlocks = totalBlocks; + } + + if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); + newCfg.maxTables = maxTables; + if (newCfg.maxTables < pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables); + terrno = TSDB_CODE_INVALID_OPTION; + } + } + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); newCfg.daysToKeep = daysToKeep; - } - + } + + if (daysToKeep1 > 0 && daysToKeep1 != pDb->cfg.daysToKeep1) { + mTrace("db:%s, daysToKeep1:%d change to %d", pDb->name, pDb->cfg.daysToKeep1, daysToKeep1); + newCfg.daysToKeep1 = daysToKeep1; + } + + if (daysToKeep2 > 0 && daysToKeep2 != pDb->cfg.daysToKeep2) { + mTrace("db:%s, daysToKeep2:%d change to %d", pDb->name, pDb->cfg.daysToKeep2, daysToKeep2); + newCfg.daysToKeep2 = daysToKeep2; + } + + if (compression > 0 && compression != pDb->cfg.compression) { + mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression); + newCfg.compression = compression; + } + if (replications > 0 && replications != pDb->cfg.replications) { - mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); - if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); - terrno = TSDB_CODE_INVALID_OPTION; - } + mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications); newCfg.replications = replications; } - - if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { - mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); - if (maxTables < TSDB_MIN_TABLES_PER_VNODE || maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); - terrno = TSDB_CODE_INVALID_OPTION; - } - if (maxTables < pDb->cfg.maxTables) { - mError("invalid db option tables: %d should larger than original:%d", maxTables, pDb->cfg.maxTables); - terrno = TSDB_CODE_INVALID_OPTION; - } - newCfg.maxTables = maxTables; + + if (replications > mgmtGetDnodesNum()) { + mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications); + terrno = TSDB_CODE_NO_ENOUGH_DNODES; + } + + if (pDb->cfg.replications - replications >= 2) { + mError("db:%s, replica number can't change from 3 to 1", pDb->name, replications); + terrno = TSDB_CODE_INVALID_OPTION; } return newCfg; @@ -731,8 +790,16 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return terrno; } + int32_t code = mgmtCheckDbCfg(&newCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t oldReplica = pDb->cfg.replications; + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; + pDb->cfgVersion++; SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, @@ -745,7 +812,20 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return TSDB_CODE_SDB_ERROR; } } - + + void *pNode = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pNode = mgmtGetNextVgroup(pNode, &pVgroup); + if (pVgroup == NULL) break; + mgmtSendCreateVgroupMsg(pVgroup, NULL); + mgmtDecVgroupRef(pVgroup); + } + + if (oldReplica != pDb->cfg.replications) { + balanceNotify(); + } + return TSDB_CODE_SUCCESS; } @@ -773,16 +853,6 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { return; } - SVgObj *pVgroup = pDb->pHead; - if (pVgroup != NULL) { - mPrint("vgroup:%d, will be altered", pVgroup->vgId); - SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); - newMsg->ahandle = pVgroup; - newMsg->expected = pVgroup->numOfVnodes; - mgmtAlterVgroup(pVgroup, newMsg); - return; - } - mTrace("db:%s, all vgroups is altered", pDb->name); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 8afeadeb0c..97360e49ea 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -42,7 +42,7 @@ int32_t tsAccessSquence = 0; extern void * tsMnodeSdb; extern void * tsVgroupSdb; -static int32_t mgmtCreateDnode(uint32_t ip); +static int32_t mgmtCreateDnode(char *ep); static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg); static void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg); @@ -68,12 +68,6 @@ static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) { pDnode->status = TAOS_DN_STATUS_OFFLINE; } - pDnode->mnodeShellPort = tsMnodeShellPort; - pDnode->mnodeDnodePort = tsMnodeDnodePort; - pDnode->dnodeShellPort = tsDnodeShellPort; - pDnode->dnodeMnodePort = tsDnodeMnodePort; - pDnode->syncPort = tsSyncPort; - return TSDB_CODE_SUCCESS; } @@ -144,9 +138,8 @@ static int32_t mgmtDnodeActionDecode(SSdbOper *pOper) { static int32_t mgmtDnodeActionRestored() { int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { - uint32_t ip = inet_addr(tsPrivateIp); - mgmtCreateDnode(ip); - SDnodeObj *pDnode = mgmtGetDnodeByIp(ip); + mgmtCreateDnode(tsLocalEp); + SDnodeObj *pDnode = mgmtGetDnodeByIp(tsLocalEp); mgmtAddMnode(pDnode->dnodeId); mgmtDecDnodeRef(pDnode); } @@ -214,14 +207,14 @@ void *mgmtGetDnode(int32_t dnodeId) { return sdbGetRow(tsDnodeSdb, &dnodeId); } -void *mgmtGetDnodeByIp(uint32_t ip) { +void *mgmtGetDnodeByIp(char *ep) { SDnodeObj *pDnode = NULL; void * pNode = NULL; while (1) { pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode); if (pDnode == NULL) break; - if (ip == pDnode->privateIp) { + if (strcmp(ep, pDnode->dnodeEp) == 0) { return pDnode; } mgmtDecDnodeRef(pDnode); @@ -253,19 +246,18 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMCfgDnodeMsg *pCmCfgDnode = pMsg->pCont; - if (pCmCfgDnode->ip[0] == 0) { - strcpy(pCmCfgDnode->ip, tsPrivateIp); + if (pCmCfgDnode->ep[0] == 0) { + strcpy(pCmCfgDnode->ep, tsLocalEp); } else { - strcpy(pCmCfgDnode->ip, pCmCfgDnode->ip); + strcpy(pCmCfgDnode->ep, pCmCfgDnode->ep); } - uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip); if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { - SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCmCfgDnode->ep); SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); - strcpy(pMdCfgDnode->ip, pCmCfgDnode->ip); + strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); strcpy(pMdCfgDnode->config, pCmCfgDnode->config); SRpcMsg rpcMdCfgDnodeMsg = { .handle = 0, @@ -279,7 +271,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { } if (rpcRsp.code == TSDB_CODE_SUCCESS) { - mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ip, pMsg->pUser->user); + mPrint("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); } rpcSendResponse(&rpcRsp); @@ -292,8 +284,6 @@ static void mgmtProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { SDMStatusMsg *pStatus = rpcMsg->pCont; pStatus->dnodeId = htonl(pStatus->dnodeId); - pStatus->privateIp = htonl(pStatus->privateIp); - pStatus->publicIp = htonl(pStatus->publicIp); pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->lastReboot = htonl(pStatus->lastReboot); pStatus->numOfCores = htons(pStatus->numOfCores); @@ -308,23 +298,21 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { - pDnode = mgmtGetDnodeByIp(pStatus->privateIp); + pDnode = mgmtGetDnodeByIp(pStatus->dnodeEp); if (pDnode == NULL) { - mTrace("dnode not created, privateIp:%s", taosIpStr(pStatus->privateIp)); + mTrace("dnode %s not created", pStatus->dnodeEp); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); return; } } else { pDnode = mgmtGetDnode(pStatus->dnodeId); if (pDnode == NULL) { - mError("dnode:%d, not exist, privateIp:%s", pStatus->dnodeId, taosIpStr(pStatus->privateIp)); + mError("dnode id:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_DNODE_NOT_EXIST); return; } } - - pDnode->privateIp = pStatus->privateIp; - pDnode->publicIp = pStatus->publicIp; + pDnode->lastReboot = pStatus->lastReboot; pDnode->numOfCores = pStatus->numOfCores; pDnode->diskAvailable = pStatus->diskAvailable; @@ -334,7 +322,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { pDnode->lastAccess = tsAccessSquence; if (pStatus->dnodeId == 0) { - mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName); + mTrace("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp); } else { //mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); } @@ -343,10 +331,11 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); + pVload->cfgVersion = htonl(pVload->cfgVersion); SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId); if (pVgroup == NULL) { - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp); mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId); mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); } else { @@ -390,25 +379,24 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { rpcSendResponse(&rpcRsp); } -static int32_t mgmtCreateDnode(uint32_t ip) { +static int32_t mgmtCreateDnode(char *ep) { int32_t grantCode = grantCheck(TSDB_GRANT_DNODE); if (grantCode != TSDB_CODE_SUCCESS) { return grantCode; } - SDnodeObj *pDnode = mgmtGetDnodeByIp(ip); + SDnodeObj *pDnode = mgmtGetDnodeByIp(ep); if (pDnode != NULL) { - mError("dnode:%d is alredy exist, ip:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp)); + mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); return TSDB_CODE_DNODE_ALREADY_EXIST; } pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj)); - pDnode->privateIp = ip; - pDnode->publicIp = ip; pDnode->createdTime = taosGetTimestampMs(); pDnode->status = TAOS_DN_STATUS_OFFLINE; pDnode->totalVnodes = TSDB_INVALID_VNODE_NUM; - sprintf(pDnode->dnodeName, "n%d", sdbGetId(tsDnodeSdb) + 1); + strcpy(pDnode->dnodeEp, ep); + taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort); SSdbOper oper = { .type = SDB_OPER_GLOBAL, @@ -445,15 +433,15 @@ int32_t mgmtDropDnode(SDnodeObj *pDnode) { return code; } -static int32_t mgmtDropDnodeByIp(uint32_t ip) { - SDnodeObj *pDnode = mgmtGetDnodeByIp(ip); +static int32_t mgmtDropDnodeByIp(char *ep) { + SDnodeObj *pDnode = mgmtGetDnodeByIp(ep); if (pDnode == NULL) { - mError("dnode:%s, is not exist", taosIpStr(ip)); + mError("dnode:%s, is not exist", ep); return TSDB_CODE_DNODE_NOT_EXIST; } - if (pDnode->privateIp == dnodeGetMnodeMasteIp()) { - mError("dnode:%d, can't drop dnode which is master", pDnode->dnodeId); + if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) { + mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep); return TSDB_CODE_NO_REMOVE_MASTER; } @@ -472,13 +460,12 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) { if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { - uint32_t ip = inet_addr(pCreate->ip); - rpcRsp.code = mgmtCreateDnode(ip); + rpcRsp.code = mgmtCreateDnode(pCreate->ep); if (rpcRsp.code == TSDB_CODE_SUCCESS) { - SDnodeObj *pDnode = mgmtGetDnodeByIp(ip); - mLPrint("dnode:%d, ip:%s is created by %s", pDnode->dnodeId, pCreate->ip, pMsg->pUser->user); + SDnodeObj *pDnode = mgmtGetDnodeByIp(pCreate->ep); + mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user); } else { - mError("failed to create dnode:%s, reason:%s", pCreate->ip, tstrerror(rpcRsp.code)); + mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(rpcRsp.code)); } } rpcSendResponse(&rpcRsp); @@ -489,15 +476,15 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMDropDnodeMsg *pDrop = pMsg->pCont; + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { - uint32_t ip = inet_addr(pDrop->ip); - rpcRsp.code = mgmtDropDnodeByIp(ip); + rpcRsp.code = mgmtDropDnodeByIp(pDrop->ep); if (rpcRsp.code == TSDB_CODE_SUCCESS) { - mLPrint("dnode:%s is dropped by %s", pDrop->ip, pMsg->pUser->user); + mLPrint("dnode:%s is dropped by %s", pDrop->ep, pMsg->pUser->user); } else { - mError("failed to drop dnode:%s, reason:%s", pDrop->ip, tstrerror(rpcRsp.code)); + mError("failed to drop dnode:%s, reason:%s", pDrop->ep, tstrerror(rpcRsp.code)); } } @@ -522,15 +509,9 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 16; + pShow->bytes[cols] = 40; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "private ip"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 16; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public ip"); + strcpy(pSchema[cols].name, "end point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -580,7 +561,6 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi int32_t cols = 0; SDnodeObj *pDnode = NULL; char *pWrite; - char ipstr[32]; while (numOfRows < rows) { pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode); @@ -592,14 +572,8 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi *(int16_t *)pWrite = pDnode->dnodeId; cols++; - tinet_ntoa(ipstr, pDnode->privateIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); - cols++; - - tinet_ntoa(ipstr, pDnode->publicIp); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); + strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -651,9 +625,9 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 16; + pShow->bytes[cols] = 40; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "ip"); + strcpy(pSchema[cols].name, "end point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -701,10 +675,8 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo *(int16_t *)pWrite = pDnode->dnodeId; cols++; - char ipstr[20]; - tinet_ntoa(ipstr, pDnode->privateIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); + strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -864,8 +836,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo SDnodeObj *pDnode = NULL; if (pShow->payloadLen > 0 ) { - uint32_t ip = ip2uint(pShow->payload); - pDnode = mgmtGetDnodeByIp(ip); + pDnode = mgmtGetDnodeByIp(pShow->payload); } else { mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode); } diff --git a/src/mnode/src/mgmtMain.c b/src/mnode/src/mgmtMain.c index e01b1e7756..437563a007 100644 --- a/src/mnode/src/mgmtMain.c +++ b/src/mnode/src/mgmtMain.c @@ -130,7 +130,7 @@ int32_t mgmtInitSystem() { struct stat dirstat; bool fileExist = (stat(tsMnodeDir, &dirstat) == 0); - bool asMaster = (strcmp(tsMasterIp, tsPrivateIp) == 0); + bool asMaster = (strcmp(tsMaster, tsLocalEp) == 0); if (asMaster || fileExist) { if (mgmtStartSystem() != 0) { diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index e8b1239380..ccc01fc304 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -171,25 +171,21 @@ char *mgmtGetMnodeRoleStr(int32_t role) { } } -void mgmtGetMnodeIpSet(SRpcIpSet *ipSet, bool usePublicIp) { +void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) { void *pNode = NULL; while (1) { SMnodeObj *pMnode = NULL; pNode = mgmtGetNextMnode(pNode, &pMnode); if (pMnode == NULL) break; - if (usePublicIp) { - ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->publicIp); - } else { - ipSet->ip[ipSet->numOfIps] = htonl(pMnode->pDnode->privateIp); - } + strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn); + ipSet->port[ipSet->numOfIps] = htons(pMnode->pDnode->dnodePort); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { ipSet->inUse = ipSet->numOfIps; } ipSet->numOfIps++; - ipSet->port = htons(pMnode->pDnode->mnodeShellPort); mgmtReleaseMnode(pMnode); } @@ -207,10 +203,7 @@ void mgmtGetMnodeInfos(void *param) { if (pMnode == NULL) break; mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId); - mnodes->nodeInfos[index].nodeIp = htonl(pMnode->pDnode->privateIp); - mnodes->nodeInfos[index].nodePort = htons(pMnode->pDnode->mnodeDnodePort); - mnodes->nodeInfos[index].syncPort = htons(pMnode->pDnode->syncPort); - strcpy(mnodes->nodeInfos[index].nodeName, pMnode->pDnode->dnodeName); + strcpy(mnodes->nodeInfos[index].nodeEp, pMnode->pDnode->dnodeEp); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { mnodes->inUse = index; } @@ -282,15 +275,9 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 16; + pShow->bytes[cols] = 40; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "private ip"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 16; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "public ip"); + strcpy(pSchema[cols].name, "end point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -327,7 +314,6 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi int32_t cols = 0; SMnodeObj *pMnode = NULL; char *pWrite; - char ipstr[32]; while (numOfRows < rows) { pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode); @@ -339,14 +325,8 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi *(int16_t *)pWrite = pMnode->mnodeId; cols++; - tinet_ntoa(ipstr, pMnode->pDnode->privateIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); - cols++; - - tinet_ntoa(ipstr, pMnode->pDnode->publicIp); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); + strncpy(pWrite, pMnode->pDnode->dnodeEp, pShow->bytes[cols]-1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 7359f95b61..25e36af6a0 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -231,8 +231,8 @@ void sdbUpdateSync() { for (int32_t i = 0; i < mnodes->nodeNum; ++i) { SDMMnodeInfo *node = &mnodes->nodeInfos[i]; syncCfg.nodeInfo[i].nodeId = node->nodeId; - syncCfg.nodeInfo[i].nodeIp = node->nodeIp; - strcpy(syncCfg.nodeInfo[i].name, node->nodeName); + taosGetFqdnPortFromEp(node->nodeEp, syncCfg.nodeInfo[i].nodeFqdn, &syncCfg.nodeInfo[i].nodePort); + syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; index++; } @@ -244,8 +244,8 @@ void sdbUpdateSync() { if (pMnode == NULL) break; syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId; - syncCfg.nodeInfo[index].nodeIp = pMnode->pDnode->privateIp; - strcpy(syncCfg.nodeInfo[index].name, pMnode->pDnode->dnodeName); + syncCfg.nodeInfo[index].nodePort = pMnode->pDnode->dnodePort + TSDB_PORT_SYNC; + strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp); index++; mgmtReleaseMnode(pMnode); @@ -253,7 +253,8 @@ void sdbUpdateSync() { } syncCfg.replica = index; - syncCfg.arbitratorIp = syncCfg.nodeInfo[0].nodeIp; + syncCfg.arbitratorPort = syncCfg.nodeInfo[0].nodePort; + strcpy(syncCfg.arbitratorFqdn, syncCfg.nodeInfo[0].nodeFqdn); if (syncCfg.replica == 1) { syncCfg.quorum = 1; } else { @@ -271,10 +272,9 @@ void sdbUpdateSync() { if (!hasThisDnode) return; if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return; - sdbPrint("work as mnode, replica:%d arbitratorIp:%s", syncCfg.replica, taosIpStr(syncCfg.arbitratorIp)); + sdbPrint("work as mnode, replica:%d arbitrator:%s", syncCfg.replica, syncCfg.arbitratorFqdn); for (int32_t i = 0; i < syncCfg.replica; ++i) { - sdbPrint("mnode:%d, ip:%s name:%s", syncCfg.nodeInfo[i].nodeId, taosIpStr(syncCfg.nodeInfo[i].nodeIp), - syncCfg.nodeInfo[i].name); + sdbPrint("mnode:%d, ip:%s", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn); } SSyncInfo syncInfo; diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index f2545f43db..48066ea8a5 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -72,7 +72,6 @@ int32_t mgmtInitShell() { } SRpcInit rpcInit = {0}; - rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; rpcInit.localPort = tsMnodeShellPort; rpcInit.label = "MND-shell"; rpcInit.numOfThreads = numOfThreads; @@ -148,14 +147,12 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { if (!sdbIsMaster()) { SRpcConnInfo connInfo; rpcGetConnInfo(rpcMsg->handle, &connInfo); - bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); SRpcIpSet ipSet = {0}; - ipSet.port = tsMnodeShellPort; - dnodeGetMnodeIpSet(&ipSet, usePublicIp); + mgmtGetMnodeIpSet(&ipSet); mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); for (int32_t i = 0; i < ipSet.numOfIps; ++i) { - mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + mTrace("index:%d ip:%s:%d", i, ipSet.fqdn[i], ipSet.port[i]); } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); @@ -343,7 +340,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { return; } - mgmtGetMnodeIpSet(&pHBRsp->ipList, pMsg->usePublicIp); + mgmtGetMnodeIpSet(&pHBRsp->ipList); /* * TODO @@ -429,7 +426,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - mgmtGetMnodeIpSet(&pConnectRsp->ipList, pMsg->usePublicIp); + mgmtGetMnodeIpSet(&pConnectRsp->ipList); connect_over: rpcRsp.code = code; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 0edb119724..7c45460cfd 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -74,10 +74,13 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg); static void mgmtProcessTableMetaMsg(SQueuedMsg *queueMsg); static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg); static void mgmtGetChildTableMeta(SQueuedMsg *pMsg); +static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg); static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); +static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); + static void mgmtDestroyChildTable(SChildTableObj *pTable) { tfree(pTable->schema); tfree(pTable->sql); @@ -610,9 +613,12 @@ static void mgmtExtractTableName(char* tableId, char* name) { static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; - pMsg->pTable = mgmtGetTable(pCreate->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pCreate->tableId); if (pMsg->pTable != NULL && pMsg->retry == 0) { - if (pCreate->igExists) { + if (pCreate->getMeta) { + mTrace("table:%s, continue to get meta", pCreate->tableId); + mgmtProcessTableMetaMsg(pMsg); + } else if (pCreate->igExists) { mTrace("table:%s, is already exist", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } else { @@ -622,7 +628,7 @@ static void mgmtProcessCreateTableMsg(SQueuedMsg *pMsg) { return; } - pMsg->pDb = mgmtGetDb(pCreate->db); + if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pCreate->db); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to create, db not selected", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); @@ -679,15 +685,21 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { SCMTableInfoMsg *pInfo = pMsg->pCont; mTrace("table:%s, table meta msg is received from thandle:%p", pInfo->tableId, pMsg->thandle); - pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDbByTableId(pInfo->tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("table:%s, failed to get table meta, db not selected", pInfo->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_DB_NOT_SELECTED); return; } + if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId); if (pMsg->pTable == NULL) { - mgmtGetChildTableMeta(pMsg); + if (htons(pInfo->createFlag) != 1) { + mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + } else { + mgmtAutoCreateChildTable(pMsg); + } } else { if (pMsg->pTable->type != TSDB_SUPER_TABLE) { mgmtGetChildTableMeta(pMsg); @@ -756,8 +768,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTables != 0) { - mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); + mgmtDropAllChildTablesInStable(pStable); + for (int32_t vg = 0; vg < pStable->vgLen; ++vg) { + int32_t vgId = pStable->vgList[vg]; + if (vgId == 0) break; + + SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); + pDrop->vgId = htonl(vgId); + pDrop->uid = htobe64(pStable->uid); + mgmtExtractTableName(pStable->info.tableId, pDrop->tableId); + + SVgObj *pVgroup = mgmtGetVgroup(vgId); + if (pVgroup != NULL) { + SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + mgmtSendMsgToDnode(&ipSet, &rpcMsg); + mgmtDecVgroupRef(pVgroup); + } + } + //mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); + //mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOper oper = { .type = SDB_OPER_GLOBAL, @@ -783,31 +813,33 @@ static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *t static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add tag, too many tags", pStable->info.tableId); + return TSDB_CODE_TOO_MANY_TAGS; } - // check if schemas have the same name - for (int32_t i = 1; i < ntags; i++) { - for (int32_t j = 0; j < i; j++) { - if (strcasecmp(schema[i].name, schema[j].name) == 0) { - return TSDB_CODE_APP_ERROR; - } + for (int32_t i = 0; i < ntags; i++) { + if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ntags); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ntags), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ntags); + memcpy(pStable->schema + pStable->numOfColumns + pStable->numOfTags, schema, sizeof(SSchema) * ntags); - SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); + SSchema *tschema = (SSchema *)(pStable->schema + pStable->numOfColumns + pStable->numOfTags); for (int32_t i = 0; i < ntags; i++) { tschema[i].colId = pStable->nextColId++; } - pStable->numOfColumns += ntags; + pStable->numOfTags += ntags; pStable->sversion++; SSdbOper oper = { @@ -822,25 +854,22 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); + mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName); - if (col <= 0 || col >= pStable->numOfTags) { - return TSDB_CODE_APP_ERROR; + if (col < 0) { + mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName); + return TSDB_CODE_TAG_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); - + memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1, + sizeof(SSchema) * (pStable->numOfTags - col - 1)); pStable->numOfTags--; pStable->sversion++; - int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); - pStable->schema = realloc(pStable->schema, schemaSize); - SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, @@ -853,27 +882,29 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to drop tag %s", pStable->info.tableId, tagName); + mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { - // Tag name does not exist - mError("table:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); - return TSDB_CODE_INVALID_MSG_TYPE; + mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); + return TSDB_CODE_TAG_NOT_EXIST; } // int32_t rowSize = 0; uint32_t len = strlen(newTagName); + if (len >= TSDB_COL_NAME_LEN) { + return TSDB_CODE_COL_NAME_TOO_LONG; + } - if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { - return TSDB_CODE_APP_ERROR; + if (mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { + return TSDB_CODE_TAG_ALREAY_EXIST; } // update - SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema)); + SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); SSdbOper oper = { @@ -888,15 +919,15 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); + mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { SSchema *schema = (SSchema *) pStable->schema; - for (int32_t i = 0; i < pStable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pStable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -905,21 +936,28 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ncols); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ncols), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ncols); + memmove(pStable->schema + pStable->numOfColumns + ncols, pStable->schema + pStable->numOfColumns, + sizeof(SSchema) * pStable->numOfTags); + memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols); SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { @@ -947,17 +985,18 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column", pStable->info.tableId); + mPrint("stable %s, succeed to add column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) { int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), + memmove(pStable->schema + col, pStable->schema + col + 1, sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); pStable->numOfColumns--; @@ -984,7 +1023,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to delete column", pStable->info.tableId); + mPrint("stable %s, succeed to delete column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -1173,47 +1212,64 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { SCMSTableVgroupMsg *pInfo = pMsg->pCont; - SSuperTableObj *pTable = mgmtGetSuperTable(pInfo->tableId); - - pMsg->pTable = (STableObj *)pTable; - if (pMsg->pTable == NULL) { - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } - - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * pTable->vgLen; - SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen); + int32_t numOfTable = htonl(pInfo->numOfTables); + + char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg); + SCMSTableVgroupRspMsg *pRsp = NULL; + + // todo set the initial size to be 10, fix me + int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable; + + pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } - int32_t vg = 0; - for (; vg < pTable->vgLen; ++vg) { - int32_t vgId = pTable->vgList[vg]; - if (vgId == 0) break; - - SVgObj *pVgroup = mgmtGetVgroup(vgId); - if (pVgroup == NULL) break; - - pRsp->vgroups[vg].vgId = htonl(vgId); - for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { - SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; - if (pDnode == NULL) break; - - pRsp->vgroups[vg].ipAddr[vn].ip = htonl(pDnode->privateIp); - pRsp->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort); - pRsp->vgroups[vg].numOfIps++; + pRsp->numOfTables = htonl(numOfTable); + char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg); + + for(int32_t i = 0; i < numOfTable; ++i) { + SSuperTableObj *pTable = mgmtGetSuperTable(name); + + pMsg->pTable = (STableObj *)pTable; + if (pMsg->pTable == NULL) { + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); + return; } - - mgmtDecVgroupRef(pVgroup); + + SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg; + + int32_t vg = 0; + for (; vg < pTable->vgLen; ++vg) { + int32_t vgId = pTable->vgList[vg]; + if (vgId == 0) break; + + SVgObj *vgItem = mgmtGetVgroup(vgId); + if (vgItem == NULL) break; + + pVgroup->vgroups[vg].vgId = htonl(vgId); + for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) { + SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode; + if (pDnode == NULL) break; + + pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort); + pVgroup->vgroups[vg].numOfIps++; + } + + mgmtDecVgroupRef(vgItem); + } + + pVgroup->numOfVgroups = htonl(vg); + + // one table is done, try the next table + msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo); } - pRsp->numOfVgroups = htonl(vg); SRpcMsg rpcRsp = {0}; rpcRsp.handle = pMsg->thandle; rpcRsp.pCont = pRsp; - rpcRsp.contLen = sizeof(SCMSTableVgroupRspMsg) + sizeof(SCMVgroupInfo) * vg; + rpcRsp.contLen = msg - (char*) pRsp; rpcSendResponse(&rpcRsp); } @@ -1472,9 +1528,9 @@ static int32_t mgmtModifyChildTableTagValue(SChildTableObj *pTable, char *tagNam static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; - for (int32_t i = 0; i < pTable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pTable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -1483,21 +1539,23 @@ static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colN static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); - memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); + memcpy(pTable->schema + pTable->numOfColumns, schema, sizeof(SSchema) * ncols); - SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); + SSchema *tschema = (SSchema *) (pTable->schema + pTable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { tschema[i].colId = pTable->nextColId++; } @@ -1507,7 +1565,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { - pAcct->acctInfo.numOfTimeSeries += ncols; + pAcct->acctInfo.numOfTimeSeries += ncols; mgmtDecAcctRef(pAcct); } @@ -1529,13 +1587,12 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) { int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pTable->numOfColumns - col - 1)); - + memmove(pTable->schema + col, pTable->schema + col + 1, sizeof(SSchema) * (pTable->numOfColumns - col - 1)); pTable->numOfColumns--; pTable->sversion++; @@ -1557,7 +1614,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column %s", pTable->info.tableId, colName); + mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); return TSDB_CODE_SUCCESS; } @@ -1577,7 +1634,6 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pT static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { SDbObj *pDb = pMsg->pDb; SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; - int8_t usePublicIp = pMsg->usePublicIp; pMeta->uid = htobe64(pTable->uid); pMeta->sid = htonl(pTable->sid); @@ -1607,13 +1663,8 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SDnodeObj *pDnode = mgmtGetDnode(pVgroup->vnodeGid[i].dnodeId); if (pDnode == NULL) break; - if (usePublicIp) { - pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->publicIp); - pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort); - } else { - pMeta->vgroup.ipAddr[i].ip = htonl(pDnode->privateIp); - pMeta->vgroup.ipAddr[i].port = htonl(tsDnodeShellPort); - } + strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn); + pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); pMeta->vgroup.numOfIps++; mgmtDecDnodeRef(pDnode); } @@ -1624,42 +1675,34 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { return TSDB_CODE_SUCCESS; } -void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; +static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) { SCMTableInfoMsg *pInfo = pMsg->pCont; - - if (pTable == NULL) { - if (htons(pInfo->createFlag) != 1) { - mError("table:%s, failed to get table meta, table not exist", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE); - return; - } else { - //TODO: on demand create table from super table if table does not exists - int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); - SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); - if (pCreateMsg == NULL) { - mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); - return; - } - memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); - strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId)); - - SQueuedMsg *newMsg = malloc(sizeof(SQueuedMsg)); - memcpy(newMsg, pMsg, sizeof(SQueuedMsg)); - pMsg->pCont = NULL; - - newMsg->ahandle = newMsg->pCont; - newMsg->pCont = pCreateMsg; - mTrace("table:%s, start to create in demand", pInfo->tableId); - mgmtAddToShellQueue(newMsg); - return; - } + int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(STagData); + SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); + if (pCreateMsg == NULL) { + mError("table:%s, failed to create table while get meta info, no enough memory", pInfo->tableId); + mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); + return; } + strncpy(pCreateMsg->tableId, pInfo->tableId, tListLen(pInfo->tableId)); + strcpy(pCreateMsg->db, pMsg->pDb->name); + pCreateMsg->igExists = 1; + pCreateMsg->getMeta = 1; + memcpy(pCreateMsg->schema, pInfo->tags, sizeof(STagData)); + + SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); + pMsg->pCont = newMsg->pCont; + newMsg->pCont = pCreateMsg; + + mTrace("table:%s, start to create on demand", pInfo->tableId); + mgmtAddToShellQueue(newMsg); +} + +static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); if (pMeta == NULL) { - mError("table:%s, failed to get table meta, no enough memory", pTable->info.tableId); + mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } @@ -1769,8 +1812,8 @@ static void mgmtProcessTableCfgMsg(SRpcMsg *rpcMsg) { mgmtDecTableRef(pTable); return; } - - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pCfg->dnode); + SDnodeObj *pDnode = mgmtGetDnode(pCfg->dnode); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp); SRpcMsg rpcRsp = { .handle = NULL, .pCont = pMDCreate, @@ -2078,7 +2121,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { return; } - pAlter->numOfCols = htons(pAlter->numOfCols); + pAlter->type = htons(pAlter->type); + if (pAlter->numOfCols > 2) { mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index e053ea66f5..3143e6065f 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -162,7 +162,7 @@ static int32_t mgmtVgroupActionEncode(SSdbOper *pOper) { SVgObj *pVgroup = pOper->pObj; memcpy(pOper->rowData, pVgroup, tsVgUpdateSize); SVgObj *pTmpVgroup = pOper->rowData; - for (int32_t i = 0; i < TSDB_VNODES_SUPPORT; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; } @@ -260,7 +260,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo } if (!dnodeExist) { - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp); mError("vgroup:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId); mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); return; @@ -272,8 +272,9 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo pVgroup->pointsWritten = htobe64(pVload->pointsWritten); } - if (pVload->replica != pVgroup->numOfVnodes) { - mError("dnode:%d, vgroup:%d replica:%d not match with mgmt:%d", pDnode->dnodeId, pVload->vgId, pVload->replica, + if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) { + mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d", + pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, pVgroup->numOfVnodes); mgmtSendCreateVgroupMsg(pVgroup, NULL); } @@ -400,9 +401,9 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 16; + pShow->bytes[cols] = 40; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "ip"); + strcpy(pSchema[cols].name, "end point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -439,7 +440,6 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo SVgObj *pVgroup = NULL; int32_t maxReplica = 0; int32_t cols = 0; - char ipstr[20]; char * pWrite; SDbObj *pDb = mgmtGetDb(pShow->db); @@ -478,10 +478,10 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo SDnodeObj *pDnode = pVgroup->vnodeGid[i].pDnode; if (pDnode != NULL) { - tinet_ntoa(ipstr, pDnode->privateIp); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strcpy(pWrite, ipstr); + strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; strcpy(pWrite, mgmtGetMnodeRoleStr(pVgroup->vnodeGid[i].role)); cols++; @@ -535,23 +535,22 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); + pCfg->cfgVersion = htonl(pDb->cfgVersion); + pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); + pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(pDb->cfg.maxTables); - pCfg->maxCacheSize = htobe64(pDb->cfg.maxCacheSize); - pCfg->maxCacheSize = htobe64(-1); //TODO - pCfg->minRowsPerFileBlock = htonl(-1); //TODO - pCfg->maxRowsPerFileBlock = htonl(-1); //TODO pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); - pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); - pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); - pCfg->daysToKeep = htonl(-1); //TODO + pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); + pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); + pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock); + pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock); pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; - pCfg->compression = -1; - pCfg->wals = 3; pCfg->commitLog = pDb->cfg.commitLog; pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->wals = 3; pCfg->quorum = 1; SMDVnodeDesc *pNodes = pVnode->nodes; @@ -559,11 +558,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SDnodeObj *pDnode = pVgroup->vnodeGid[j].pDnode; if (pDnode != NULL) { pNodes[j].nodeId = htonl(pDnode->dnodeId); - pNodes[j].nodeIp = htonl(pDnode->privateIp); - strcpy(pNodes[j].nodeName, pDnode->dnodeName); - if (j == 0) { - pCfg->arbitratorIp = htonl(pDnode->privateIp); - } + strcpy(pNodes[j].nodeEp, pDnode->dnodeEp); } } @@ -574,21 +569,21 @@ SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup) { SRpcIpSet ipSet = { .numOfIps = pVgroup->numOfVnodes, .inUse = 0, - .port = tsDnodeMnodePort }; for (int i = 0; i < pVgroup->numOfVnodes; ++i) { - ipSet.ip[i] = pVgroup->vnodeGid[i].pDnode->privateIp; + strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); + ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEMNODE; } return ipSet; } -SRpcIpSet mgmtGetIpSetFromIp(uint32_t ip) { - SRpcIpSet ipSet = { - .ip[0] = ip, - .numOfIps = 1, - .inUse = 0, - .port = tsDnodeMnodePort - }; +SRpcIpSet mgmtGetIpSetFromIp(char *ep) { + SRpcIpSet ipSet; + + ipSet.numOfIps = 1; + ipSet.inUse = 0; + taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]); + ipSet.port[0] += TSDB_PORT_DNODEMNODE; return ipSet; } @@ -608,7 +603,7 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); } } @@ -674,7 +669,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->privateIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); } } @@ -737,7 +732,7 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_SUCCESS); - SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->privateIp); + SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp); mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL); } @@ -745,16 +740,14 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { void *pNode = NULL; void *pLastNode = NULL; int32_t numOfVgroups = 0; - int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name); - while (1) { pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; - if (strncmp(pDropDb->name, pVgroup->dbName, dbNameLen) == 0) { + if (pVgroup->pDb == pDropDb) { SSdbOper oper = { .type = SDB_OPER_LOCAL, .table = tsVgroupSdb, @@ -763,23 +756,11 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { sdbDeleteRow(&oper); pNode = pLastNode; numOfVgroups++; + mgmtSendDropVgroupMsg(pVgroup, NULL); } - mgmtSendDropVgroupMsg(pVgroup, NULL); mgmtDecVgroupRef(pVgroup); } mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } - -void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { - assert(ahandle != NULL); - - if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { - // TODO: - // mgmtSendAlterVgroupMsg(pVgroup, NULL); - } else { - mgmtAddToShellQueue(ahandle); - } -} - diff --git a/src/os/darwin/src/darwinEnv.c b/src/os/darwin/src/darwinEnv.c index d9152e673d..27d5a7c99b 100644 --- a/src/os/darwin/src/darwinEnv.c +++ b/src/os/darwin/src/darwinEnv.c @@ -23,8 +23,8 @@ void osInit() { strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(dataDir, "/var/lib/taos"); - strcpy(logDir, "~/TDengineLog"); - strcpy(scriptDir, "/etc/taos"); - strcpy(osName, "Darwin"); + strcpy(tsDataDir, "/var/lib/taos"); + strcpy(tsLogDir, "~/TDengineLog"); + strcpy(tsScriptDir, "/etc/taos"); + strcpy(tsOsName, "Darwin"); } diff --git a/src/os/linux/src/linuxEnv.c b/src/os/linux/src/linuxEnv.c index bc1a309c9e..0a9d81311a 100644 --- a/src/os/linux/src/linuxEnv.c +++ b/src/os/linux/src/linuxEnv.c @@ -23,8 +23,8 @@ void osInit() { strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(dataDir, "/var/lib/taos"); - strcpy(logDir, "/var/log/taos"); - strcpy(scriptDir, "/etc/taos"); - strcpy(osName, "Linux"); + strcpy(tsDataDir, "/var/lib/taos"); + strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsScriptDir, "/etc/taos"); + strcpy(tsOsName, "Linux"); } \ No newline at end of file diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index 54e24c28e5..98d80ded5d 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -290,11 +290,12 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { bool taosGetDisk() { struct statvfs info; const double unit = 1024 * 1024 * 1024; - + if (tscEmbedded) { - if (statvfs(dataDir, &info)) { - tsTotalDataDirGB = 0; - tsAvailDataDirGB = 0; + if (statvfs(tsDataDir, &info)) { + //tsTotalDataDirGB = 0; + //tsAvailDataDirGB = 0; + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); return false; } else { tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); @@ -302,9 +303,10 @@ bool taosGetDisk() { } } - if (statvfs(logDir, &info)) { - tsTotalLogDirGB = 0; - tsAvailLogDirGB = 0; + if (statvfs(tsLogDir, &info)) { + //tsTotalLogDirGB = 0; + //tsAvailLogDirGB = 0; + uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno)); return false; } else { tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); @@ -312,8 +314,9 @@ bool taosGetDisk() { } if (statvfs("/tmp", &info)) { - tsTotalTmpDirGB = 0; - tsAvailTmpDirGB = 0; + //tsTotalTmpDirGB = 0; + //tsAvailTmpDirGB = 0; + uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); return false; } else { tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); @@ -361,6 +364,8 @@ static bool taosGetCardName(char *ip, char *name) { static bool taosGetCardInfo(int64_t *bytes) { static char tsPublicCard[1000] = {0}; + static char tsPrivateIp[40]; + if (tsPublicCard[0] == 0) { if (!taosGetCardName(tsPrivateIp, tsPublicCard)) { uError("can't get card name from ip:%s", tsPrivateIp); diff --git a/src/os/windows/src/twinenv.c b/src/os/windows/src/twinenv.c index 2b18acdaab..840a8d9f7a 100644 --- a/src/os/windows/src/twinenv.c +++ b/src/os/windows/src/twinenv.c @@ -23,8 +23,8 @@ void osInit() { strcpy(tsVnodeDir, "C:/TDengine/data"); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(dataDir, "C:/TDengine/data"); - strcpy(logDir, "C:/TDengine/log"); - strcpy(scriptDir, "C:/TDengine/script"); - strcpy(osName, "Windows"); + strcpy(tsDataDir, "C:/TDengine/data"); + strcpy(tsLogDir, "C:/TDengine/log"); + strcpy(tsScriptDir, "C:/TDengine/script"); + strcpy(tsOsName, "Windows"); } \ No newline at end of file diff --git a/src/plugins/http/inc/httpHandle.h b/src/plugins/http/inc/httpHandle.h index a62dabc800..d8b3b8eade 100644 --- a/src/plugins/http/inc/httpHandle.h +++ b/src/plugins/http/inc/httpHandle.h @@ -210,7 +210,7 @@ typedef struct HttpThread { typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; - char serverIp[16]; + uint32_t serverIp; uint16_t serverPort; int cacheContext; int sessionExpire; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 0e2e7b3f85..d9507072de 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -199,7 +199,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { } else {} } - httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_INVALID_SQL, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 0d0bc3e0fd..45cc3fcfdf 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -239,7 +239,7 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_INVALID_SQL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:invalidsql, sqlObj:%p, error:%s", + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 355fdf2fd0..89dc11796f 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -48,7 +48,7 @@ int httpInitSystem() { memset(httpServer, 0, sizeof(HttpServer)); strcpy(httpServer->label, "rest"); - strcpy(httpServer->serverIp, tsHttpIp); + httpServer->serverIp = 0; httpServer->serverPort = tsHttpPort; httpServer->cacheContext = tsHttpCacheSessions; httpServer->sessionExpire = tsHttpSessionExpire; @@ -117,7 +117,7 @@ void httpCleanUpSystem() { httpPrint("http service cleanup"); httpStopSystem(); -#if 1 +#if 0 if (httpServer == NULL) { return; } diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index dade813d9c..a4f9f32050 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -68,7 +68,7 @@ typedef enum { typedef struct { void * conn; void * timer; - char privateIpStr[TSDB_IPv4ADDR_LEN]; + char ep[TSDB_FQDN_LEN]; int8_t cmdIndex; int8_t state; char sql[SQL_LENGTH]; @@ -112,14 +112,8 @@ static void monitorInitConn(void *para, void *unused) { monitorPrint("starting to initialize monitor service .."); tsMonitorConn.state = MONITOR_STATE_INITIALIZING; - if (tsMonitorConn.privateIpStr[0] == 0) { - strcpy(tsMonitorConn.privateIpStr, tsPrivateIp); - for (int32_t i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { - if (tsMonitorConn.privateIpStr[i] == '.') { - tsMonitorConn.privateIpStr[i] = '_'; - } - } - } + if (tsMonitorConn.ep[0] == 0) + strcpy(tsMonitorConn.ep, tsLocalEp); if (tsMonitorConn.conn == NULL) { taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn)); @@ -163,7 +157,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) { tsMonitorDbName, IP_LEN_STR + 1); } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, - tsMonitorConn.privateIpStr, tsMonitorDbName, tsPrivateIp); + tsMonitorConn.ep, tsMonitorDbName, tsLocalEp); } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct(ts timestamp " @@ -214,7 +208,7 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) { monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { - monitorPrint("dnode:%s is started", tsPrivateIp); + monitorPrint("dnode:%s is started", tsLocalEp); } tsMonitorConn.cmdIndex++; monitorInitDatabase(); @@ -346,7 +340,7 @@ static void monitorSaveSystemInfo() { int64_t ts = taosGetTimestampUs(); char * sql = tsMonitorConn.sql; - int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.privateIpStr, ts); + int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.ep, ts); pos += monitorBuildCpuSql(sql + pos); pos += monitorBuildMemorySql(sql + pos); @@ -414,7 +408,7 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { va_end(argpointer); if (len > max_length) len = max_length; - len += sprintf(sql + len, "', '%s')", tsPrivateIp); + len += sprintf(sql + len, "', '%s')", tsLocalEp); sql[len++] = 0; monitorTrace("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql); diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 4a7781d8c6..8698b3af82 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -91,9 +91,10 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res); uint8_t getBinaryExprOptr(SSQLToken *pToken); -SBuffer exprTreeToBinary(tExprNode* pExprTree); +void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); +void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromTableName(const char* tbnameCond); #ifdef __cplusplus diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 3f6376bb94..676be4f7be 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -108,17 +108,17 @@ enum { SECONDARY_STAGE_MERGE = 0x20u, }; -#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) -#define QUERY_IS_JOIN_QUERY(type) (((type)&TSDB_QUERY_TYPE_JOIN_QUERY) != 0) +#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) +#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) #define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) -#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) +#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) typedef struct SArithmeticSupport { SExprInfo *pArithExpr; - int32_t numOfCols; - SColumnInfo* colList; - int32_t offset; - char** data; + int32_t numOfCols; + SColumnInfo *colList; + int32_t offset; + char** data; } SArithmeticSupport; typedef struct SQLPreAggVal { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 1f72eaf487..e78f36ed31 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -31,6 +31,7 @@ #include "tskiplist.h" #include "queryLog.h" #include "tsdbMain.h" +#include "exception.h" /* * @@ -43,7 +44,6 @@ * */ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); -static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -427,7 +427,7 @@ void tSQLBinaryExprToString(tExprNode *pExpr, char *dst, int32_t *len) { static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); } -static void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { +void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { if (pNode == NULL) { return; } @@ -1015,104 +1015,116 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { } } -static void exprTreeToBinaryImpl(tExprNode* pExprTree, SBuffer* pBuf) { - tbufWrite(pBuf, &pExprTree->nodeType, sizeof(pExprTree->nodeType)); +static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { + tbufWriteUint8(bw, expr->nodeType); - if (pExprTree->nodeType == TSQL_NODE_VALUE) { - tVariant* pVal = pExprTree->pVal; + if (expr->nodeType == TSQL_NODE_VALUE) { + tVariant* pVal = expr->pVal; - tbufWrite(pBuf, &pVal->nType, sizeof(pVal->nType)); + tbufWriteUint32(bw, pVal->nType); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { - tbufWrite(pBuf, &pVal->nLen, sizeof(pVal->nLen)); - tbufWrite(pBuf, pVal->pz, pVal->nLen); + tbufWriteInt32(bw, pVal->nLen); + tbufWrite(bw, pVal->pz, pVal->nLen); } else { - tbufWrite(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + tbufWriteInt64(bw, pVal->i64Key); } - } else if (pExprTree->nodeType == TSQL_NODE_COL) { - SSchema* pSchema = pExprTree->pSchema; - tbufWrite(pBuf, &pSchema->colId, sizeof(pSchema->colId)); - tbufWrite(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); - tbufWrite(pBuf, &pSchema->type, sizeof(pSchema->type)); + } else if (expr->nodeType == TSQL_NODE_COL) { + SSchema* pSchema = expr->pSchema; + tbufWriteInt16(bw, pSchema->colId); + tbufWriteInt16(bw, pSchema->bytes); + tbufWriteUint8(bw, pSchema->type); + tbufWriteString(bw, pSchema->name); - int32_t len = strlen(pSchema->name); - tbufWriteStringLen(pBuf, pSchema->name, len); - - } else if (pExprTree->nodeType == TSQL_NODE_EXPR) { - tbufWrite(pBuf, &pExprTree->_node.optr, sizeof(pExprTree->_node.optr)); - tbufWrite(pBuf, &pExprTree->_node.hasPK, sizeof(pExprTree->_node.hasPK)); - - exprTreeToBinaryImpl(pExprTree->_node.pLeft, pBuf); - exprTreeToBinaryImpl(pExprTree->_node.pRight, pBuf); + } else if (expr->nodeType == TSQL_NODE_EXPR) { + tbufWriteUint8(bw, expr->_node.optr); + tbufWriteUint8(bw, expr->_node.hasPK); + exprTreeToBinaryImpl(bw, expr->_node.pLeft); + exprTreeToBinaryImpl(bw, expr->_node.pRight); } } -SBuffer exprTreeToBinary(tExprNode* pExprTree) { - SBuffer buf = {0}; - if (pExprTree == NULL) { - return buf; +void exprTreeToBinary(SBufferWriter* bw, tExprNode* expr) { + if (expr != NULL) { + exprTreeToBinaryImpl(bw, expr); } - - int32_t code = tbufBeginWrite(&buf); - if (code != 0) { - return buf; - } - - exprTreeToBinaryImpl(pExprTree, &buf); - return buf; } -static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) { - tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - pExpr->nodeType = tbufReadUint8(pBuf); +// TODO: these three functions should be made global +static void* exception_calloc(size_t nmemb, size_t size) { + void* p = calloc(nmemb, size); + if (p == NULL) { + THROW(TSDB_CODE_SERV_OUT_OF_MEMORY); + } + return p; +} + +static void* exception_malloc(size_t size) { + void* p = malloc(size); + if (p == NULL) { + THROW(TSDB_CODE_SERV_OUT_OF_MEMORY); + } + return p; +} + +static char* exception_strdup(const char* str) { + char* p = strdup(str); + if (p == NULL) { + THROW(TSDB_CODE_SERV_OUT_OF_MEMORY); + } + return p; +} + + +static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { + int32_t anchor = CLEANUP_GET_ANCHOR(); + + tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode)); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL); + + pExpr->nodeType = tbufReadUint8(br); if (pExpr->nodeType == TSQL_NODE_VALUE) { - tVariant* pVal = calloc(1, sizeof(tVariant)); - if (pVal == NULL) { - // TODO: - } + tVariant* pVal = exception_calloc(1, sizeof(tVariant)); pExpr->pVal = pVal; - pVal->nType = tbufReadUint32(pBuf); + pVal->nType = tbufReadUint32(br); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { - tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); + tbufReadToBuffer(br, &pVal->nLen, sizeof(pVal->nLen)); pVal->pz = calloc(1, pVal->nLen + 1); - tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); + tbufReadToBuffer(br, pVal->pz, pVal->nLen); } else { - pVal->i64Key = tbufReadInt64(pBuf); + pVal->i64Key = tbufReadInt64(br); } } else if (pExpr->nodeType == TSQL_NODE_COL) { - SSchema* pSchema = calloc(1, sizeof(SSchema)); - if (pSchema == NULL) { - // TODO: - } + SSchema* pSchema = exception_calloc(1, sizeof(SSchema)); pExpr->pSchema = pSchema; - pSchema->colId = tbufReadInt16(pBuf); - pSchema->bytes = tbufReadInt16(pBuf); - pSchema->type = tbufReadUint8(pBuf); - tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); + pSchema->colId = tbufReadInt16(br); + pSchema->bytes = tbufReadInt16(br); + pSchema->type = tbufReadUint8(br); + tbufReadToString(br, pSchema->name, TSDB_COL_NAME_LEN); } else if (pExpr->nodeType == TSQL_NODE_EXPR) { - pExpr->_node.optr = tbufReadUint8(pBuf); - pExpr->_node.hasPK = tbufReadUint8(pBuf); - pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf); - pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf); + pExpr->_node.optr = tbufReadUint8(br); + pExpr->_node.hasPK = tbufReadUint8(br); + pExpr->_node.pLeft = exprTreeFromBinaryImpl(br); + pExpr->_node.pRight = exprTreeFromBinaryImpl(br); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); } + CLEANUP_EXECUTE_TO(anchor, false); return pExpr; } -tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { +tExprNode* exprTreeFromBinary(const void* data, size_t size) { if (size == 0) { return NULL; } - SBuffer rbuf = {0}; - tbufBeginRead(&rbuf, pBuf, size); - return exprTreeFromBinaryImpl(&rbuf); + SBufferReader br = tbufInitReader(data, size, false); + return exprTreeFromBinaryImpl(&br); } tExprNode* exprTreeFromTableName(const char* tbnameCond) { @@ -1120,23 +1132,18 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { return NULL; } - tExprNode* expr = calloc(1, sizeof(tExprNode)); - if (expr == NULL) { - // TODO: - } + int32_t anchor = CLEANUP_GET_ANCHOR(); + + tExprNode* expr = exception_calloc(1, sizeof(tExprNode)); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + expr->nodeType = TSQL_NODE_EXPR; - tExprNode* left = calloc(1, sizeof(tExprNode)); - if (left == NULL) { - // TODO: - } + tExprNode* left = exception_calloc(1, sizeof(tExprNode)); expr->_node.pLeft = left; left->nodeType = TSQL_NODE_COL; - SSchema* pSchema = calloc(1, sizeof(SSchema)); - if (pSchema == NULL) { - // TODO: - } + SSchema* pSchema = exception_calloc(1, sizeof(SSchema)); left->pSchema = pSchema; pSchema->type = TSDB_DATA_TYPE_BINARY; @@ -1144,36 +1151,24 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { strcpy(pSchema->name, TSQL_TBNAME_L); pSchema->colId = -1; - tExprNode* right = calloc(1, sizeof(tExprNode)); - if (right == NULL) { - // TODO - } + tExprNode* right = exception_calloc(1, sizeof(tExprNode)); expr->_node.pRight = right; if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { right->nodeType = TSQL_NODE_VALUE; expr->_node.optr = TSDB_RELATION_LIKE; - tVariant* pVal = calloc(1, sizeof(tVariant)); - if (pVal == NULL) { - // TODO: - } + tVariant* pVal = exception_calloc(1, sizeof(tVariant)); right->pVal = pVal; - pVal->nType = TSDB_DATA_TYPE_BINARY; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; - pVal->pz = malloc(len); - if (pVal->pz == NULL) { - // TODO: - } + pVal->pz = exception_malloc(len); memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + pVal->nType = TSDB_DATA_TYPE_BINARY; pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { right->nodeType = TSQL_NODE_VALUE; expr->_node.optr = TSDB_RELATION_IN; - tVariant* pVal = calloc(1, sizeof(tVariant)); - if (pVal == NULL) { - // TODO: - } + tVariant* pVal = exception_calloc(1, sizeof(tVariant)); right->pVal = pVal; pVal->nType = TSDB_DATA_TYPE_ARRAY; pVal->arr = taosArrayInit(2, sizeof(char*)); @@ -1184,7 +1179,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { cond = e + 1; } else if (*e == ',') { size_t len = e - cond + 1; - char* p = malloc( len ); + char* p = exception_malloc( len ); memcpy(p, cond, len); p[len - 1] = 0; cond += len; @@ -1193,12 +1188,13 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { } if (*cond != 0) { - char* p = strdup( cond ); + char* p = exception_strdup( cond ); taosArrayPush(pVal->arr, &p); } taosArraySortString(pVal->arr); } + CLEANUP_EXECUTE_TO(anchor, false); return expr; } \ No newline at end of file diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e626de6243..9f21d0b370 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -55,8 +55,8 @@ typedef struct SPointInterpoSupporter { int32_t numOfCols; - char ** pPrevPoint; - char ** pNextPoint; + SArray* prev; + SArray* next; } SPointInterpoSupporter; typedef enum { @@ -1627,7 +1627,7 @@ static bool isFirstLastRowQuery(SQuery *pQuery) { return false; } -static bool notHasQueryTimeRange(SQuery *pQuery) { +static UNUSED_FUNC bool notHasQueryTimeRange(SQuery *pQuery) { return (pQuery->window.skey == 0 && pQuery->window.ekey == INT64_MAX && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->window.skey == INT64_MAX && pQuery->window.ekey == 0 && (!QUERY_IS_ASC_QUERY(pQuery))); } @@ -1950,7 +1950,7 @@ static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) { } } -static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { +static UNUSED_FUNC void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t index, char *data) { assert(pCtx->param[index].pz == NULL); int32_t len = 0; @@ -2015,6 +2015,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t * @param pInterpoRaw */ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupport) { +#if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2024,7 +2025,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI } int32_t count = 1; - TSKEY key = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; + TSKEY key = *(TSKEY *)pPointInterpSupport->next[0]; if (key == pQuery->window.skey) { // the queried timestamp has value, return it directly without interpolation @@ -2107,9 +2108,11 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI } } } +#endif } void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSupport) { +#if 0 if (isPointInterpoQuery(pQuery)) { pInterpoSupport->pPrevPoint = malloc(pQuery->numOfCols * POINTER_BYTES); pInterpoSupport->pNextPoint = malloc(pQuery->numOfCols * POINTER_BYTES); @@ -2136,9 +2139,11 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu offset += pQuery->colList[i].bytes; } } +#endif } void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { +#if 0 if (pPointInterpSupport->numOfCols <= 0 || pPointInterpSupport->pPrevPoint == NULL) { return; } @@ -2150,6 +2155,7 @@ void pointInterpSupporterDestroy(SPointInterpoSupporter *pPointInterpSupport) { tfree(pPointInterpSupport->pNextPoint); pPointInterpSupport->numOfCols = 0; +#endif } static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void *pMeterObj) { @@ -4223,7 +4229,11 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) .numOfCols = pQuery->numOfCols, }; - if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { + + // normal query setup the queryhandle here + if (isFirstLastRowQuery(pQuery)) { // in case of last_row query, invoke a different API. + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo); + } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); } @@ -4285,36 +4295,36 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SPointInterpoSupporter interpInfo = {0}; - pointInterpSupporterInit(pQuery, &interpInfo); +// SPointInterpoSupporter interpInfo = {0}; +// pointInterpSupporterInit(pQuery, &interpInfo); /* - * in case of last_row query without query range, we set the query timestamp to - * pMeterObj->lastKey. Otherwise, keep the initial query time range unchanged. + * in case of last_row query without query range, we set the query timestamp to be + * STable->lastKey. Otherwise, keep the initial query time range unchanged. */ - if (isFirstLastRowQuery(pQuery) && notHasQueryTimeRange(pQuery)) { - if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { - sem_post(&pQInfo->dataReady); - pointInterpSupporterDestroy(&interpInfo); - return TSDB_CODE_SUCCESS; - } - } +// if (isFirstLastRowQuery(pQuery)) { +// if (!normalizeUnBoundLastRowQuery(pQInfo, &interpInfo)) { +// sem_post(&pQInfo->dataReady); +// pointInterpSupporterDestroy(&interpInfo); +// return TSDB_CODE_SUCCESS; +// } +// } /* * here we set the value for before and after the specified time into the * parameter for interpolation query */ - pointInterpSupporterSetData(pQInfo, &interpInfo); - pointInterpSupporterDestroy(&interpInfo); +// pointInterpSupporterSetData(pQInfo, &interpInfo); +// pointInterpSupporterDestroy(&interpInfo); - int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, - pQuery->precision); - taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); +// int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, +// pQuery->precision); +// taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); // allocMemForInterpo(pQInfo, pQuery, pMeterObj); - if (!isPointInterpoQuery(pQuery)) { +// if (!isPointInterpoQuery(pQuery)) { // assert(pQuery->pos >= 0 && pQuery->slot >= 0); - } +// } // the pQuery->window.skey is changed during normalizedFirstQueryRange, so set the newest lastkey value pQuery->lastKey = pQuery->window.skey; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 6a78cfbe53..dee85ef630 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -550,11 +550,12 @@ tExprNode* createExpr2() { void exprSerializeTest1() { tExprNode* p1 = createExpr1(); - SBuffer buf = exprTreeToBinary(p1); + SBufferWriter bw = tbufInitWriter(NULL, false); + exprTreeToBinary(&bw, p1); - size_t size = tbufTell(&buf); + size_t size = tbufTell(&bw); ASSERT_TRUE(size > 0); - char* b = tbufGetData(&buf, false); + char* b = tbufGetData(&bw, false); tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); @@ -581,16 +582,17 @@ void exprSerializeTest1() { tExprTreeDestroy(&p1, nullptr); tExprTreeDestroy(&p2, nullptr); - tbufClose(&buf, false); + tbufClose(&bw); } void exprSerializeTest2() { tExprNode* p1 = createExpr2(); - SBuffer buf = exprTreeToBinary(p1); + SBufferWriter bw = tbufInitWriter(NULL, false); + exprTreeToBinary(&bw, p1); - size_t size = tbufTell(&buf); + size_t size = tbufTell(&bw); ASSERT_TRUE(size > 0); - char* b = tbufGetData(&buf, false); + char* b = tbufGetData(&bw, false); tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); @@ -625,7 +627,7 @@ void exprSerializeTest2() { tExprTreeDestroy(&p1, nullptr); tExprTreeDestroy(&p2, nullptr); - tbufClose(&buf, false); + tbufClose(&bw); } } // namespace TEST(testCase, astTest) { diff --git a/src/rpc/inc/rpcCache.h b/src/rpc/inc/rpcCache.h index 2a386c066e..3a996aab7c 100644 --- a/src/rpc/inc/rpcCache.h +++ b/src/rpc/inc/rpcCache.h @@ -22,8 +22,8 @@ extern "C" { void *rpcOpenConnCache(int maxSessions, void (*cleanFp)(void *), void *tmrCtrl, int64_t keepTimer); void rpcCloseConnCache(void *handle); -void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType); -void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType); +void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType); +void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType); #ifdef __cplusplus } diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h index 16972dbc7e..40fab00056 100644 --- a/src/rpc/inc/rpcTcp.h +++ b/src/rpc/inc/rpcTcp.h @@ -20,12 +20,12 @@ extern "C" { #endif -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); +void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); void taosCleanUpTcpServer(void *param); -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle); +void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); void taosCleanUpTcpClient(void *chandle); -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port); +void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); diff --git a/src/rpc/inc/rpcUdp.h b/src/rpc/inc/rpcUdp.h index a84f7c4a49..fd60f4a089 100644 --- a/src/rpc/inc/rpcUdp.h +++ b/src/rpc/inc/rpcUdp.h @@ -22,10 +22,10 @@ extern "C" { #include "taosdef.h" -void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int, void *fp, void *shandle); +void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle); void taosCleanUpUdpConnection(void *handle); int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle); -void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port); +void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 48d68c7511..ab9e679acf 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -23,7 +23,7 @@ #include "rpcCache.h" typedef struct SConnHash { - uint32_t ip; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; char connType; struct SConnHash *prev; @@ -46,7 +46,7 @@ typedef struct { int64_t *lockedBy; } SConnCache; -static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType); +static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType); static void rpcLockCache(int64_t *lockedBy); static void rpcUnlockCache(int64_t *lockedBy); static void rpcCleanConnCache(void *handle, void *tmrId); @@ -114,7 +114,7 @@ void rpcCloseConnCache(void *handle) { free(pCache); } -void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, int8_t connType) { +void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, int8_t connType) { int hash; SConnHash * pNode; SConnCache *pCache; @@ -125,9 +125,9 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i assert(pCache); assert(data); - hash = rpcHashConn(pCache, ip, port, connType); + hash = rpcHashConn(pCache, fqdn, port, connType); pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool); - pNode->ip = ip; + strcpy(pNode->fqdn, fqdn); pNode->port = port; pNode->connType = connType; pNode->data = data; @@ -147,12 +147,12 @@ void rpcAddConnIntoCache(void *handle, void *data, uint32_t ip, uint16_t port, i pCache->total++; - tTrace("%p ip:0x%x:%hu:%d:%d:%p added into cache, connections:%d", data, ip, port, connType, hash, pNode, pCache->count[hash]); + tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); return; } -void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connType) { +void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connType) { int hash; SConnHash * pNode; SConnCache *pCache; @@ -163,7 +163,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT uint64_t time = taosGetTimestampMs(); - hash = rpcHashConn(pCache, ip, port, connType); + hash = rpcHashConn(pCache, fqdn, port, connType); rpcLockCache(pCache->lockedBy+hash); pNode = pCache->connHashList[hash]; @@ -174,7 +174,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT break; } - if (pNode->ip == ip && pNode->port == port && pNode->connType == connType) break; + if (strcmp(pNode->fqdn, fqdn) == 0 && pNode->port == port && pNode->connType == connType) break; pNode = pNode->next; } @@ -201,7 +201,7 @@ void *rpcGetConnFromCache(void *handle, uint32_t ip, uint16_t port, int8_t connT rpcUnlockCache(pCache->lockedBy+hash); if (pData) { - tTrace("%p ip:0x%x:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, ip, port, connType, hash, pNode, pCache->count[hash]); + tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); } return pData; @@ -239,7 +239,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pNext = pNode->next; pCache->total--; pCache->count[hash]--; - tTrace("%p ip:0x%x:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->ip, pNode->port, pNode->connType, hash, pNode, + tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, pCache->count[hash]); taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); pNode = pNext; @@ -251,12 +251,16 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pCache->connHashList[hash] = NULL; } -static int rpcHashConn(void *handle, uint32_t ip, uint16_t port, int8_t connType) { +static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) { SConnCache *pCache = (SConnCache *)handle; int hash = 0; + char *temp = fqdn; + + while (*temp) { + hash += *temp; + ++temp; + } - hash = ip >> 16; - hash += (unsigned short)(ip & 0xFFFF); hash += port; hash += connType; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index dede169349..f616cd00b7 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -44,7 +44,6 @@ typedef struct { int sessions; // number of sessions allowed int numOfThreads; // number of threads to process incoming messages int idleTime; // milliseconds; - char localIp[TSDB_IPv4ADDR_LEN]; uint16_t localPort; int8_t connType; int index; // for UDP server only, round robin for multiple threads @@ -101,9 +100,8 @@ typedef struct SRpcConn { uint16_t localPort; // for UDP only uint32_t linkUid; // connection unique ID assigned by client uint32_t peerIp; // peer IP - uint32_t destIp; // server destination IP to handle NAT uint16_t peerPort; // peer port - char peerIpstr[TSDB_IPv4ADDR_LEN]; // peer IP string + char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string uint16_t tranId; // outgoing transcation ID, for build message uint16_t outTranId; // outgoing transcation ID uint16_t inTranId; // transcation ID for incoming msg @@ -140,7 +138,7 @@ int tsRpcOverhead; #define RPC_CONN_TCPC 3 #define RPC_CONN_TCP 2 -void *(*taosInitConn[])(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { +void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, @@ -161,7 +159,7 @@ int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *cha taosSendTcpData }; -void *(*taosOpenConn[])(void *shandle, void *thandle, char *ip, uint16_t port) = { +void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = { taosOpenUdpConnection, taosOpenUdpConnection, NULL, @@ -175,7 +173,7 @@ void (*taosCloseConn[])(void *chandle) = { taosCloseTcpConnection }; -static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType); +static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType); static void rpcCloseConn(void *thandle); static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext); static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc); @@ -217,7 +215,6 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; - if (pInit->localIp) strcpy(pRpc->localIp, pInit->localIp); pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; pRpc->sessions = pInit->sessions; @@ -229,13 +226,13 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; - pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(pRpc->localIp, pRpc->localPort, pRpc->label, + pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); - pRpc->udphandle = (*taosInitConn[pRpc->connType])(pRpc->localIp, pRpc->localPort, pRpc->label, + pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) { - tError("%s failed to init network, %s:%d", pRpc->label, pRpc->localIp, pRpc->localPort); + tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort); rpcClose(pRpc); return NULL; } @@ -457,7 +454,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { pInfo->clientIp = pConn->peerIp; pInfo->clientPort = pConn->peerPort; - pInfo->serverIp = pConn->destIp; + // pInfo->serverIp = pConn->destIp; strcpy(pInfo->user, pConn->user); return 0; @@ -490,27 +487,32 @@ static void rpcFreeMsg(void *msg) { } } -static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerIpStr, uint16_t peerPort, int8_t connType) { +static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; + uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); + if (peerIp == -1) { + tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); + return NULL; + } + pConn = rpcAllocateClientConn(pRpc); if (pConn) { - strcpy(pConn->peerIpstr, peerIpStr); - pConn->peerIp = inet_addr(peerIpStr); + strcpy(pConn->peerFqdn, peerFqdn); + pConn->peerIp = peerIp; pConn->peerPort = peerPort; strcpy(pConn->user, pRpc->user); pConn->connType = connType; if (taosOpenConn[connType]) { void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle; - pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIpstr, pConn->peerPort); + pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); if (pConn->chandle) { - tTrace("%s %p, rpc connection is set up, sid:%d id:%s ip:%s:%hu connType:%d", pRpc->label, - pConn, pConn->sid, pRpc->user, pConn->peerIpstr, pConn->peerPort, pConn->connType); + tTrace("%s %p, rpc connection is set up, sid:%d id:%s %s:%hu connType:%d", pRpc->label, + pConn, pConn->sid, pRpc->user, peerFqdn, pConn->peerPort, pConn->connType); } else { - tError("%s %p, failed to set up connection to ip:%s:%hu", pRpc->label, pConn, - pConn->peerIpstr, pConn->peerPort); + tError("%s %p, failed to set up connection to %s:%hu", pRpc->label, pConn, peerFqdn, pConn->peerPort); terrno = TSDB_CODE_NETWORK_UNAVAIL; rpcCloseConn(pConn); pConn = NULL; @@ -661,12 +663,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcInfo *pRpc = pContext->pRpc; SRpcIpSet *pIpSet = &pContext->ipSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->ip[pIpSet->inUse], pIpSet->port, pContext->connType); + pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); if ( pConn == NULL || pConn->user[0] == 0) { - char ipstr[20] = {0}; - tinet_ntoa(ipstr, pIpSet->ip[pIpSet->inUse]); - pConn = rpcOpenConn(pRpc, ipstr, pIpSet->port, pContext->connType); - if (pConn) pConn->destIp = pIpSet->ip[pIpSet->inUse]; + pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); } else { tTrace("%s %p, connection is retrieved from cache", pRpc->label, pConn); } @@ -789,7 +788,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { pConn->peerIp = pRecv->ip; char ipstr[20] = {0}; tinet_ntoa(ipstr, pRecv->ip); - strcpy(pConn->peerIpstr, ipstr); + strcpy(pConn->peerFqdn, ipstr); } if (pRecv->port) pConn->peerPort = pRecv->port; @@ -922,7 +921,6 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if ( rpcIsReq(pHead->msgType) ) { rpcMsg.handle = pConn; - pConn->destIp = pHead->destIp; taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer); (*(pRpc->cfp))(&rpcMsg); } else { @@ -932,7 +930,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { pConn->pContext = NULL; // for UDP, port may be changed by server, the port in ipSet shall be used for cache - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerIp, pContext->ipSet.port, pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType); if (pHead->code == TSDB_CODE_REDIRECT) { pContext->redirect = 1; @@ -1053,7 +1051,6 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { pHead->tranId = pConn->tranId; pHead->sourceId = pConn->ownId; pHead->destId = pConn->peerId; - pHead->destIp = pConn->destIp; pHead->port = 0; pHead->linkUid = pConn->linkUid; if (!pConn->secured) memcpy(pHead->user, pConn->user, tListLen(pHead->user)); @@ -1081,12 +1078,12 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { if ( rpcIsReq(pHead->msgType)) { if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) tTrace("%s %p, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, + pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) tTrace( "%s %p, %s is sent to %s:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIpstr, pConn->peerPort, + pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } @@ -1141,13 +1138,13 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { if (pConn->retry < 4) { tTrace("%s %p, re-send msg:%s to %s:%hud", pRpc->label, pConn, - taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort); + taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer); } else { // close the connection tTrace("%s %p, failed to send msg:%s to %s:%hu", pRpc->label, pConn, - taosMsg[pConn->outType], pConn->peerIpstr, pConn->peerPort); + taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); reportDisc = 1; } } else { diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index bac2ae879a..1260a34512 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -40,7 +40,7 @@ typedef struct SThreadObj { SFdObj * pHead; pthread_mutex_t mutex; pthread_cond_t fdReady; - char ipstr[TSDB_IPv4ADDR_LEN]; + uint32_t ip; int pollFd; int numOfFds; int threadId; @@ -50,7 +50,7 @@ typedef struct SThreadObj { } SThreadObj; typedef struct { - char ip[TSDB_IPv4ADDR_LEN]; + uint32_t ip; uint16_t port; char label[12]; int numOfThreads; @@ -65,12 +65,12 @@ static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); static void taosAcceptTcpConnection(void *arg); -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { +void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { SServerObj *pServerObj; SThreadObj *pThreadObj; pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); - strcpy(pServerObj->ip, ip); + pServerObj->ip = ip; pServerObj->port = port; strcpy(pServerObj->label, label); pServerObj->numOfThreads = numOfThreads; @@ -138,7 +138,7 @@ void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, free(pServerObj); pServerObj = NULL; } else { - tTrace("%s TCP server is initialized, ip:%s port:%hu numOfThreads:%d", label, ip, port, numOfThreads); + tTrace("%s TCP server is initialized, ip:0x%x port:%hu numOfThreads:%d", label, ip, port, numOfThreads); } return (void *)pServerObj; @@ -222,14 +222,14 @@ static void taosAcceptTcpConnection(void *arg) { } } -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { +void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) { SThreadObj *pThreadObj; pthread_attr_t thattr; pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); memset(pThreadObj, 0, sizeof(SThreadObj)); strcpy(pThreadObj->label, label); - strcpy(pThreadObj->ipstr, ip); + pThreadObj->ip = ip; pThreadObj->shandle = shandle; if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { @@ -284,21 +284,19 @@ void taosCleanUpTcpClient(void *chandle) { tfree(pThreadObj); } -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { +void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SThreadObj * pThreadObj = shandle; - struct in_addr destIp; - int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ipstr); + int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd <= 0) return NULL; - inet_aton(ip, &destIp); SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { pFdObj->thandle = thandle; pFdObj->port = port; - pFdObj->ip = destIp.s_addr; - tTrace("%s %p, TCP connection to %s:%hu is created, FD:%p numOfFds:%d", + pFdObj->ip = ip; + tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d", pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds); } else { close(fd); @@ -403,7 +401,7 @@ static void *taosProcessTcpData(void *param) { continue; } - // tTrace("%s TCP data is received, ip:%s:%u len:%d", pThreadObj->label, pFdObj->ipstr, pFdObj->port, msgLen); + // tTrace("%s TCP data is received, ip:0x%x:%u len:%d", pThreadObj->label, pFdObj->ip, pFdObj->port, msgLen); memcpy(msg, &rpcHead, sizeof(SRpcHead)); recvInfo.msg = msg; diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 25668fcb14..4a5fc7ba27 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -51,7 +51,7 @@ typedef struct { typedef struct { int index; int server; - char ip[16]; // local IP + uint32_t ip; // local IP uint16_t port; // local Port void *shandle; // handle passed by upper layer during server initialization int threads; @@ -77,7 +77,7 @@ static void *taosRecvUdpData(void *param); static SUdpBuf *taosCreateUdpBuf(SUdpConn *pConn, uint32_t ip, uint16_t port); static void taosProcessUdpBufTimer(void *param, void *tmrId); -void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { +void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { SUdpConn *pConn; SUdpConnSet *pSet; @@ -89,7 +89,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v } memset(pSet, 0, (size_t)size); - strcpy(pSet->ip, ip); + pSet->ip = ip; pSet->port = port; pSet->shandle = shandle; pSet->fp = fp; @@ -111,7 +111,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v ownPort = (port ? port + i : 0); pConn->fd = taosOpenUdpSocket(ip, ownPort); if (pConn->fd < 0) { - tError("%s failed to open UDP socket %s:%hu", label, ip, port); + tError("%s failed to open UDP socket %x:%hu", label, ip, port); taosCleanUpUdpConnection(pSet); return NULL; } @@ -157,7 +157,7 @@ void *taosInitUdpConnection(char *ip, uint16_t port, char *label, int threads, v ++pSet->threads; } - tTrace("%s UDP connection is initialized, ip:%s port:%hu threads:%d", label, ip, port, threads); + tTrace("%s UDP connection is initialized, ip:%x port:%hu threads:%d", label, ip, port, threads); return pSet; } @@ -190,7 +190,7 @@ void taosCleanUpUdpConnection(void *handle) { tfree(pSet); } -void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t port) { +void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { SUdpConnSet *pSet = (SUdpConnSet *)shandle; pSet->index = (pSet->index + 1) % pSet->threads; @@ -198,7 +198,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, char *ip, uint16_t por SUdpConn *pConn = pSet->udpConn + pSet->index; pConn->port = port; - tTrace("%s UDP connection is setup, ip: %s:%hu, local: %s:%d", pConn->label, ip, port, pSet->ip, + tTrace("%s UDP connection is setup, ip:%x:%hu, local:%x:%d", pConn->label, ip, port, pSet->ip, ntohs((uint16_t)pConn->localPort)); return pConn; diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 8fb549df8f..2aa1f0e4e9 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -88,13 +88,13 @@ int main(int argc, char *argv[]) { // server info ipSet.numOfIps = 1; ipSet.inUse = 0; - ipSet.port = 7000; - ipSet.ip[0] = inet_addr(serverIp); - ipSet.ip[1] = inet_addr("192.168.0.1"); + ipSet.port[0] = 7000; + ipSet.port[1] = 7000; + strcpy(ipSet.fqdn[0], serverIp); + strcpy(ipSet.fqdn[1], "192.168.0.1"); // client info memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localIp = "0.0.0.0"; rpcInit.localPort = 0; rpcInit.label = "APP"; rpcInit.numOfThreads = 1; @@ -110,11 +110,9 @@ int main(int argc, char *argv[]) { for (int i=1; ilastKey) + void * tsdbEncodeTable(STable *pTable, int *contLen); STable *tsdbDecodeTable(void *cont, int contLen); void tsdbFreeEncode(void *cont); @@ -129,9 +132,6 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); char * getTupleKey(const void *data); -// ------------------------------ TSDB CACHE INTERFACES ------------------------------ -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ - typedef struct { int blockId; int offset; diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 942afd70a6..3e241773ed 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -26,6 +26,9 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) { if (pCache == NULL) return NULL; if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + cacheBlockSize *= (1024 * 1024); + + if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS; pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 289ad0f0d0..5da8d303d9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1,4 +1,5 @@ #include "os.h" +#include "taosdef.h" #include "tulog.h" #include "talgo.h" #include "tsdb.h" @@ -11,24 +12,6 @@ #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 100000 -#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 60 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_KEEP 3650 -#define TSDB_MIN_KEEP 1 -#define TSDB_MAX_KEEP INT_MAX -#define TSDB_DEFAULT_CACHE_SIZE (16 * 1024 * 1024) // 16M -#define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024) // 4M -#define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024) // 1G #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" @@ -70,7 +53,6 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) { pCfg->minRowsPerFileBlock = -1; pCfg->maxRowsPerFileBlock = -1; pCfg->keep = -1; - pCfg->maxCacheSize = -1; pCfg->compression = TWO_STAGE_COMP; } @@ -163,6 +145,34 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) { return 0; } +static int tsdbRestoreInfo(STsdbRepo *pRepo) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + SFileGroup *pFGroup = NULL; + + SFileGroupIter iter; + SRWHelper rhelper = {0}; + + if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; + tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); + while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { + if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable * pTable = pMeta->tables[i]; + SCompIdx *pIdx = &rhelper.pCompIdx[i]; + + if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + } + } + + tsdbDestroyHelper(&rhelper); + return 0; + +_err: + tsdbDestroyHelper(&rhelper); + return -1; +} + /** * Open an existing TSDB storage repository * @param tsdbDir the existing TSDB root directory @@ -192,7 +202,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo); + pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -210,6 +220,16 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } + // Restore key from file + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbCloseFileH(pRepo->tsdbFileH); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (TsdbRepoT *)pRepo; @@ -612,13 +632,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1; } - // Check maxCacheSize - if (pCfg->maxCacheSize == -1) { - pCfg->maxCacheSize = TSDB_DEFAULT_CACHE_SIZE; - } else { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_SIZE) return -1; - } - return 0; } @@ -755,6 +768,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListPut(pTable->mem->pData, pNode); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + if (key > pTable->lastKey) pTable->lastKey = key; pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); @@ -901,7 +915,7 @@ _exit: } static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) { - + char dataDir[128] = {0}; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 8a2fe22104..3ad9b38366 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -315,6 +315,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->tableId = pCfg->tableId; table->name = strdup(pCfg->name); + table->lastKey = 0; if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE table->type = TSDB_CHILD_TABLE; table->superUid = pCfg->superUid; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 74068f4543..110c114540 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -18,6 +18,7 @@ #include "talgo.h" #include "tutil.h" #include "tcompare.h" +#include "exception.h" #include "../../../query/inc/qast.h" // todo move to common module #include "../../../query/inc/tlosertree.h" // todo move to util module @@ -25,7 +26,6 @@ #include "tsdbMain.h" #define EXTRA_BYTES 2 -#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoData*)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) #define ASCENDING_ORDER_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) @@ -34,6 +34,11 @@ enum { QUERY_RANGE_GREATER_EQUAL = 1, }; +enum { + TSDB_QUERY_TYPE_ALL_ROWS = 1, + TSDB_QUERY_TYPE_LAST_ROW = 2, +}; + typedef struct SField { // todo need the definition } SField; @@ -66,8 +71,8 @@ typedef struct STableCheckInfo { SCompInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks; // number of qualified data blocks not the original blocks - - SDataCols* pDataCols; + SDataCols* pDataCols; + SSkipListIterator* iter; } STableCheckInfo; @@ -105,11 +110,11 @@ typedef struct STsdbQueryHandle { SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t realNumOfRows; - SArray* pTableCheckInfo; + SArray* pTableCheckInfo; //SArray int32_t activeIndex; bool checkFiles; // check file stage void* qinfo; // query info handle, for debug purpose - + int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows STableBlockInfo* pDataBlockInfo; SFileGroup* pFileGroup; @@ -117,6 +122,8 @@ typedef struct STsdbQueryHandle { SRWHelper rhelper; } STsdbQueryHandle; +static void changeQueryHandleForQuery(TsdbQueryHandleT pqHandle); + static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; pBlockLoadInfo->sid = -1; @@ -191,7 +198,17 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); - return (TsdbQueryHandleT)pQueryHandle; + return (TsdbQueryHandleT) pQueryHandle; +} + +TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList) { + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList); + + pQueryHandle->type = TSDB_QUERY_TYPE_LAST_ROW; + pQueryHandle->order = TSDB_ORDER_DESC; + + changeQueryHandleForQuery(pQueryHandle); + return pQueryHandle; } static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { @@ -932,6 +949,54 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pqHandle) { } } +void changeQueryHandleForQuery(TsdbQueryHandleT pqHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; + assert(!ASCENDING_ORDER_TRAVERSE(pQueryHandle->order)); + + // starts from the buffer in case of descending timestamp order check data blocks + + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + TSKEY key = INT64_MIN; + int32_t index = -1; + + for(int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pCheckInfo->pTableObj->lastKey > key) { + key = pCheckInfo->pTableObj->lastKey; + index = i; + } + } + + // erase all other elements in array list, todo refactor + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + if (i == index) { + continue; + } + + STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + tSkipListDestroyIter(pTableCheckInfo->iter); + + if (pTableCheckInfo->pDataCols != NULL) { + tfree(pTableCheckInfo->pDataCols->buf); + } + + tfree(pTableCheckInfo->pDataCols); + tfree(pTableCheckInfo->pCompInfo); + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); + taosArrayDestroy(pQueryHandle->pTableCheckInfo); + + pQueryHandle->pTableCheckInfo = taosArrayInit(1, sizeof(STableCheckInfo)); + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {key, key}; +} + static int tsdbReadRowsFromCache(SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; @@ -1101,10 +1166,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -int32_t tsdbResetQuery(TsdbQueryHandleT* pQueryHandle, STimeWindow* window, TsdbPosT position, int16_t order) { - return 0; -} - SArray* tsdbRetrieveDataRow(TsdbQueryHandleT* pQueryHandle, SArray* pIdList, SQueryRowCond* pCond) { return NULL; } TsdbQueryHandleT* tsdbQueryFromTagConds(STsdbQueryCond* pCond, int16_t stableId, const char* pTagFilterStr) { @@ -1412,7 +1473,6 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } - static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; @@ -1440,7 +1500,6 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } - int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -1464,21 +1523,35 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC } int32_t ret = TSDB_CODE_SUCCESS; + tExprNode* expr = NULL; - tExprNode* expr = exprTreeFromTableName(tbnameCond); - tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); - if (tagExpr != NULL) { + TRY(32) { + expr = exprTreeFromTableName(tbnameCond); if (expr == NULL) { - expr = tagExpr; + expr = exprTreeFromBinary(pTagCond, len); } else { - tExprNode* tbnameExpr = expr; - expr = calloc(1, sizeof(tExprNode)); - expr->nodeType = TSQL_NODE_EXPR; - expr->_node.optr = tagNameRelType; - expr->_node.pLeft = tagExpr; - expr->_node.pRight = tbnameExpr; + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); + if (tagExpr != NULL) { + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, tagExpr, NULL); + tExprNode* tbnameExpr = expr; + expr = calloc(1, sizeof(tExprNode)); + if (expr == NULL) { + THROW( TSDB_CODE_SERV_OUT_OF_MEMORY ); + } + expr->nodeType = TSQL_NODE_EXPR; + expr->_node.optr = tagNameRelType; + expr->_node.pLeft = tagExpr; + expr->_node.pRight = tbnameExpr; + } } - } + CLEANUP_EXECUTE(); + + } CATCH( code ) { + CLEANUP_EXECUTE(); + ret = code; + // TODO: more error handling + } END_TRY doQueryTableList(pTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); diff --git a/src/util/inc/exception.h b/src/util/inc/exception.h new file mode 100644 index 0000000000..41f01d68dd --- /dev/null +++ b/src/util/inc/exception.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2020 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_EXCEPTION_H +#define TDENGINE_EXCEPTION_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * cleanup actions + */ +typedef struct SCleanupAction { + bool failOnly; + uint8_t wrapper; + uint16_t reserved; + void* func; + union { + void* Ptr; + bool Bool; + char Char; + int8_t Int8; + uint8_t Uint8; + int16_t Int16; + uint16_t Uint16; + int Int; + unsigned int Uint; + int32_t Int32; + uint32_t Uint32; + int64_t Int64; + uint64_t Uint64; + float Float; + double Double; + } arg1, arg2; +} SCleanupAction; + + +/* + * exception hander registration + */ +typedef struct SExceptionNode { + struct SExceptionNode* prev; + jmp_buf jb; + int32_t code; + int32_t maxCleanupAction; + int32_t numCleanupAction; + SCleanupAction* cleanupActions; +} SExceptionNode; + +//////////////////////////////////////////////////////////////////////////////// +// functions & macros for auto-cleanup + +void cleanupPush_void_ptr_ptr ( bool failOnly, void* func, void* arg1, void* arg2 ); +void cleanupPush_void_ptr_bool ( bool failOnly, void* func, void* arg1, bool arg2 ); +void cleanupPush_void_ptr ( bool failOnly, void* func, void* arg ); +void cleanupPush_int_int ( bool failOnly, void* func, int arg ); +void cleanupPush_void ( bool failOnly, void* func ); + +int32_t cleanupGetActionCount(); +void cleanupExecuteTo( int32_t anchor, bool failed ); +void cleanupExecute( SExceptionNode* node, bool failed ); + +#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) ) +#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) ) +#define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) ) +#define CLEANUP_PUSH_INT_INT( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) ) +#define CLEANUP_PUSH_VOID( failOnly, func ) cleanupPush_void( (failOnly), (void*)(func) ) +#define CLEANUP_PUSH_FREE( failOnly, arg ) cleanupPush_void_ptr( (failOnly), free, (void*)(arg) ) +#define CLEANUP_PUSH_CLOSE( failOnly, arg ) cleanupPush_int_int( (failOnly), close, (int)(arg) ) + +#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() +#define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) ) + + +//////////////////////////////////////////////////////////////////////////////// +// functions & macros for exception handling + +void exceptionPushNode( SExceptionNode* node ); +int32_t exceptionPopNode(); +void exceptionThrow( int code ); + +#define TRY(maxCleanupActions) do { \ + SExceptionNode exceptionNode = { 0 }; \ + SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \ + exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \ + exceptionNode.cleanupActions = cleanupActions; \ + exceptionPushNode( &exceptionNode ); \ + int caughtException = setjmp( exceptionNode.jb ); \ + if( caughtException == 0 ) + +#define CATCH( code ) int code = exceptionPopNode(); \ + if( caughtException == 1 ) + +#define FINALLY( code ) int code = exceptionPopNode(); + +#define END_TRY } while( 0 ); + +#define THROW( x ) exceptionThrow( (x) ) +#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1)) +#define CLEANUP_EXECUTE() cleanupExecute( &exceptionNode, CAUGHT_EXCEPTION() ) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index 9bc0fd9f43..e2bdb815d7 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -16,122 +16,163 @@ #ifndef TDENGINE_TBUFFER_H #define TDENGINE_TBUFFER_H -#include "setjmp.h" -#include "os.h" +#include +#include #ifdef __cplusplus extern "C" { #endif +//////////////////////////////////////////////////////////////////////////////// +// usage example /* -SBuffer can be used to read or write a buffer, but cannot be used for both -read & write at a same time. Below is an example: +#include +#include "exception.h" -int main(int argc, char** argv) { - //--------------------- write ------------------------ - SBuffer wbuf; - int32_t code = tbufBeginWrite(&wbuf); - if (code != 0) { - // handle errors - return 0; - } +int main( int argc, char** argv ) { + SBufferWriter bw = tbufInitWriter( NULL, false ); - // reserve 1024 bytes for the buffer to improve performance - tbufEnsureCapacity(&wbuf, 1024); + TRY( 1 ) { + //--------------------- write ------------------------ + // reserve 1024 bytes for the buffer to improve performance + tbufEnsureCapacity( &bw, 1024 ); - // write 5 integers to the buffer - for (int i = 0; i < 5; i++) { - tbufWriteInt32(&wbuf, i); - } + // reserve space for the interger count + size_t pos = tbufReserve( &bw, sizeof(int32_t) ); + // write 5 integers to the buffer + for( int i = 0; i < 5; i++) { + tbufWriteInt32( &bw, i ); + } + // write the integer count to buffer at reserved position + tbufWriteInt32At( &bw, pos, 5 ); - // write a string to the buffer - tbufWriteString(&wbuf, "this is a string.\n"); + // write a string to the buffer + tbufWriteString( &bw, "this is a string.\n" ); + // acquire the result and close the write buffer + size_t size = tbufTell( &bw ); + char* data = tbufGetData( &bw, false ); - // acquire the result and close the write buffer - size_t size = tbufTell(&wbuf); - char* data = tbufGetData(&wbuf, true); - tbufClose(&wbuf, true); - - - //------------------------ read ----------------------- - SBuffer rbuf; - code = tbufBeginRead(&rbuf, data, size); - if (code != 0) { - printf("you will see this message after print out 5 integers and a string.\n"); - tbufClose(&rbuf, false); - return 0; - } - - // read & print out 5 integers - for (int i = 0; i < 5; i++) { - printf("%d\n", tbufReadInt32(&rbuf)); - } - - // read & print out a string - printf(tbufReadString(&rbuf, NULL)); - - // try read another integer, this result in an error as there no this integer - tbufReadInt32(&rbuf); - - printf("you should not see this message.\n"); - tbufClose(&rbuf, false); + //------------------------ read ----------------------- + SBufferReader br = tbufInitReader( data, size, false ); + // read & print out all integers + int32_t count = tbufReadInt32( &br ); + for( int i = 0; i < count; i++ ) { + printf( "%d\n", tbufReadInt32(&br) ); + } + // read & print out a string + puts( tbufReadString(&br, NULL) ); + // try read another integer, this result in an error as there no this integer + tbufReadInt32( &br ); + printf( "you should not see this message.\n" ); + } CATCH( code ) { + printf( "exception code is: %d, you will see this message after print out 5 integers and a string.\n", code ); + } END_TRY + tbufCloseWriter( &bw ); return 0; } */ + typedef struct { - jmp_buf jb; - char* data; - size_t pos; - size_t size; -} SBuffer; + bool endian; + const char* data; + size_t pos; + size_t size; +} SBufferReader; -// common functions can be used in both read & write -#define tbufThrowError(buf, code) longjmp((buf)->jb, (code)) -size_t tbufTell(SBuffer* buf); -size_t tbufSeekTo(SBuffer* buf, size_t pos); -size_t tbufSkip(SBuffer* buf, size_t size); -void tbufClose(SBuffer* buf, bool keepData); +typedef struct { + bool endian; + char* data; + size_t pos; + size_t size; + void* (*allocator)( void*, size_t ); +} SBufferWriter; -// basic read functions -#define tbufBeginRead(buf, _data, len) ((buf)->data = (char*)(_data), ((buf)->pos = 0), ((buf)->size = ((_data) == NULL) ? 0 : (len)), setjmp((buf)->jb)) -char* tbufRead(SBuffer* buf, size_t size); -void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size); -const char* tbufReadString(SBuffer* buf, size_t* len); -size_t tbufReadToString(SBuffer* buf, char* dst, size_t size); +//////////////////////////////////////////////////////////////////////////////// +// common functions & macros for both reader & writer -// basic write functions -#define tbufBeginWrite(buf) ((buf)->data = NULL, ((buf)->pos = 0), ((buf)->size = 0), setjmp((buf)->jb)) -void tbufEnsureCapacity(SBuffer* buf, size_t size); -char* tbufGetData(SBuffer* buf, bool takeOver); -void tbufWrite(SBuffer* buf, const void* data, size_t size); -void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size); -void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len); -void tbufWriteString(SBuffer* buf, const char* str); +#define tbufTell( buf ) ((buf)->pos) -// read & write function for primitive types -#ifndef TBUFFER_DEFINE_FUNCTION -#define TBUFFER_DEFINE_FUNCTION(type, name) \ - type tbufRead##name(SBuffer* buf); \ - void tbufWrite##name(SBuffer* buf, type data); \ - void tbufWrite##name##At(SBuffer* buf, size_t pos, type data); -#endif -TBUFFER_DEFINE_FUNCTION(bool, Bool) -TBUFFER_DEFINE_FUNCTION(char, Char) -TBUFFER_DEFINE_FUNCTION(int8_t, Int8) -TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8) -TBUFFER_DEFINE_FUNCTION(int16_t, Int16) -TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) -TBUFFER_DEFINE_FUNCTION(int32_t, Int32) -TBUFFER_DEFINE_FUNCTION(uint32_t, Uint32) -TBUFFER_DEFINE_FUNCTION(int64_t, Int64) -TBUFFER_DEFINE_FUNCTION(uint64_t, Uint64) -TBUFFER_DEFINE_FUNCTION(float, Float) -TBUFFER_DEFINE_FUNCTION(double, Double) +//////////////////////////////////////////////////////////////////////////////// +// reader functions & macros + +// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically +#define tbufInitReader( Data, Size, Endian ) {.endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 :(Size))} + +size_t tbufSkip( SBufferReader* buf, size_t size ); + +const char* tbufRead( SBufferReader* buf, size_t size ); +void tbufReadToBuffer( SBufferReader* buf, void* dst, size_t size ); +const char* tbufReadString( SBufferReader* buf, size_t* len ); +size_t tbufReadToString( SBufferReader* buf, char* dst, size_t size ); +const char* tbufReadBinary( SBufferReader* buf, size_t *len ); +size_t tbufReadToBinary( SBufferReader* buf, void* dst, size_t size ); + +bool tbufReadBool( SBufferReader* buf ); +char tbufReadChar( SBufferReader* buf ); +int8_t tbufReadInt8( SBufferReader* buf ); +uint8_t tbufReadUint8( SBufferReader* buf ); +int16_t tbufReadInt16( SBufferReader* buf ); +uint16_t tbufReadUint16( SBufferReader* buf ); +int32_t tbufReadInt32( SBufferReader* buf ); +uint32_t tbufReadUint32( SBufferReader* buf ); +int64_t tbufReadInt64( SBufferReader* buf ); +uint64_t tbufReadUint64( SBufferReader* buf ); +float tbufReadFloat( SBufferReader* buf ); +double tbufReadDouble( SBufferReader* buf ); + + +//////////////////////////////////////////////////////////////////////////////// +// writer functions & macros + +// *Allocator*, function to allocate memory, will use 'realloc' if NULL +// *Endian*, if true, writer functions of primitive types will do 'hton' automatically +#define tbufInitWriter( Allocator, Endian ) {.endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator))} +void tbufCloseWriter( SBufferWriter* buf ); + +void tbufEnsureCapacity( SBufferWriter* buf, size_t size ); +size_t tbufReserve( SBufferWriter* buf, size_t size ); +char* tbufGetData( SBufferWriter* buf, bool takeOver ); + +void tbufWrite( SBufferWriter* buf, const void* data, size_t size ); +void tbufWriteAt( SBufferWriter* buf, size_t pos, const void* data, size_t size ); +void tbufWriteStringLen( SBufferWriter* buf, const char* str, size_t len ); +void tbufWriteString( SBufferWriter* buf, const char* str ); +// the prototype of tbufWriteBinary and tbufWrite are identical +// the difference is: tbufWriteBinary writes the length of the data to the buffer +// first, then the actual data, which means the reader don't need to know data +// size before read. Write only write the data itself, which means the reader +// need to know data size before read. +void tbufWriteBinary( SBufferWriter* buf, const void* data, size_t len ); + +void tbufWriteBool( SBufferWriter* buf, bool data ); +void tbufWriteBoolAt( SBufferWriter* buf, size_t pos, bool data ); +void tbufWriteChar( SBufferWriter* buf, char data ); +void tbufWriteCharAt( SBufferWriter* buf, size_t pos, char data ); +void tbufWriteInt8( SBufferWriter* buf, int8_t data ); +void tbufWriteInt8At( SBufferWriter* buf, size_t pos, int8_t data ); +void tbufWriteUint8( SBufferWriter* buf, uint8_t data ); +void tbufWriteUint8At( SBufferWriter* buf, size_t pos, uint8_t data ); +void tbufWriteInt16( SBufferWriter* buf, int16_t data ); +void tbufWriteInt16At( SBufferWriter* buf, size_t pos, int16_t data ); +void tbufWriteUint16( SBufferWriter* buf, uint16_t data ); +void tbufWriteUint16At( SBufferWriter* buf, size_t pos, uint16_t data ); +void tbufWriteInt32( SBufferWriter* buf, int32_t data ); +void tbufWriteInt32At( SBufferWriter* buf, size_t pos, int32_t data ); +void tbufWriteUint32( SBufferWriter* buf, uint32_t data ); +void tbufWriteUint32At( SBufferWriter* buf, size_t pos, uint32_t data ); +void tbufWriteInt64( SBufferWriter* buf, int64_t data ); +void tbufWriteInt64At( SBufferWriter* buf, size_t pos, int64_t data ); +void tbufWriteUint64( SBufferWriter* buf, uint64_t data ); +void tbufWriteUint64At( SBufferWriter* buf, size_t pos, uint64_t data ); +void tbufWriteFloat( SBufferWriter* buf, float data ); +void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data ); +void tbufWriteDouble( SBufferWriter* buf, double data ); +void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data ); #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 9eb4c26464..60d339fa5c 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,44 +20,27 @@ extern "C" { #endif -#include -#include -#include -#include - -int taosNonblockwrite(int fd, char *ptr, int nbytes); - int taosReadn(int sock, char *buffer, int len); - int taosWriteMsg(int fd, void *ptr, int nbytes); - int taosReadMsg(int fd, void *ptr, int nbytes); +int taosNonblockwrite(int fd, char *ptr, int nbytes); +int taosCopyFds(int sfd, int dfd, int64_t len); +int taosSetNonblocking(int sock, int on); -int taosOpenUdpSocket(char *ip, uint16_t port); - -int taosOpenTcpClientSocket(char *ip, uint16_t port, char *localIp); - -int taosOpenTcpServerSocket(char *ip, uint16_t port); - -int taosKeepTcpAlive(int sockFd); - +int taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +int taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +int taosOpenTcpServerSocket(uint32_t ip, uint16_t port); +int taosKeepTcpAlive(int sockFd); void taosCloseTcpSocket(int sockFd); -int taosOpenUDServerSocket(char *ip, uint16_t port); +int taosOpenUDServerSocket(uint32_t ip, uint16_t port); +int taosOpenUDClientSocket(uint32_t ip, uint16_t port); +int taosOpenRawSocket(uint32_t ip); -int taosOpenUDClientSocket(char *ip, uint16_t port); - -int taosOpenRawSocket(char *ip); - -int taosCopyFds(int sfd, int dfd, int64_t len); - -int taosGetPublicIp(char *const ip); - -int taosGetPrivateIp(char *const ip); - -void tinet_ntoa(char *ipstr, unsigned int ip); - -int taosSetNonblocking(int sock, int on); +int taosGetFqdn(char *); +uint32_t taosGetIpFromFqdn(const char *); +void tinet_ntoa(char *ipstr, unsigned int ip); +uint32_t ip2uint(const char *const ip_addr); #ifdef __cplusplus } diff --git a/src/util/src/exception.c b/src/util/src/exception.c new file mode 100644 index 0000000000..7f8f91c784 --- /dev/null +++ b/src/util/src/exception.c @@ -0,0 +1,132 @@ +#include "exception.h" + + +static _Thread_local SExceptionNode* expList; + +void exceptionPushNode( SExceptionNode* node ) { + node->prev = expList; + expList = node; +} + +int32_t exceptionPopNode() { + SExceptionNode* node = expList; + expList = node->prev; + return node->code; +} + +void exceptionThrow( int code ) { + expList->code = code; + longjmp( expList->jb, 1 ); +} + + + +static void cleanupWrapper_void_ptr_ptr( SCleanupAction* ca ) { + void (*func)( void*, void* ) = ca->func; + func( ca->arg1.Ptr, ca->arg2.Ptr ); +} + +static void cleanupWrapper_void_ptr_bool( SCleanupAction* ca ) { + void (*func)( void*, bool ) = ca->func; + func( ca->arg1.Ptr, ca->arg2.Bool ); +} + +static void cleanupWrapper_void_ptr( SCleanupAction* ca ) { + void (*func)( void* ) = ca->func; + func( ca->arg1.Ptr ); +} + +static void cleanupWrapper_int_int( SCleanupAction* ca ) { + int (*func)( int ) = ca->func; + func( (int)(intptr_t)(ca->arg1.Int) ); +} + +static void cleanupWrapper_void_void( SCleanupAction* ca ) { + void (*func)() = ca->func; + func(); +} + +typedef void (*wrapper)(SCleanupAction*); +static wrapper wrappers[] = { + cleanupWrapper_void_ptr_ptr, + cleanupWrapper_void_ptr_bool, + cleanupWrapper_void_ptr, + cleanupWrapper_int_int, + cleanupWrapper_void_void, +}; + + +void cleanupPush_void_ptr_ptr( bool failOnly, void* func, void* arg1, void* arg2 ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 0; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Ptr = arg2; +} + +void cleanupPush_void_ptr_bool( bool failOnly, void* func, void* arg1, bool arg2 ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 1; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Bool = arg2; +} + +void cleanupPush_void_ptr( bool failOnly, void* func, void* arg ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 2; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} + +void cleanupPush_int_int( bool failOnly, void* func, int arg ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 3; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Int = arg; +} + +void cleanupPush_void( bool failOnly, void* func ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 4; + ca->failOnly = failOnly; + ca->func = func; +} + + + +int32_t cleanupGetActionCount() { + return expList->numCleanupAction; +} + + +static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed ) { + while( node->numCleanupAction > anchor ) { + --node->numCleanupAction; + SCleanupAction *ca = node->cleanupActions + node->numCleanupAction; + if( failed || !(ca->failOnly) ) + wrappers[ca->wrapper]( ca ); + } +} + +void cleanupExecuteTo( int32_t anchor, bool failed ) { + doExecuteCleanup( expList, anchor, failed ); +} + +void cleanupExecute( SExceptionNode* node, bool failed ) { + doExecuteCleanup( node, 0, failed ); +} \ No newline at end of file diff --git a/src/util/src/hash.c b/src/util/src/hash.c index e0e13c1b22..05a99903ba 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -520,7 +520,8 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { static SHashNode *getNextHashNode(SHashMutableIterator *pIter) { assert(pIter != NULL); - + pIter->entryIndex += 1; + while (pIter->entryIndex < pIter->pHashObj->capacity) { SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; if (pEntry->next == NULL) { @@ -540,7 +541,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } size_t size = taosHashGetSize(pIter->pHashObj); - if (size == 0 || pIter->num >= size) { + if (size == 0) { return false; } diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 3b12864ec8..5ef4417710 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -176,7 +176,6 @@ SArray* taosArrayClone(const SArray* pSrc) { return dst; } - void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c index a83d7dddb0..3b4cc74cc3 100644 --- a/src/util/src/tbuffer.c +++ b/src/util/src/tbuffer.c @@ -16,150 +16,384 @@ #include #include #include - -#define TBUFFER_DEFINE_FUNCTION(type, name) \ - type tbufRead##name(SBuffer* buf) { \ - type ret; \ - tbufReadToBuffer(buf, &ret, sizeof(type)); \ - return ret; \ - }\ - void tbufWrite##name(SBuffer* buf, type data) {\ - tbufWrite(buf, &data, sizeof(data));\ - }\ - void tbufWrite##name##At(SBuffer* buf, size_t pos, type data) {\ - tbufWriteAt(buf, pos, &data, sizeof(data));\ - } - +#include #include "tbuffer.h" - +#include "exception.h" +#include //////////////////////////////////////////////////////////////////////////////// -// common functions +// reader functions -size_t tbufTell(SBuffer* buf) { - return buf->pos; -} - -size_t tbufSeekTo(SBuffer* buf, size_t pos) { - if (pos > buf->size) { - // TODO: update error code, other tbufThrowError need to be changed too - tbufThrowError(buf, 1); +size_t tbufSkip(SBufferReader* buf, size_t size) { + if( (buf->pos + size) > buf->size ) { + THROW( TSDB_CODE_MEMORY_CORRUPTED ); } size_t old = buf->pos; - buf->pos = pos; + buf->pos += size; return old; } -size_t tbufSkip(SBuffer* buf, size_t size) { - return tbufSeekTo(buf, buf->pos + size); -} - -void tbufClose(SBuffer* buf, bool keepData) { - if (!keepData) { - free(buf->data); - } - buf->data = NULL; - buf->pos = 0; - buf->size = 0; -} - -//////////////////////////////////////////////////////////////////////////////// -// read functions - -char* tbufRead(SBuffer* buf, size_t size) { - char* ret = buf->data + buf->pos; - tbufSkip(buf, size); +const char* tbufRead( SBufferReader* buf, size_t size ) { + const char* ret = buf->data + buf->pos; + tbufSkip( buf, size ); return ret; } -void tbufReadToBuffer(SBuffer* buf, void* dst, size_t size) { - assert(dst != NULL); +void tbufReadToBuffer( SBufferReader* buf, void* dst, size_t size ) { + assert( dst != NULL ); // always using memcpy, leave optimization to compiler - memcpy(dst, tbufRead(buf, size), size); + memcpy( dst, tbufRead(buf, size), size ); } -const char* tbufReadString(SBuffer* buf, size_t* len) { - uint16_t l = tbufReadUint16(buf); - char* ret = buf->data + buf->pos; - tbufSkip(buf, l + 1); - ret[l] = 0; // ensure the string end with '\0' - if (len != NULL) { +static size_t tbufReadLength( SBufferReader* buf ) { + // maximum length is 65535, if larger length is required + // this function and the corresponding write function need to be + // revised. + uint16_t l = tbufReadUint16( buf ); + return l; +} + +const char* tbufReadString( SBufferReader* buf, size_t* len ) { + size_t l = tbufReadLength( buf ); + const char* ret = buf->data + buf->pos; + tbufSkip( buf, l + 1 ); + if( ret[l] != 0 ) { + THROW( TSDB_CODE_MEMORY_CORRUPTED ); + } + if( len != NULL ) { *len = l; } return ret; } -size_t tbufReadToString(SBuffer* buf, char* dst, size_t size) { - assert(dst != NULL); - size_t len; - const char* str = tbufReadString(buf, &len); +size_t tbufReadToString( SBufferReader* buf, char* dst, size_t size ) { + assert( dst != NULL ); + size_t len; + const char* str = tbufReadString( buf, &len ); if (len >= size) { len = size - 1; } - memcpy(dst, str, len); + memcpy( dst, str, len ); dst[len] = 0; return len; } +const char* tbufReadBinary( SBufferReader* buf, size_t *len ) { + size_t l = tbufReadLength( buf ); + const char* ret = buf->data + buf->pos; + tbufSkip( buf, l ); + if( len != NULL ) { + *len = l; + } + return ret; +} + +size_t tbufReadToBinary( SBufferReader* buf, void* dst, size_t size ) { + assert( dst != NULL ); + size_t len; + const char* data = tbufReadBinary( buf, &len ); + if( len >= size ) { + len = size; + } + memcpy( dst, data, len ); + return len; +} + +bool tbufReadBool( SBufferReader* buf ) { + bool ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + return ret; +} + +char tbufReadChar( SBufferReader* buf ) { + char ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + return ret; +} + +int8_t tbufReadInt8( SBufferReader* buf ) { + int8_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + return ret; +} + +uint8_t tbufReadUint8( SBufferReader* buf ) { + uint8_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + return ret; +} + +int16_t tbufReadInt16( SBufferReader* buf ) { + int16_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return (int16_t)ntohs( ret ); + } + return ret; +} + +uint16_t tbufReadUint16( SBufferReader* buf ) { + uint16_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return ntohs( ret ); + } + return ret; +} + +int32_t tbufReadInt32( SBufferReader* buf ) { + int32_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return (int32_t)ntohl( ret ); + } + return ret; +} + +uint32_t tbufReadUint32( SBufferReader* buf ) { + uint32_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return ntohl( ret ); + } + return ret; +} + +int64_t tbufReadInt64( SBufferReader* buf ) { + int64_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return (int64_t)htobe64( ret ); // TODO: ntohll + } + return ret; +} + +uint64_t tbufReadUint64( SBufferReader* buf ) { + uint64_t ret; + tbufReadToBuffer( buf, &ret, sizeof(ret) ); + if( buf->endian ) { + return htobe64( ret ); // TODO: ntohll + } + return ret; +} + +float tbufReadFloat( SBufferReader* buf ) { + uint32_t ret = tbufReadUint32( buf ); + return *(float*)( &ret ); +} + +double tbufReadDouble(SBufferReader* buf) { + uint64_t ret = tbufReadUint64( buf ); + return *(double*)( &ret ); +} //////////////////////////////////////////////////////////////////////////////// -// write functions +// writer functions -void tbufEnsureCapacity(SBuffer* buf, size_t size) { +void tbufCloseWriter( SBufferWriter* buf ) { + (*buf->allocator)( buf->data, 0 ); + buf->data = NULL; + buf->pos = 0; + buf->size = 0; +} + +void tbufEnsureCapacity( SBufferWriter* buf, size_t size ) { size += buf->pos; - if (size > buf->size) { + if( size > buf->size ) { size_t nsize = size + buf->size; - char* data = realloc(buf->data, nsize); - if (data == NULL) { - tbufThrowError(buf, 2); + char* data = (*buf->allocator)( buf->data, nsize ); + // TODO: the exception should be thrown by the allocator function + if( data == NULL ) { + THROW( TSDB_CODE_SERV_OUT_OF_MEMORY ); } buf->data = data; buf->size = nsize; } } -char* tbufGetData(SBuffer* buf, bool takeOver) { +size_t tbufReserve( SBufferWriter* buf, size_t size ) { + tbufEnsureCapacity( buf, size ); + size_t old = buf->pos; + buf->pos += size; + return old; +} + +char* tbufGetData( SBufferWriter* buf, bool takeOver ) { char* ret = buf->data; - if (takeOver) { + if( takeOver ) { buf->pos = 0; buf->size = 0; buf->data = NULL; } - return ret; } -void tbufEndWrite(SBuffer* buf) { - free(buf->data); - buf->data = NULL; - buf->pos = 0; - buf->size = 0; -} - -void tbufWrite(SBuffer* buf, const void* data, size_t size) { - assert(data != NULL); - tbufEnsureCapacity(buf, size); - memcpy(buf->data + buf->pos, data, size); +void tbufWrite( SBufferWriter* buf, const void* data, size_t size ) { + assert( data != NULL ); + tbufEnsureCapacity( buf, size ); + memcpy( buf->data + buf->pos, data, size ); buf->pos += size; } -void tbufWriteAt(SBuffer* buf, size_t pos, const void* data, size_t size) { - assert(data != NULL); +void tbufWriteAt( SBufferWriter* buf, size_t pos, const void* data, size_t size ) { + assert( data != NULL ); // this function can only be called to fill the gap on previous writes, // so 'pos + size <= buf->pos' must be true - assert(pos + size <= buf->pos); - memcpy(buf->data + pos, data, size); + assert( pos + size <= buf->pos ); + memcpy( buf->data + pos, data, size ); } -void tbufWriteStringLen(SBuffer* buf, const char* str, size_t len) { - // maximum string length is 65535, if longer string is required +static void tbufWriteLength( SBufferWriter* buf, size_t len ) { + // maximum length is 65535, if larger length is required // this function and the corresponding read function need to be // revised. - assert(len <= 0xffff); - tbufWriteUint16(buf, (uint16_t)len); - tbufWrite(buf, str, len + 1); + assert( len <= 0xffff ); + tbufWriteUint16( buf, (uint16_t)len ); } -void tbufWriteString(SBuffer* buf, const char* str) { - tbufWriteStringLen(buf, str, strlen(str)); +void tbufWriteStringLen( SBufferWriter* buf, const char* str, size_t len ) { + tbufWriteLength( buf, len ); + tbufWrite( buf, str, len ); + tbufWriteChar( buf, '\0' ); +} + +void tbufWriteString( SBufferWriter* buf, const char* str ) { + tbufWriteStringLen( buf, str, strlen(str) ); +} + +void tbufWriteBinary( SBufferWriter* buf, const void* data, size_t len ) { + tbufWriteLength( buf, len ); + tbufWrite( buf, data, len ); +} + +void tbufWriteBool( SBufferWriter* buf, bool data ) { + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteBoolAt( SBufferWriter* buf, size_t pos, bool data ) { + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteChar( SBufferWriter* buf, char data ) { + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteCharAt( SBufferWriter* buf, size_t pos, char data ) { + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteInt8( SBufferWriter* buf, int8_t data ) { + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteInt8At( SBufferWriter* buf, size_t pos, int8_t data ) { + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteUint8( SBufferWriter* buf, uint8_t data ) { + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteUint8At( SBufferWriter* buf, size_t pos, uint8_t data ) { + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteInt16( SBufferWriter* buf, int16_t data ) { + if( buf->endian ) { + data = (int16_t)htons( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteInt16At( SBufferWriter* buf, size_t pos, int16_t data ) { + if( buf->endian ) { + data = (int16_t)htons( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteUint16( SBufferWriter* buf, uint16_t data ) { + if( buf->endian ) { + data = htons( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteUint16At( SBufferWriter* buf, size_t pos, uint16_t data ) { + if( buf->endian ) { + data = htons( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteInt32( SBufferWriter* buf, int32_t data ) { + if( buf->endian ) { + data = (int32_t)htonl( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteInt32At( SBufferWriter* buf, size_t pos, int32_t data ) { + if( buf->endian ) { + data = (int32_t)htonl( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteUint32( SBufferWriter* buf, uint32_t data ) { + if( buf->endian ) { + data = htonl( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteUint32At( SBufferWriter* buf, size_t pos, uint32_t data ) { + if( buf->endian ) { + data = htonl( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteInt64( SBufferWriter* buf, int64_t data ) { + if( buf->endian ) { + data = (int64_t)htobe64( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteInt64At( SBufferWriter* buf, size_t pos, int64_t data ) { + if( buf->endian ) { + data = (int64_t)htobe64( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteUint64( SBufferWriter* buf, uint64_t data ) { + if( buf->endian ) { + data = htobe64( data ); + } + tbufWrite( buf, &data, sizeof(data) ); +} + +void tbufWriteUint64At( SBufferWriter* buf, size_t pos, uint64_t data ) { + if( buf->endian ) { + data = htobe64( data ); + } + tbufWriteAt( buf, pos, &data, sizeof(data) ); +} + +void tbufWriteFloat( SBufferWriter* buf, float data ) { + tbufWriteUint32( buf, *(uint32_t*)(&data) ); +} + +void tbufWriteFloatAt( SBufferWriter* buf, size_t pos, float data ) { + tbufWriteUint32At( buf, pos, *(uint32_t*)(&data) ); +} + +void tbufWriteDouble( SBufferWriter* buf, double data ) { + tbufWriteUint64( buf, *(uint64_t*)(&data) ); +} + +void tbufWriteDoubleAt( SBufferWriter* buf, size_t pos, double data ) { + tbufWriteUint64At( buf, pos, *(uint64_t*)(&data) ); } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 6101d4471b..dfbdc5e35d 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -249,7 +249,7 @@ void taosReadGlobalLogCfg() { } wordfree(&full_path); - taosReadLogOption("logDir", logDir); + taosReadLogOption("tsLogDir", tsLogDir); sprintf(fileName, "%s/taos.cfg", configDir); fp = fopen(fileName, "r"); diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 64ccfbc9da..47521fc36e 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -66,7 +66,7 @@ int32_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 0.1; -char logDir[TSDB_FILENAME_LEN] = "/var/log/taos"; +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos"; static SLogObj tsLogObj = { .fileNum = 1 }; static void * taosAsyncOutputLog(void *param); @@ -298,7 +298,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { void taosPrintLog(const char *const flags, int32_t dflag, const char *const format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); fflush(stdout); return; } @@ -356,7 +356,7 @@ void taosPrintLog(const char *const flags, int32_t dflag, const char *const form void taosDumpData(unsigned char *msg, int32_t len) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); fflush(stdout); return; } @@ -385,7 +385,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { void taosPrintLongString(const char *const flags, int32_t dflag, const char *const format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", logDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); fflush(stdout); return; } diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 9e0f6597ae..91d586322d 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -27,10 +27,10 @@ void taosInitNote(int numOfNoteLines, int maxNotes, char* lable) if (strcasecmp(lable, "http_note") == 0) { pNote = &m_HttpNote; - sprintf(temp, "%s/httpnote", logDir); + sprintf(temp, "%s/httpnote", tsLogDir); } else if (strcasecmp(lable, "tsc_note") == 0) { pNote = &m_TscNote; - sprintf(temp, "%s/tscnote-%d", logDir, getpid()); + sprintf(temp, "%s/tscnote-%d", tsLogDir, getpid()); } else { return; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 4d3687e50d..9fe306921d 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -19,89 +19,23 @@ #include "tsocket.h" #include "tutil.h" -/* - * Function to get the public ip address of current machine. If get IP - * successfully, return 0, else, return -1. The return values is ip. - * - * Use: - * if (taosGetPublicIp(ip) != 0) { - * perror("Fail to get public IP address\n"); - * exit(EXIT_FAILURE); - * } - */ -int taosGetPublicIp(char *const ip) { - /* bool flag; */ - int flag; - int sock; - char ** pptr = NULL; - struct sockaddr_in destAddr; - struct hostent * ptr = NULL; - char destIP[128]; - char szBuffer[] = { - "GET / HTTP/1.1\nHost: ident.me\nUser-Agent: curl/7.47.0\nAccept: " - "*/*\n\n"}; - char res[1024]; - - // Create socket - sock = (int)socket(AF_INET, SOCK_STREAM, 0); - if (sock == -1) { - return -1; - } - - bzero((void *)&destAddr, sizeof(destAddr)); - destAddr.sin_family = AF_INET; - destAddr.sin_port = htons(80); - - ptr = gethostbyname("ident.me"); - if (ptr == NULL) { - return -1; - } - - // Loop to find a valid IP address - for (flag = 0, pptr = ptr->h_addr_list; NULL != *pptr; ++pptr) { - inet_ntop(ptr->h_addrtype, *pptr, destIP, sizeof(destIP)); - destAddr.sin_addr.s_addr = inet_addr(destIP); - if (connect(sock, (struct sockaddr *)&destAddr, sizeof(struct sockaddr)) != -1) { - flag = 1; - break; - } - } - - // Check if the host is available. - if (flag == 0) { - return -1; - } - - // Check send. - if (strlen(szBuffer) != taosWriteSocket(sock, szBuffer, (size_t)strlen(szBuffer))) { - return -1; - } - - // Receive response. - if (taosReadSocket(sock, res, 1024) == -1) { - return -1; - } - - // Extract the IP address from the response. - int c_start = 0, c_end = 0; - for (; c_start < (int)strlen(res); c_start = c_end + 1) { - for (c_end = c_start; c_end < (int)strlen(res) && res[c_end] != '\n'; c_end++) { - } - - if (c_end >= (int)strlen(res)) { - return -1; - } - - if (res[c_start] >= '0' && res[c_start] <= '9') { - strncpy(ip, res + c_start, (size_t)(c_end - c_start)); - ip[c_end - c_start] = '\0'; - break; - } - } +int taosGetFqdn(char *fqdn) { + char hostname[1024]; + hostname[1023] = '\0'; + gethostname(hostname, 1023); + struct hostent* h; + h = gethostbyname(hostname); + strcpy(fqdn, h->h_name); return 0; } +uint32_t taosGetIpFromFqdn(const char *fqdn) { + struct hostent * record = gethostbyname(fqdn); + if(record == NULL) return -1; + return ((struct in_addr *)record->h_addr)->s_addr; +} + // Function converting an IP address string to an unsigned int. uint32_t ip2uint(const char *const ip_addr) { char ip_addr_cpy[20]; @@ -259,7 +193,7 @@ int taosReadn(int fd, char *ptr, int nbytes) { return (nbytes - nleft); } -int taosOpenUdpSocket(char *ip, uint16_t port) { +int taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; int sockFd; int ttl = 128; @@ -270,7 +204,7 @@ int taosOpenUdpSocket(char *ip, uint16_t port) { memset((char *)&localAddr, 0, sizeof(localAddr)); localAddr.sin_family = AF_INET; - localAddr.sin_addr.s_addr = inet_addr(ip); + localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) < 0) { @@ -325,13 +259,11 @@ int taosOpenUdpSocket(char *ip, uint16_t port) { return sockFd; } -int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) { +int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { int sockFd = 0; struct sockaddr_in serverAddr, clientAddr; int ret; - // uTrace("open tcp client socket:%s:%d, local Ip:%s", destIp, destPort, clientIp); - sockFd = (int)socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); if (sockFd < 0) { @@ -339,16 +271,16 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) { return -1; } - if (clientIp && clientIp[0] && clientIp[0] != '0') { + if ( clientIp != 0) { memset((char *)&clientAddr, 0, sizeof(clientAddr)); clientAddr.sin_family = AF_INET; - clientAddr.sin_addr.s_addr = inet_addr(clientIp); + clientAddr.sin_addr.s_addr = clientIp; clientAddr.sin_port = 0; /* bind socket to client address */ if (bind(sockFd, (struct sockaddr *)&clientAddr, sizeof(clientAddr)) < 0) { - uError("bind tcp client socket failed, client(%s:0), dest(%s:%d), reason:%d(%s)", - clientIp, destIp, destPort, errno, strerror(errno)); + uError("bind tcp client socket failed, client(0x%x:0), dest(0x%x:%d), reason:(%s)", + clientIp, destIp, destPort, strerror(errno)); close(sockFd); return -1; } @@ -356,13 +288,13 @@ int taosOpenTcpClientSocket(char *destIp, uint16_t destPort, char *clientIp) { memset((char *)&serverAddr, 0, sizeof(serverAddr)); serverAddr.sin_family = AF_INET; - serverAddr.sin_addr.s_addr = inet_addr(destIp); + serverAddr.sin_addr.s_addr = destIp; serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); if (ret != 0) { - //uError("failed to connect socket, ip:%s, port:%hu, reason: %s", destIp, destPort, strerror(errno)); + //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); taosCloseSocket(sockFd); sockFd = -1; } @@ -420,7 +352,7 @@ int taosKeepTcpAlive(int sockFd) { return 0; } -int taosOpenTcpServerSocket(char *ip, uint16_t port) { +int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; int sockFd; int reuse; @@ -429,7 +361,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) { bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; - serverAdd.sin_addr.s_addr = inet_addr(ip); + serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { @@ -447,7 +379,7 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) { /* bind socket to server address */ if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { - uError("bind tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno)); + uError("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); close(sockFd); return -1; } @@ -455,14 +387,14 @@ int taosOpenTcpServerSocket(char *ip, uint16_t port) { if (taosKeepTcpAlive(sockFd) < 0) return -1; if (listen(sockFd, 10) < 0) { - uError("listen tcp server socket failed, %s:%hu, reason:%d(%s)", ip, port, errno, strerror(errno)); + uError("listen tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); return -1; } return sockFd; } -int taosOpenRawSocket(char *ip) { +int taosOpenRawSocket(uint32_t ip) { int fd, hold; struct sockaddr_in rawAdd; @@ -483,10 +415,10 @@ int taosOpenRawSocket(char *ip) { bzero((char *)&rawAdd, sizeof(rawAdd)); rawAdd.sin_family = AF_INET; - rawAdd.sin_addr.s_addr = inet_addr(ip); + rawAdd.sin_addr.s_addr = ip; if (bind(fd, (struct sockaddr *)&rawAdd, sizeof(rawAdd)) < 0) { - uError("failed to bind RAW socket: %d (%s)", errno, strerror(errno)); + uError("failed to bind RAW socket:(%s)", strerror(errno)); close(fd); return -1; } diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index f1b9d55d54..d99cb1c96c 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -36,6 +36,7 @@ typedef struct { void *sync; void *events; void *cq; // continuous query + int32_t cfgVersion; STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 26791ae77c..2e31b72e8b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.maxCacheSize = pVnodeCfg->cfg.maxCacheSize; - + char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); @@ -198,7 +197,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; - sprintf(syncInfo.path, "%s/tsdb/", rootDir); + sprintf(syncInfo.path, "%s", rootDir); syncInfo.ahandle = pVnode; syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; @@ -287,7 +286,7 @@ void *vnodeGetVnode(int32_t vgId) { SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_INVALID_VGROUP_ID; - dError("vgId:%d not exist", vgId); + dPrint("vgId:%d not exist", vgId); return NULL; } @@ -312,7 +311,7 @@ void *vnodeGetWqueue(int32_t vgId) { SVnodeObj *pVnode = vnodeAccquireVnode(vgId); if (pVnode == NULL) return NULL; return pVnode->wqueue; -} +} void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; @@ -332,6 +331,7 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); + pLoad->cfgVersion = htonl(pVnode->cfgVersion); pLoad->totalStorage = htobe64(pLoad->totalStorage); pLoad->compStorage = htobe64(pLoad->compStorage); pLoad->pointsWritten = htobe64(pLoad->pointsWritten); @@ -341,10 +341,13 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { } static void vnodeCleanUp(SVnodeObj *pVnode) { - taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); - //syncStop(pVnode->sync); + if (pVnode->sync) { + syncStop(pVnode->sync); + pVnode->sync = NULL; + } + tsdbCloseRepo(pVnode->tsdb); walClose(pVnode->wal); vnodeSaveVersion(pVnode); @@ -379,46 +382,39 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnodeCfg->cfg.vgId); FILE *fp = fopen(cfgFile, "w"); if (!fp) { - dError("vgId:%d, failed to open vnode cfg file for write, error:%s", pVnodeCfg->cfg.vgId, strerror(errno)); + dError("vgId:%d, failed to open vnode cfg file for write, file:%s error:%s", pVnodeCfg->cfg.vgId, cfgFile, + strerror(errno)); return errno; } - char ipStr[20]; int32_t len = 0; int32_t maxLen = 1000; char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - - len += snprintf(content + len, maxLen - len, " \"maxCacheSize\": %" PRId64 ",\n", pVnodeCfg->cfg.maxCacheSize); - + len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"commitLog\": %d,\n", pVnodeCfg->cfg.commitLog); - len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - - uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp; - sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); - len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr); - - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - + len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); - - uint32_t ipInt = pVnodeCfg->nodes[i].nodeIp; - sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); - len += snprintf(content + len, maxLen - len, " \"nodeIp\": \"%s\",\n", ipStr); - - len += snprintf(content + len, maxLen - len, " \"nodeName\": \"%s\"\n", pVnodeCfg->nodes[i].nodeName); + len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", pVnodeCfg->nodes[i].nodeEp); if (i < pVnodeCfg->cfg.replications - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); @@ -442,7 +438,8 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { sprintf(cfgFile, "%s/vnode%d/config.json", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(cfgFile, "r"); if (!fp) { - dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, error:%s", pVnode, pVnode->vgId, strerror(errno)); + dError("pVnode:%p vgId:%d, failed to open vnode cfg file for read, file:%s, error:%s", pVnode, pVnode->vgId, + cfgFile, strerror(errno)); return errno; } @@ -463,19 +460,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!cfgVersion || cfgVersion->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + pVnode->cfgVersion = cfgVersion->valueint; - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; + pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; + + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); if (!maxTables || maxTables->type != cJSON_Number) { @@ -484,13 +488,34 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxTables = maxTables->valueint; - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode, pVnode->vgId); goto PARSE_OVER; } pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode, pVnode->vgId); @@ -505,19 +530,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + if (!commitTime || commitTime->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, commitTime not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.keep = daysToKeep->valueint; + pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - cJSON *maxCacheSize = cJSON_GetObjectItem(root, "maxCacheSize"); - if (!maxCacheSize || maxCacheSize->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, maxCacheSize not found", pVnode, pVnode->vgId); + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.maxCacheSize = maxCacheSize->valueint; + pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.compression = (int8_t)compression->valueint; cJSON *commitLog = cJSON_GetObjectItem(root, "commitLog"); if (!commitLog || commitLog->type != cJSON_Number) { @@ -534,12 +566,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { pVnode->walCfg.wals = (int8_t)wals->valueint; pVnode->walCfg.keep = 0; - cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); - if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); + pVnode->syncCfg.replica = (int8_t)replica->valueint; cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -548,13 +580,6 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); - goto PARSE_OVER; - } - pVnode->syncCfg.replica = (int8_t)replica->valueint; - cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeInfos not found", pVnode, pVnode->vgId); @@ -578,27 +603,22 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->syncCfg.nodeInfo[i].nodeId = nodeId->valueint; - cJSON *nodeIp = cJSON_GetObjectItem(nodeInfo, "nodeIp"); - if (!nodeIp || nodeIp->type != cJSON_String || nodeIp->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeIp not found", pVnode, pVnode->vgId); + cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); + if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeFqdn not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.nodeInfo[i].nodeIp = inet_addr(nodeIp->valuestring); - cJSON *nodeName = cJSON_GetObjectItem(nodeInfo, "nodeName"); - if (!nodeName || nodeName->type != cJSON_String || nodeName->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, nodeName not found", pVnode, pVnode->vgId); - goto PARSE_OVER; - } - strncpy(pVnode->syncCfg.nodeInfo[i].name, nodeName->valuestring, TSDB_NODE_NAME_LEN); + taosGetFqdnPortFromEp(nodeEp->valuestring, pVnode->syncCfg.nodeInfo[i].nodeFqdn, &pVnode->syncCfg.nodeInfo[i].nodePort); + pVnode->syncCfg.nodeInfo[i].nodePort += TSDB_PORT_SYNC; } ret = 0; dPrint("pVnode:%p vgId:%d, read vnode cfg successed, replcia:%d", pVnode, pVnode->vgId, pVnode->syncCfg.replica); for (int32_t i = 0; i < pVnode->syncCfg.replica; i++) { - dPrint("pVnode:%p vgId:%d, dnode:%d, ip:%s name:%s", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId, - taosIpStr(pVnode->syncCfg.nodeInfo[i].nodeIp), pVnode->syncCfg.nodeInfo[i].name); + dPrint("pVnode:%p vgId:%d, dnode:%d, %s:%d", pVnode, pVnode->vgId, pVnode->syncCfg.nodeInfo[i].nodeId, + pVnode->syncCfg.nodeInfo[i].nodeFqdn, pVnode->syncCfg.nodeInfo[i].nodePort); } PARSE_OVER: @@ -608,13 +628,13 @@ PARSE_OVER: return ret; } - static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { char versionFile[TSDB_FILENAME_LEN + 30] = {0}; sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); FILE *fp = fopen(versionFile, "w"); if (!fp) { - dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno)); + dError("pVnode:%p vgId:%d, failed to open vnode version file for write, file:%s error:%s", pVnode, pVnode->vgId, + versionFile, strerror(errno)); return errno; } @@ -630,7 +650,7 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { fclose(fp); free(content); - dPrint("pVnode:%p vgId:%d, save vnode version successed", pVnode, pVnode->vgId); + dPrint("pVnode:%p vgId:%d, save vnode version:%" PRId64 " successed", pVnode, pVnode->vgId, pVnode->version); return 0; } @@ -638,9 +658,10 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { static bool vnodeReadVersion(SVnodeObj *pVnode) { char versionFile[TSDB_FILENAME_LEN + 30] = {0}; sprintf(versionFile, "%s/vnode%d/version.json", tsVnodeDir, pVnode->vgId); - FILE *fp = fopen(versionFile, "w"); + FILE *fp = fopen(versionFile, "r"); if (!fp) { - dError("pVnode:%p vgId:%d, failed to open vnode version file for write, error:%s", pVnode, pVnode->vgId, strerror(errno)); + dTrace("pVnode:%p vgId:%d, failed to open version file:%s error:%s", pVnode, pVnode->vgId, + versionFile, strerror(errno)); return false; } @@ -677,4 +698,4 @@ PARSE_OVER: cJSON_Delete(root); fclose(fp); return ret; -} \ No newline at end of file +} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 02743c3e5d..a7d1daf777 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -58,7 +58,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { return TSDB_CODE_NOT_ACTIVE_VNODE; if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) - return TSDB_CODE_NO_MASTER; + return TSDB_CODE_NOT_READY; // assign version pVnode->version++; @@ -74,15 +74,15 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // write into WAL code = walWrite(pVnode->wal, pHead); - if ( code < 0) return code; - + if (code < 0) return code; + + int32_t syncCode = syncForwardToPeer(pVnode->sync, pHead, item); + if (syncCode < 0) return syncCode; + code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); if (code < 0) return code; - if (pVnode->syncCfg.replica > 1) - code = syncForwardToPeer(pVnode->sync, pHead, item); - - return code; + return syncCode; } static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { @@ -225,11 +225,10 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId); - // int64_t uid = htobe64(pTable->uid); - // TODO: drop stable in vvnode + //int64_t uid = htobe64(pTable->uid); //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); - //rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid); + //rpcRsp.code = tsdbDropTable(pTsdb, pTable->uid); code = TSDB_CODE_SUCCESS; dTrace("pVnode:%p vgId:%d, stable:%s, drop stable result:%x", pVnode, pTable->tableId, code); diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index 6a4ecee1c3..ad33391b98 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -269,7 +269,7 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) { if (*index < first && *index > pWal->id) { code = -1; // index out of range } else { - sprintf(name, "%s/%s%d", pWal->path, walPrefix, *index); + sprintf(name, "wal/%s%d", walPrefix, *index); code = (*index == pWal->id) ? 0:1; } diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index e5a70740f1..6bade385bd 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,7 +75,9 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); - doQuery(taos, "select count(*),k,sum(k) from m1 group by k"); +// doQuery(taos, "select a from m1"); +// doQuery(taos, "select m2.u from m1, m2 where m1.ts=m2.ts and m1.a=m2.b;"); + doQuery(taos, "select last_row(ts) from tm99"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh new file mode 100755 index 0000000000..ec06eb38ef --- /dev/null +++ b/tests/pytest/fulltest.sh @@ -0,0 +1,77 @@ +#!/bin/bash +python3 ./test.py $1 -f insert/basic.py +python3 ./test.py $1 -f insert/int.py +python3 ./test.py $1 -f insert/float.py +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py $1 -f insert/bool.py +python3 ./test.py $1 -f insert/double.py +python3 ./test.py $1 -f insert/smallint.py +python3 ./test.py $1 -f insert/tinyint.py +python3 ./test.py $1 -f insert/date.py +python3 ./test.py $1 -f insert/binary.py +python3 ./test.py $1 -f import_merge/importBlock1HO.py +python3 ./test.py $1 -f import_merge/importBlock1HPO.py +python3 ./test.py $1 -f import_merge/importBlock1H.py +python3 ./test.py $1 -f import_merge/importBlock1S.py +python3 ./test.py $1 -f import_merge/importBlock1Sub.py +python3 ./test.py $1 -f import_merge/importBlock1TO.py +python3 ./test.py $1 -f import_merge/importBlock1TPO.py +python3 ./test.py $1 -f import_merge/importBlock1T.py +python3 ./test.py $1 -f import_merge/importBlock2HO.py +python3 ./test.py $1 -f import_merge/importBlock2HPO.py +python3 ./test.py $1 -f import_merge/importBlock2H.py +python3 ./test.py $1 -f import_merge/importBlock2S.py +python3 ./test.py $1 -f import_merge/importBlock2Sub.py +python3 ./test.py $1 -f import_merge/importBlock2TO.py +python3 ./test.py $1 -f import_merge/importBlock2TPO.py +python3 ./test.py $1 -f import_merge/importBlock2T.py +python3 ./test.py $1 -f import_merge/importBlockbetween.py +python3 ./test.py $1 -f import_merge/importCacheFileHO.py +python3 ./test.py $1 -f import_merge/importCacheFileHPO.py +python3 ./test.py $1 -f import_merge/importCacheFileH.py +python3 ./test.py $1 -f import_merge/importCacheFileS.py +python3 ./test.py $1 -f import_merge/importCacheFileSub.py +python3 ./test.py $1 -f import_merge/importCacheFileTO.py +python3 ./test.py $1 -f import_merge/importCacheFileTPO.py +python3 ./test.py $1 -f import_merge/importCacheFileT.py +python3 ./test.py $1 -f import_merge/importDataH2.py +python3 ./test.py $1 -f import_merge/importDataHO2.py +python3 ./test.py $1 -f import_merge/importDataHO.py +python3 ./test.py $1 -f import_merge/importDataHPO.py +python3 ./test.py $1 -f import_merge/importDataLastHO.py +python3 ./test.py $1 -f import_merge/importDataLastHPO.py +python3 ./test.py $1 -f import_merge/importDataLastH.py +python3 ./test.py $1 -f import_merge/importDataLastS.py +python3 ./test.py $1 -f import_merge/importDataLastSub.py +python3 ./test.py $1 -f import_merge/importDataLastTO.py +python3 ./test.py $1 -f import_merge/importDataLastTPO.py +python3 ./test.py $1 -f import_merge/importDataLastT.py +python3 ./test.py $1 -f import_merge/importDataS.py +python3 ./test.py $1 -f import_merge/importDataSub.py +python3 ./test.py $1 -f import_merge/importDataTO.py +python3 ./test.py $1 -f import_merge/importDataTPO.py +python3 ./test.py $1 -f import_merge/importDataT.py +python3 ./test.py $1 -f import_merge/importHeadOverlap.py +python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py +python3 ./test.py $1 -f import_merge/importHead.py +python3 ./test.py $1 -f import_merge/importHORestart.py +python3 ./test.py $1 -f import_merge/importHPORestart.py +python3 ./test.py $1 -f import_merge/importHRestart.py +python3 ./test.py $1 -f import_merge/importLastHO.py +python3 ./test.py $1 -f import_merge/importLastHPO.py +python3 ./test.py $1 -f import_merge/importLastH.py +python3 ./test.py $1 -f import_merge/importLastS.py +python3 ./test.py $1 -f import_merge/importLastSub.py +python3 ./test.py $1 -f import_merge/importLastTO.py +python3 ./test.py $1 -f import_merge/importLastTPO.py +python3 ./test.py $1 -f import_merge/importLastT.py +python3 ./test.py $1 -f import_merge/importSpan.py +python3 ./test.py $1 -f import_merge/importSRestart.py +python3 ./test.py $1 -f import_merge/importSubRestart.py +python3 ./test.py $1 -f import_merge/importTailOverlap.py +python3 ./test.py $1 -f import_merge/importTailPartOverlap.py +python3 ./test.py $1 -f import_merge/importTail.py +python3 ./test.py $1 -f import_merge/importToCommit.py +python3 ./test.py $1 -f import_merge/importTORestart.py +python3 ./test.py $1 -f import_merge/importTPORestart.py +python3 ./test.py $1 -f import_merge/importTRestart.py diff --git a/tests/pytest/import_merge/__init__.py b/tests/pytest/import_merge/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/import_merge/importBlock1H.py b/tests/pytest/import_merge/importBlock1H.py new file mode 100644 index 0000000000..a1ba905b17 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1H.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(39) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1HO.py b/tests/pytest/import_merge/importBlock1HO.py new file mode 100644 index 0000000000..73aec07a90 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1HO.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 10 data before with overlap") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(43) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1HPO.py b/tests/pytest/import_merge/importBlock1HPO.py new file mode 100644 index 0000000000..ad224e5c65 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1HPO.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(15, 43): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 20 data before with partly overlap") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(47) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1S.py b/tests/pytest/import_merge/importBlock1S.py new file mode 100644 index 0000000000..37c2ad6631 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1S.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 50 data covering existing data") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 51): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(50) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1Sub.py b/tests/pytest/import_merge/importBlock1Sub.py new file mode 100644 index 0000000000..5228563651 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1Sub.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1T.py b/tests/pytest/import_merge/importBlock1T.py new file mode 100644 index 0000000000..75f41b98cf --- /dev/null +++ b/tests/pytest/import_merge/importBlock1T.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 1 data after") + startTime = self.startTime + 38 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(39) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1TO.py b/tests/pytest/import_merge/importBlock1TO.py new file mode 100644 index 0000000000..b43428da27 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1TO.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 10 data later with overlap") + startTime = self.startTime + 30 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(40) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock1TPO.py b/tests/pytest/import_merge/importBlock1TPO.py new file mode 100644 index 0000000000..913ca1cc02 --- /dev/null +++ b/tests/pytest/import_merge/importBlock1TPO.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 38 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 31): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(35, 43): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(38) + + tdLog.info("================= step4") + tdLog.info("import 30 data later with partly overlap") + startTime = self.startTime + 25 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 31): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(55) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2H.py b/tests/pytest/import_merge/importBlock2H.py new file mode 100644 index 0000000000..bacd88cbe7 --- /dev/null +++ b/tests/pytest/import_merge/importBlock2H.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(77) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2HO.py b/tests/pytest/import_merge/importBlock2HO.py new file mode 100644 index 0000000000..01c0f622b6 --- /dev/null +++ b/tests/pytest/import_merge/importBlock2HO.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 10 data before with overlap") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2HPO.py b/tests/pytest/import_merge/importBlock2HPO.py new file mode 100644 index 0000000000..ee8d580dfe --- /dev/null +++ b/tests/pytest/import_merge/importBlock2HPO.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(15, 81): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 20 data before with partly overlap") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(85) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2S.py b/tests/pytest/import_merge/importBlock2S.py new file mode 100644 index 0000000000..d85074bfeb --- /dev/null +++ b/tests/pytest/import_merge/importBlock2S.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 90 data covering existing data") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 91): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(90) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2Sub.py b/tests/pytest/import_merge/importBlock2Sub.py new file mode 100644 index 0000000000..deb1dc8337 --- /dev/null +++ b/tests/pytest/import_merge/importBlock2Sub.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2T.py b/tests/pytest/import_merge/importBlock2T.py new file mode 100644 index 0000000000..ded698d28c --- /dev/null +++ b/tests/pytest/import_merge/importBlock2T.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 1 data after") + startTime = self.startTime + 76 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(77) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2TO.py b/tests/pytest/import_merge/importBlock2TO.py new file mode 100644 index 0000000000..ffc88c2c99 --- /dev/null +++ b/tests/pytest/import_merge/importBlock2TO.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 77): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 10 data later with overlap") + startTime = self.startTime + 70 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(80) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlock2TPO.py b/tests/pytest/import_merge/importBlock2TPO.py new file mode 100644 index 0000000000..8b6c70c32b --- /dev/null +++ b/tests/pytest/import_merge/importBlock2TPO.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 61): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(65, 81): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import 30 data later with partly overlap") + startTime = self.startTime + 55 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 31): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(85) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importBlockbetween.py b/tests/pytest/import_merge/importBlockbetween.py new file mode 100644 index 0000000000..c3482b3776 --- /dev/null +++ b/tests/pytest/import_merge/importBlockbetween.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("one block can import 38 records") + + tdLog.info("================= step2") + tdLog.info("import 76 sequential data with gap between 2 blocks") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 39): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + for rid in range(40, 78): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(76) + + tdLog.info("================= step4") + tdLog.info("import data into the gap between 2 blocks") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(39, 40): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(77) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileH.py b/tests/pytest/import_merge/importCacheFileH.py new file mode 100644 index 0000000000..cd2b3a73f1 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileH.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(21) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileHO.py b/tests/pytest/import_merge/importCacheFileHO.py new file mode 100644 index 0000000000..2e65c337b7 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileHO.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 10 data before with overlap") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(25) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileHPO.py b/tests/pytest/import_merge/importCacheFileHPO.py new file mode 100644 index 0000000000..f01ebd0d60 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileHPO.py @@ -0,0 +1,95 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 9 sequential data with gap") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + if (rid == 5): + continue + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(9) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 9 data again with gap") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + if (rid == 5): + continue + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(18) + + tdLog.info("================= step7") + tdLog.info("import 20 data before with partly overlap") + startTime = self.startTime - 3 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(23) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileS.py b/tests/pytest/import_merge/importCacheFileS.py new file mode 100644 index 0000000000..0bb9107562 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileS.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 30 data covering existing data") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 31): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(30) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileSub.py b/tests/pytest/import_merge/importCacheFileSub.py new file mode 100644 index 0000000000..cd5d250968 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileSub.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileT.py b/tests/pytest/import_merge/importCacheFileT.py new file mode 100644 index 0000000000..be79e26bc7 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileT.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 1 data later") + startTime = self.startTime + 20 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, rid)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(21) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileTO.py b/tests/pytest/import_merge/importCacheFileTO.py new file mode 100644 index 0000000000..dd17de3adf --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileTO.py @@ -0,0 +1,91 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data again") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step7") + tdLog.info("import 10 data later with overlap") + startTime = self.startTime + 15 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(25) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importCacheFileTPO.py b/tests/pytest/import_merge/importCacheFileTPO.py new file mode 100644 index 0000000000..948b99ed21 --- /dev/null +++ b/tests/pytest/import_merge/importCacheFileTPO.py @@ -0,0 +1,95 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 9 sequential data with gap") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + if (rid == 7): + continue + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(9) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 9 data again with gap") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + if (rid == 7): + continue + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(18) + + tdLog.info("================= step7") + tdLog.info("import 20 data later with partly overlap") + startTime = self.startTime + 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step8") + tdSql.query('select * from tb1') + tdSql.checkRows(25) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py new file mode 100644 index 0000000000..d49abff374 --- /dev/null +++ b/tests/pytest/import_merge/importDataH2.py @@ -0,0 +1,94 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import %d sequential data" % (self.rows / 2)) + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, self.rows / 2 + 1): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(self.rows / 2) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step7") + tdSql.execute('reset query cache') + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(self.rows / 2 + 1) + + tdLog.info("================= step8") + tdLog.info("import 10 data in batch before") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime - rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step9") + tdSql.execute('reset query cache') + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(self.rows / 2 + 11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py new file mode 100644 index 0000000000..0483e6844c --- /dev/null +++ b/tests/pytest/import_merge/importDataHO.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + self.rowsPerTable = 20 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import %d sequential data" % self.rowsPerTable) + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, self.rowsPerTable + 1): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, self.rowsPerTable) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 6 data before with overlap") + startTime = self.startTime - 3 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 7): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(self.rowsPerTable + 3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataHO2.py b/tests/pytest/import_merge/importDataHO2.py new file mode 100644 index 0000000000..ab7044d2a7 --- /dev/null +++ b/tests/pytest/import_merge/importDataHO2.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + self.rowsPerTable = 100 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import %d sequential data" % self.rowsPerTable) + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, self.rowsPerTable + 1): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, self.rowsPerTable) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 6 data before with overlap") + startTime = self.startTime - 3 + sqlcmd = ['import into tb1 values'] + for rid in range(6, 0, -1): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(self.rowsPerTable + 3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataHPO.py b/tests/pytest/import_merge/importDataHPO.py new file mode 100644 index 0000000000..f165bd7b5a --- /dev/null +++ b/tests/pytest/import_merge/importDataHPO.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + self.rowsPerTable = 20 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import %d sequential data" % self.rowsPerTable) + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 10): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(14, self.rowsPerTable + 5): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, self.rowsPerTable) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 20 data before with partly overlap") + startTime = self.startTime - 4 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(self.rowsPerTable + 8) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastH.py b/tests/pytest/import_merge/importDataLastH.py new file mode 100644 index 0000000000..319fd40677 --- /dev/null +++ b/tests/pytest/import_merge/importDataLastH.py @@ -0,0 +1,80 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(206) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastHO.py b/tests/pytest/import_merge/importDataLastHO.py new file mode 100644 index 0000000000..5a71c5db65 --- /dev/null +++ b/tests/pytest/import_merge/importDataLastHO.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 5 data before with overlap") + startTime = self.startTime - 2 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(207) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastHPO.py b/tests/pytest/import_merge/importDataLastHPO.py new file mode 100644 index 0000000000..f2c95cbd4d --- /dev/null +++ b/tests/pytest/import_merge/importDataLastHPO.py @@ -0,0 +1,85 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(14, 209): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 20 data before with partly overlap") + startTime = self.startTime - 2 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(210) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py new file mode 100644 index 0000000000..929e02dd1e --- /dev/null +++ b/tests/pytest/import_merge/importDataLastS.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 250 data covering the existing data") + startTime = self.startTime - 15 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 251): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(250) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py new file mode 100644 index 0000000000..158fa0fb3c --- /dev/null +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastT.py b/tests/pytest/import_merge/importDataLastT.py new file mode 100644 index 0000000000..9bc90a8275 --- /dev/null +++ b/tests/pytest/import_merge/importDataLastT.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data later") + startTime = self.startTime + 205 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(206) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastTO.py b/tests/pytest/import_merge/importDataLastTO.py new file mode 100644 index 0000000000..0c93ac430b --- /dev/null +++ b/tests/pytest/import_merge/importDataLastTO.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 206): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 5 data later with overlap") + startTime = self.startTime + 203 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(208) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataLastTPO.py b/tests/pytest/import_merge/importDataLastTPO.py new file mode 100644 index 0000000000..188e93e0db --- /dev/null +++ b/tests/pytest/import_merge/importDataLastTPO.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than %d rows less than %d rows will go to data and last file" % + (self.rows, 10 + self.rows)) + + tdLog.info("================= step2") + tdLog.info("import 205 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 196): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(200, 210): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(205) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 20 data later with partly overlap") + startTime = self.startTime + 192 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(212) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataS.py b/tests/pytest/import_merge/importDataS.py new file mode 100644 index 0000000000..65d4087c3d --- /dev/null +++ b/tests/pytest/import_merge/importDataS.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import 20 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 30 data covering the existing data") + startTime = self.startTime - 5 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 31): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(30) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py new file mode 100644 index 0000000000..4bf85f2bdd --- /dev/null +++ b/tests/pytest/import_merge/importDataSub.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import %d sequential data" % (self.rows / 2)) + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, self.rows / 2 + 1): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(self.rows / 2) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + 10 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 11): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step9") + tdSql.execute('reset query cache') + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(self.rows / 2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataT.py b/tests/pytest/import_merge/importDataT.py new file mode 100644 index 0000000000..66016c5555 --- /dev/null +++ b/tests/pytest/import_merge/importDataT.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import 20 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data later") + startTime = self.startTime + 20 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(21) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataTO.py b/tests/pytest/import_merge/importDataTO.py new file mode 100644 index 0000000000..a3c17b2846 --- /dev/null +++ b/tests/pytest/import_merge/importDataTO.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import 20 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 6 data later with overlap") + startTime = self.startTime + 18 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 7): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(24) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importDataTPO.py b/tests/pytest/import_merge/importDataTPO.py new file mode 100644 index 0000000000..20eb41cc08 --- /dev/null +++ b/tests/pytest/import_merge/importDataTPO.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info( + "More than 10 rows less than %d rows will go to data file" % + self.rows) + + tdLog.info("================= step2") + tdLog.info("import 20 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 18): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(22, 25): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + tdLog.sleep(5) + + tdLog.info("================= step5") + tdLog.info("import 20 data later with partly overlap") + startTime = self.startTime + 15 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(35) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHORestart.py b/tests/pytest/import_merge/importHORestart.py new file mode 100644 index 0000000000..cfbfa61c90 --- /dev/null +++ b/tests/pytest/import_merge/importHORestart.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdSql.prepare() + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 5 data before with overlap") + startTime = self.startTime - 2 + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(12) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHPORestart.py b/tests/pytest/import_merge/importHPORestart.py new file mode 100644 index 0000000000..7e96d44a1a --- /dev/null +++ b/tests/pytest/import_merge/importHPORestart.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdSql.prepare() + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 8 sequential data with gap") + startTime = self.startTime + for rid in range(1, 4): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + for rid in range(6, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(8) + + tdLog.info("================= step4") + tdLog.info("import 8 data before with partly overlap") + startTime = self.startTime - 2 + for rid in range(1, 9): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(12) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHRestart.py b/tests/pytest/import_merge/importHRestart.py new file mode 100644 index 0000000000..aa1783977e --- /dev/null +++ b/tests/pytest/import_merge/importHRestart.py @@ -0,0 +1,69 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdSql.prepare() + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHead.py b/tests/pytest/import_merge/importHead.py new file mode 100644 index 0000000000..6971986ebc --- /dev/null +++ b/tests/pytest/import_merge/importHead.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHeadOverlap.py b/tests/pytest/import_merge/importHeadOverlap.py new file mode 100644 index 0000000000..df5f07b5a2 --- /dev/null +++ b/tests/pytest/import_merge/importHeadOverlap.py @@ -0,0 +1,65 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdSql.prepare() + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 5 data before with overlap") + startTime = self.startTime - 2 + for rid in range(1, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(12) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importHeadPartOverlap.py b/tests/pytest/import_merge/importHeadPartOverlap.py new file mode 100644 index 0000000000..8c9885e22f --- /dev/null +++ b/tests/pytest/import_merge/importHeadPartOverlap.py @@ -0,0 +1,69 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdSql.prepare() + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 8 sequential data with gap") + startTime = self.startTime + for rid in range(1, 4): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + startTime += 2 + for rid in range(6, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(8) + + tdLog.info("================= step4") + tdLog.info("import 8 data before with partly overlap") + startTime = self.startTime - 2 + for rid in range(1, 9): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(12) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastH.py b/tests/pytest/import_merge/importLastH.py new file mode 100644 index 0000000000..c69f453971 --- /dev/null +++ b/tests/pytest/import_merge/importLastH.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data before") + startTime = self.startTime - 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastHO.py b/tests/pytest/import_merge/importLastHO.py new file mode 100644 index 0000000000..ec930d1807 --- /dev/null +++ b/tests/pytest/import_merge/importLastHO.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 4 data before with overlap") + startTime = self.startTime - 2 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 5): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(7) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastHPO.py b/tests/pytest/import_merge/importLastHPO.py new file mode 100644 index 0000000000..9603a7b852 --- /dev/null +++ b/tests/pytest/import_merge/importLastHPO.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 6 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 4): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(6, 9): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(6) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 8 data before with partly overlap") + startTime = self.startTime - 2 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 9): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastS.py b/tests/pytest/import_merge/importLastS.py new file mode 100644 index 0000000000..7dbe74e2ca --- /dev/null +++ b/tests/pytest/import_merge/importLastS.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 20 data covering existing data") + startTime = self.startTime - 10 + for rid in range(1, 21): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastSub.py b/tests/pytest/import_merge/importLastSub.py new file mode 100644 index 0000000000..f028ba5fd7 --- /dev/null +++ b/tests/pytest/import_merge/importLastSub.py @@ -0,0 +1,77 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 3 data totally repetitive") + startTime = self.startTime + 1 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 4): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastT.py b/tests/pytest/import_merge/importLastT.py new file mode 100644 index 0000000000..3fe4e0006c --- /dev/null +++ b/tests/pytest/import_merge/importLastT.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 1 data later") + startTime = self.startTime + 5 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime + 1, 1)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastTO.py b/tests/pytest/import_merge/importLastTO.py new file mode 100644 index 0000000000..76e5016bdb --- /dev/null +++ b/tests/pytest/import_merge/importLastTO.py @@ -0,0 +1,81 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 5 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 6): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(5) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 4 data later with overlap") + startTime = self.startTime + 3 + sqlcmd = ['import into tb1 values'] + for rid in range(1, 5): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(7) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importLastTPO.py b/tests/pytest/import_merge/importLastTPO.py new file mode 100644 index 0000000000..08f4168063 --- /dev/null +++ b/tests/pytest/import_merge/importLastTPO.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + self.rows = 200 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db rows %d' % self.rows) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + tdLog.info("less than 10 rows will go to last file") + + tdLog.info("================= step2") + tdLog.info("import 6 sequential data") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(1, 4): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + for rid in range(6, 9): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(6) + + tdLog.info("================= step4") + tdDnodes.stop(1) + tdLog.sleep(5) + tdDnodes.start(1) + + tdLog.info("================= step5") + tdLog.info("import 8 data later with partly overlap") + startTime = self.startTime + 2 + for rid in range(1, 9): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importSRestart.py b/tests/pytest/import_merge/importSRestart.py new file mode 100644 index 0000000000..0771b8bf9c --- /dev/null +++ b/tests/pytest/import_merge/importSRestart.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 20 data cover existing data") + startTime = self.startTime - 5 + for rid in range(1, 21): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importSpan.py b/tests/pytest/import_merge/importSpan.py new file mode 100644 index 0000000000..736c4bad64 --- /dev/null +++ b/tests/pytest/import_merge/importSpan.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 20 data cover existing data") + startTime = self.startTime - 5 + for rid in range(1, 21): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importSubRestart.py b/tests/pytest/import_merge/importSubRestart.py new file mode 100644 index 0000000000..f7f33d32c1 --- /dev/null +++ b/tests/pytest/import_merge/importSubRestart.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 20 sequential data") + startTime = self.startTime + for rid in range(1, 21): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + tdLog.info("================= step4") + tdLog.info("import 10 data totally repetitive") + startTime = self.startTime + 5 + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(20) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTORestart.py b/tests/pytest/import_merge/importTORestart.py new file mode 100644 index 0000000000..194756cd12 --- /dev/null +++ b/tests/pytest/import_merge/importTORestart.py @@ -0,0 +1,79 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 6 data after with overlap") + startTime = self.startTime + 8 + for rid in range(1, 7): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(14) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTPORestart.py b/tests/pytest/import_merge/importTPORestart.py new file mode 100644 index 0000000000..36d4b64390 --- /dev/null +++ b/tests/pytest/import_merge/importTPORestart.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 8 sequential data with gap") + startTime = self.startTime + for rid in range(1, 6): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + for rid in range(8, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(8) + + tdLog.info("================= step4") + tdLog.info("import 8 data after with partly overlap") + startTime = self.startTime + 3 + for rid in range(1, 9): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTRestart.py b/tests/pytest/import_merge/importTRestart.py new file mode 100644 index 0000000000..9308518d8c --- /dev/null +++ b/tests/pytest/import_merge/importTRestart.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 1 data after") + startTime = self.startTime + 11 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdDnodes.forcestop(1) + tdDnodes.start(1) + tdLog.sleep(10) + + tdLog.info("================= step6") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTail.py b/tests/pytest/import_merge/importTail.py new file mode 100644 index 0000000000..a80db730a0 --- /dev/null +++ b/tests/pytest/import_merge/importTail.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 1 data after") + startTime += 1 + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTailOverlap.py b/tests/pytest/import_merge/importTailOverlap.py new file mode 100644 index 0000000000..98596d2f77 --- /dev/null +++ b/tests/pytest/import_merge/importTailOverlap.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 10 sequential data") + startTime = self.startTime + for rid in range(1, 11): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(10) + + tdLog.info("================= step4") + tdLog.info("import 6 data after with overlap") + startTime = self.startTime + 8 + for rid in range(1, 7): + tdSql.execute('import into tb1 values(%ld, %d)' % (startTime, rid)) + startTime += 1 + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(14) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importTailPartOverlap.py b/tests/pytest/import_merge/importTailPartOverlap.py new file mode 100644 index 0000000000..0263114a25 --- /dev/null +++ b/tests/pytest/import_merge/importTailPartOverlap.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, speed int)') + + tdLog.info("================= step2") + tdLog.info("import 8 sequential data with gap") + startTime = self.startTime + for rid in range(1, 6): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + for rid in range(8, 11): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step3") + tdSql.query('select * from tb1') + tdSql.checkRows(8) + + tdLog.info("================= step4") + tdLog.info("import 8 data after with partly overlap") + startTime = self.startTime + 3 + for rid in range(1, 9): + tdSql.execute( + 'import into tb1 values(%ld, %d)' % + (startTime + rid, rid)) + + tdLog.info("================= step5") + tdSql.query('select * from tb1') + tdSql.checkRows(11) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/import_merge/importToCommit.py b/tests/pytest/import_merge/importToCommit.py new file mode 100644 index 0000000000..7a408bcdce --- /dev/null +++ b/tests/pytest/import_merge/importToCommit.py @@ -0,0 +1,85 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 1 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db cache 512 tables 10') + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 table") + tdSql.execute('create table tb1 (ts timestamp, i int)') + tdLog.info( + "one block can import 38 records and totally there are 40 blocks") + + tdLog.info("================= step2") + tdLog.info('insert data until the first commit') + dnodesDir = tdDnodes.getDnodesRootDir() + dataDir = dnodesDir + '/dnode1/data/data' + startTime = self.startTime + rid0 = 1 + while (True): + sqlcmd = 'insert into tb1 values(%ld, %d)' % ( + startTime + rid0 * 2, rid0) + tdSql.execute(sqlcmd) + rid0 += 1 + vnodes = os.listdir(dataDir) + if (len(vnodes) > 0): + tdLog.info("data is committed, stop inserting") + break + + tdLog.info("================= step5") + tdLog.info("import 1 data before ") + startTime = self.startTime + sqlcmd = ['import into tb1 values'] + for rid in range(3, 4): + sqlcmd.append('(%ld, %d)' % (startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + + tdLog.info("================= step6") + tdSql.execute('reset query cache') + tdSql.query('select * from tb1 order by ts desc') + tdSql.checkRows(rid0 - 1 + 1) + + tdLog.info("================= step7") + tdSql.execute('reset query cache') + tdSql.query('select count(*) from tb1') + tdSql.checkData(0, 0, rid0 - 1 + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py index e8698e9d05..35f830d951 100644 --- a/tests/pytest/insert/basic.py +++ b/tests/pytest/insert/basic.py @@ -12,7 +12,6 @@ # -*- coding: utf-8 -*- import sys -import taos from util.log import * from util.cases import * from util.sql import * diff --git a/tests/pytest/insert/bigint.py b/tests/pytest/insert/bigint.py new file mode 100644 index 0000000000..214200900a --- /dev/null +++ b/tests/pytest/insert/bigint.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bigint)') + tdSql.execute('create table tb (ts timestamp, speed bigint)') + tdLog.info("insert into tb values (now, -9223372036854770000)") + tdSql.execute("insert into tb values (now, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1a, -9223372036854770000)") + tdSql.execute("insert into tb values (now+1a, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+2a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+3a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py new file mode 100644 index 0000000000..9989865f96 --- /dev/null +++ b/tests/pytest/insert/binary.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed binary(5))') + tdSql.execute('create table tb (ts timestamp, speed binary(5))') + tdLog.info("insert into tb values (now, ) -x step1") + tdSql.error("insert into tb values (now, )") + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1a, '1234')") + tdSql.execute("insert into tb values (now+1a, '1234')") + tdLog.info('select speed from tb order by ts desc') + tdSql.query('select speed from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 0, 1234)') + tdSql.checkData(0, 0, 1234) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, '23456')") + tdSql.execute("insert into tb values (now+2a, '23456')") + tdLog.info('select speed from tb order by ts desc') + tdSql.query('select speed from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('==> $data00') + tdLog.info('tdSql.checkData(0, 0, 23456)') + tdSql.checkData(0, 0, 23456) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, '345678')") + tdSql.error("insert into tb values (now+3a, '345678')") + tdLog.info("insert into tb values (now+3a, '34567')") + tdSql.execute("insert into tb values (now+3a, '34567')") + tdLog.info('select speed from tb order by ts desc') + tdSql.query('select speed from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('==> $data00') + tdLog.info('tdSql.checkData(0, 0, 34567)') + tdSql.checkData(0, 0, 34567) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/bool.py b/tests/pytest/insert/bool.py new file mode 100644 index 0000000000..062563f4ab --- /dev/null +++ b/tests/pytest/insert/bool.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bool)') + tdSql.execute('create table tb (ts timestamp, speed bool)') + tdLog.info("insert into tb values (now, true)") + tdSql.execute("insert into tb values (now, true)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, 1)") + tdSql.execute("insert into tb values (now+1m, 1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 2)") + tdSql.execute("insert into tb values (now+2m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 0)") + tdSql.execute("insert into tb values (now+3m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, -1)") + tdSql.execute("insert into tb values (now+4m, -1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, false)") + tdSql.execute("insert into tb values (now+5m, false)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/date.py b/tests/pytest/insert/date.py new file mode 100644 index 0000000000..716d799cb0 --- /dev/null +++ b/tests/pytest/insert/date.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: system sh/ip.sh -i 1 -s up + # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 + # TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: $i = 0 + # TSIM: $dbPrefix = lm_da_db + # TSIM: $tbPrefix = lm_da_tb + # TSIM: $db = $dbPrefix . $i + # TSIM: $tb = $tbPrefix . $i + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: + # TSIM: sql create table $tb (ts timestamp, speed int) + tdLog.info("create table tb0 (ts timestamp, speed int)") + tdSql.execute('create table tb0 (ts timestamp, speed int)') + # TSIM: sql insert into $tb values ('2017-01-01 08:00:00.001', 1) + tdLog.info("insert into tb0 values ('2017-01-01 08:00:00.001', 1)") + tdSql.execute("insert into tb0 values ('2017-01-01 08:00:00.001', 1)") + # TSIM: sql select ts from $tb + tdLog.info('select ts from tb0') + tdSql.query('select ts from tb0') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data00 != @17-01-01 08:00:00.001@ then + tdLog.info("tdSql.checkData(0, 0, 17-01-01 08:00:00.001)") + expectedData = datetime.datetime.strptime( + "17-01-01 08:00:00.001", "%y-%m-%d %H:%M:%S.%f") + tdSql.checkData(0, 0, expectedData) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sql insert into $tb values ('2017-08-28 00:23:46.429+ 1a', 2) + tdLog.info("insert into tb0 values ('2017-08-28 00:23:46.429+ 1a', 2)") + tdSql.execute( + "insert into tb0 values ('2017-08-28 00:23:46.429+ 1a', 2)") + # TSIM: #sql insert into $tb values ('2017-08-28 00:23:46cd .429', 2) + # TSIM: sql select ts from $tb + tdLog.info('select ts from tb0') + tdSql.query('select ts from tb0') + # TSIM: if $rows != 2 then + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: #sql insert into $tb values ('1970-01-01 08:00:00.000', 3) + # TSIM: #sql insert into $tb values ('1970-01-01 08:00:00.000', 3) + # TSIM: sql select ts from $tb + tdLog.info('select ts from tb0') + tdSql.query('select ts from tb0') + # TSIM: if $rows != 2 then + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql insert into $tb values(now, 4); + tdLog.info("insert into tb0 values(now, 4);") + tdSql.execute("insert into tb0 values(now, 4);") + # TSIM: sql insert into $tb values(now+1a, 5); + tdLog.info("insert into tb0 values(now+1a, 5);") + tdSql.execute("insert into tb0 values(now+1a, 5);") + # TSIM: sql insert into $tb values(now+1s, 6); + tdLog.info("insert into tb0 values(now+1s, 6);") + tdSql.execute("insert into tb0 values(now+1s, 6);") + # TSIM: sql insert into $tb values(now+1m, 7); + tdLog.info("insert into tb0 values(now+1m, 7);") + tdSql.execute("insert into tb0 values(now+1m, 7);") + # TSIM: sql insert into $tb values(now+1h, 8); + tdLog.info("insert into tb0 values(now+1h, 8);") + tdSql.execute("insert into tb0 values(now+1h, 8);") + # TSIM: sql insert into $tb values(now+1d, 9); + tdLog.info("insert into tb0 values(now+1d, 9);") + tdSql.execute("insert into tb0 values(now+1d, 9);") + # TSIM: sql_error insert into $tb values(now+3w, 10); + tdLog.info("insert into tb0 values(now+3w, 10);") + tdSql.error("insert into tb0 values(now+3w, 10);") + # TSIM: sql_error insert into $tb values(now+1n, 11); + tdLog.info("insert into tb0 values(now+1n, 11);") + tdSql.error("insert into tb0 values(now+1n, 11);") + # TSIM: sql_error insert into $tb values(now+1y, 12); + tdLog.info("insert into tb0 values(now+1y, 12);") + tdSql.error("insert into tb0 values(now+1y, 12);") + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql_error insert into $tb values ('9999-12-31 213:59:59.999', + # 13) + tdLog.info("insert into tb0 values ('9999-12-31 213:59:59.999', 13)") + tdSql.error("insert into tb0 values ('9999-12-31 213:59:59.999', 13)") + # TSIM: sql select ts from $tb + tdLog.info('select ts from tb0') + tdSql.query('select ts from tb0') + # TSIM: print $rows + tdLog.info('$rows') + # TSIM: if $rows != 8 then + tdLog.info('tdSql.checkRow(8)') + tdSql.checkRows(8) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql_error insert into $tb values ('9999-12-99 23:59:59.999', + # 13) + tdLog.info("insert into tb0 values ('9999-12-99 23:59:59.999', 13)") + tdSql.error("insert into tb0 values ('9999-12-99 23:59:59.999', 13)") + # TSIM: + # TSIM: sql select ts from $tb + tdLog.info('select ts from tb0') + tdSql.query('select ts from tb0') + # TSIM: if $rows != 8 then + tdLog.info('tdSql.checkRow(8)') + tdSql.checkRows(8) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: $i = 1 + # TSIM: $tb = $tbPrefix . $i + # TSIM: sql create table $tb (ts timestamp, ts2 timestamp) + tdLog.info("create table tb1 (ts timestamp, ts2 timestamp)") + tdSql.execute('create table tb1 (ts timestamp, ts2 timestamp)') + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql insert into $tb values (now, now) + tdLog.info("insert into tb1 values (now, now)") + tdSql.execute("insert into tb1 values (now, now)") + # TSIM: sql select * from $tb + tdLog.info('select * from tb1') + tdSql.query('select * from tb1') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/double.py b/tests/pytest/insert/double.py new file mode 100644 index 0000000000..19be9653c0 --- /dev/null +++ b/tests/pytest/insert/double.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed double)') + tdSql.execute('create table tb (ts timestamp, speed double)') + tdLog.info('=============== step2') + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 2.85)") + tdSql.execute("insert into tb values (now+2a, 2.85)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, 2.850000000)') + tdSql.checkData(0, 1, 2.850000000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 3.4)") + tdSql.execute("insert into tb values (now+3a, 3.4)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, 3.400000000)') + tdSql.checkData(0, 1, 3.400000000) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4a, a2) -x step51") + tdSql.error("insert into tb values (now+4a, a2)1") + tdLog.info("insert into tb values (now+4a, 0)") + tdSql.execute("insert into tb values (now+4a, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 0.000000000)') + tdSql.checkData(0, 1, 0.000000000) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5a, 2a) -x step6") + tdSql.error("insert into tb values (now+5a, 2a)") + tdLog.info("insert into tb values(now+5a, 2)") + tdSql.execute("insert into tb values(now+5a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6a, 2a'1) -x step7") + tdSql.error("insert into tb values (now+6a, 2a'1)") + tdLog.info("insert into tb values(now+6a, 2)") + tdSql.execute("insert into tb values(now+6a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py index 7c4474f7ad..30d7e223f1 100644 --- a/tests/pytest/insert/float.py +++ b/tests/pytest/insert/float.py @@ -13,9 +13,6 @@ import sys import datetime - -import taos - from util.log import * from util.cases import * from util.sql import * diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py index 8d38337576..d007cc56ea 100644 --- a/tests/pytest/insert/int.py +++ b/tests/pytest/insert/int.py @@ -12,7 +12,6 @@ # -*- coding: utf-8 -*- import sys -import taos import datetime from util.log import * diff --git a/tests/pytest/insert/smallint.py b/tests/pytest/insert/smallint.py new file mode 100644 index 0000000000..0d85e45cfa --- /dev/null +++ b/tests/pytest/insert/smallint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed smallint)') + tdSql.execute('create table tb (ts timestamp, speed smallint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -32768) -x step2") + tdSql.error("insert into tb values (now+1m, -32768)") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 32767)") + tdSql.execute("insert into tb values (now+2m, 32767)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 32767)') + tdSql.checkData(0, 1, 32767) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 32768) -x step4") + tdSql.error("insert into tb values (now+3m, 32768)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/tinyint.py b/tests/pytest/insert/tinyint.py new file mode 100644 index 0000000000..6d7178054e --- /dev/null +++ b/tests/pytest/insert/tinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint)') + tdSql.execute('create table tb (ts timestamp, speed tinyint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -128) -x step2") + tdSql.error("insert into tb values (now+1m, -128) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 127)") + tdSql.execute("insert into tb values (now+2m, 127)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 127)') + tdSql.checkData(0, 1, 127) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 128) -x step4") + tdSql.error("insert into tb values (now+3m, 128)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/tbname.py b/tests/pytest/query/tbname.py index d2799efa25..5ea89fff82 100644 --- a/tests/pytest/query/tbname.py +++ b/tests/pytest/query/tbname.py @@ -26,12 +26,14 @@ class TDTestCase: def run(self): tdSql.prepare() - tdSql.execute('create table cars (ts timestamp, speed int) tags(id int)') + tdSql.execute( + 'create table cars (ts timestamp, speed int) tags(id int)') tdSql.execute("create table carzero using cars tags(0)") tdSql.execute("create table carone using cars tags(1)") tdSql.execute("create table cartwo using cars tags(2)") - tdSql.execute("insert into carzero values(now, 100) carone values(now, 110)") + tdSql.execute( + "insert into carzero values(now, 100) carone values(now, 110)") tdSql.query("select * from cars where tbname in ('carzero', 'carone')") tdSql.checkRows(2) @@ -39,13 +41,16 @@ class TDTestCase: tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") tdSql.checkRows(1) - tdSql.query("select * from cars where id=1 or tbname in ('carzero', 'cartwo')") + tdSql.query( + "select * from cars where id=1 or tbname in ('carzero', 'cartwo')") tdSql.checkRows(2) - tdSql.query("select * from cars where id=1 and tbname in ('carzero', 'cartwo')") + tdSql.query( + "select * from cars where id=1 and tbname in ('carzero', 'cartwo')") tdSql.checkRows(0) - tdSql.query("select * from cars where id=0 and tbname in ('carzero', 'cartwo')") + tdSql.query( + "select * from cars where id=0 and tbname in ('carzero', 'cartwo')") tdSql.checkRows(1) """ diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh deleted file mode 100755 index 58345e13ee..0000000000 --- a/tests/pytest/simpletest.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -python3 ./test.py -f insert/basic.py $1 -python3 ./test.py -s $1 -sleep 1 -python3 ./test.py -f insert/int.py $1 -python3 ./test.py -s $1 -sleep 1 -python3 ./test.py -f insert/float.py $1 -python3 ./test.py -s $1 diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh new file mode 100755 index 0000000000..71d19df5c0 --- /dev/null +++ b/tests/pytest/smoketest.sh @@ -0,0 +1,55 @@ +#!/bin/bash +python3 ./test.py $1 -f insert/basic.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/int.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/float.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bool.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/double.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/smallint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/tinyint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importDataLastTO.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importDataLastT.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importDataTO.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importDataT.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importHeadOverlap.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importHeadPartOverlap.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importHORestart.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importHPORestart.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importHRestart.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f import_merge/importLastSub.py +python3 ./test.py -s $1 +sleep 1 diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 1771c23620..9a4f0fc98a 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -71,7 +71,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) - tdLog.notice("%s failed: %s" % (__file__, fileName)) + tdLog.exit("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 0008468130..a1f7dd2f64 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -44,7 +44,7 @@ class TDSql: except BaseException: expectErrNotOccured = False if expectErrNotOccured: - tdLog.exit("sql:%.40s, expect error not occured" % (sql)) + tdLog.exit("failed: sql:%.40s, expect error not occured" % (sql)) else: tdLog.info("sql:%.40s, expect error occured" % (sql)) @@ -71,43 +71,51 @@ class TDSql: def checkData(self, row, col, data): if row < 0: tdLog.exit( - "sql:%.40s, row:%d is smaller than zero" % + "failed: sql:%.40s, row:%d is smaller than zero" % (self.sql, row)) if col < 0: tdLog.exit( - "sql:%.40s, col:%d is smaller than zero" % + "failed: sql:%.40s, col:%d is smaller than zero" % (self.sql, col)) if row >= self.queryRows: tdLog.exit( - "sql:%.40s, row:%d is larger than queryRows:%d" % + "failed: sql:%.40s, row:%d is larger than queryRows:%d" % (self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "sql:%.40s, col:%d is larger than queryRows:%d" % + "failed: sql:%.40s, col:%d is larger than queryRows:%d" % (self.sql, col, self.queryCols)) if self.queryResult[row][col] != data: tdLog.exit( - "sql:%.40s row:%d col:%d data:%s != expect:%s" % + "failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % - (self.sql, row, col, self.queryResult[row][col], data)) + + if data is None: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + else: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): if row < 0: tdLog.exit( - "sql:%.40s, row:%d is smaller than zero" % + "failed: sql:%.40s, row:%d is smaller than zero" % (self.sql, row)) if col < 0: tdLog.exit( - "sql:%.40s, col:%d is smaller than zero" % + "failed: sql:%.40s, col:%d is smaller than zero" % (self.sql, col)) if row >= self.queryRows: tdLog.exit( - "sql:%.40s, row:%d is larger than queryRows:%d" % + "failed: sql:%.40s, row:%d is larger than queryRows:%d" % (self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "sql:%.40s, col:%d is larger than queryRows:%d" % + "failed: sql:%.40s, col:%d is larger than queryRows:%d" % (self.sql, col, self.queryCols)) return self.queryResult[row][col] @@ -126,7 +134,7 @@ class TDSql: def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: - tdLog.exit("sql:%.40s, affectedRows:%d != expect:%d" % + tdLog.exit("failed: sql:%.40s, affectedRows:%d != expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 6682aac352..b42f440068 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -83,19 +83,19 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 192.168.0.1:6020/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","tables","rows","cache(b)","ablocks","tblocks","ctime(s)","clog","comp","time precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","create time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","tables","cache(MB)","blocks","minrows","maxrows","ctime(s)","clog","comp","precision","status"],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 192.168.0.1:6020/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 192.168.0.1:6020/rest/sql print 13-> $system_content -if $system_content != @{"status":"error","code":33,"desc":"DB already there"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"DB already there"}@ then return -1 endi @@ -121,14 +121,14 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 192.168.0.1:6020/rest/sql print 17-> $system_content -if $system_content != @{"status":"error","code":28,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then return -1 endi #18 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 192.168.0.1:6020/rest/sql print 18-> $system_content -if $system_content != @{"status":"error","code":38,"desc":"DB not selected"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"db not selected"}@ then return -1 endi @@ -137,7 +137,7 @@ print =============== step3 - db #19 #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 192.168.0.1:6020/rest/sql/d4 #print 19-> $system_content -#if $system_content != @{"status":"error","code":102,"desc":"invalid DB"}@ then +#if $system_content != @{"status":"error","code":1000,"desc":"invalid DB"}@ then # return -1 #endi @@ -149,13 +149,13 @@ print =============== step3 - db system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 192.168.0.1:6020/rest/sql print 21-> $system_content -if $system_content != @{"status":"error","code":32,"desc":"invalid table"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"invalid table name"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 192.168.0.1:6020/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi @@ -209,13 +209,13 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 192.168.0.1:6020/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 192.168.0.1:6020/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 1667034f0e..b782a9b9bf 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -106,7 +106,7 @@ if $system_content != @{"status":"error","code":1029,"desc":"metric name length return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content if $system_content != @{"status":"error","code":1030,"desc":"metric name length too long"}@ then @@ -204,7 +204,7 @@ if $system_content != @{"status":"error","code":1042,"desc":"table is null"}@ th return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content if $system_content != @{"status":"error","code":1043,"desc":"table name length too long"}@ then diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index 092eebdbdf..336d606a37 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,6 +1,8 @@ run general/http/restful.sim -run general/http/restful_full.sim -run general/http/telegraf.sim -run general/http/prepare.sim -run general/http/grafana_bug.sim -run general/http/grafana.sim +run general/http/restful_insert.sim +#run general/http/restful_limit.sim +#run general/http/restful_full.sim +#run general/http/prepare.sim +#run general/http/telegraf.sim +#run general/http/grafana_bug.sim +#run general/http/grafana.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 907ca6848b..63c65d0345 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -93,7 +93,7 @@ echo "privateIp $NODE_IP" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 131" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG @@ -101,6 +101,7 @@ echo "monitorDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 131" >> $TAOS_CFG echo "jnidebugFlag 131" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG +echo "http 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 100000000" >> $TAOS_CFG diff --git a/tests/script/sh/exec_up.sh b/tests/script/sh/exec_up.sh index ee4aabc175..abe2822450 100755 --- a/tests/script/sh/exec_up.sh +++ b/tests/script/sh/exec_up.sh @@ -70,24 +70,15 @@ if [ "$CLEAR_OPTION" = "clear" ]; then rm -rf $MGMT_DIR fi -if [ "$SHELL_OPTION" = "true" ]; then - if [ "$EXEC_OPTON" = "start" ]; then - echo "ExcuteCmd:" $EXE_DIR/taos -c $CFG_DIR -u $USERS -p - $EXE_DIR/taos -c $CFG_DIR -u $USERS -p - else - #relative path - RCFG_DIR=sim/$NODE_NAME/cfg - PID=`ps -ef|grep -v taosd | grep taos | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` - if [ -n "$PID" ]; then - sudo kill -9 $PID - fi - fi - return -fi - if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + + if [ "$SHELL_OPTION" = "true" ]; then + nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + fi + #TT=`date +%s` #mkdir ${LOG_DIR}/${TT} #echo valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 731b707434..7890864360 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -1,4 +1,30 @@ system sh/stop_dnodes.sh +system sh/ip.sh -i 1 -s up +system sh/ip.sh -i 2 -s up +system sh/ip.sh -i 3 -s up +system sh/ip.sh -i 4 -s up + system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 -system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 \ No newline at end of file +system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 +system sh/deploy.sh -n dnode4 -m 192.168.0.1 -i 192.168.0.4 + +system sh/cfg.sh -n dnode1 -c commitLog -v 2 +system sh/cfg.sh -n dnode2 -c commitLog -v 2 +system sh/cfg.sh -n dnode3 -c commitLog -v 2 +system sh/cfg.sh -n dnode4 -c commitLog -v 2 + +system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1 + +system sh/cfg.sh -n dnode1 -c mgmtEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mgmtEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 \ No newline at end of file diff --git a/tests/script/unique/db/commit.sim b/tests/script/unique/db/commit.sim index 7111574081..a9aa674a27 100644 --- a/tests/script/unique/db/commit.sim +++ b/tests/script/unique/db/commit.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/cfg.sh -n dnode2 -c commitLog -v 0 -system sh/cfg.sh -n dnode3 -c commitLog -v 0 +system sh/cfg.sh -n dnode1 -c commitLog -v 2 +system sh/cfg.sh -n dnode2 -c commitLog -v 2 +system sh/cfg.sh -n dnode3 -c commitLog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 @@ -17,13 +17,13 @@ system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 print ========= start dnode1 as master -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sleep 3000 print ========= start other dnodes sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 3000 print ======== step1 create db @@ -50,9 +50,9 @@ if $data01 != 40 then endi print ======== step2 stop dnode -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 3000 sql select * from tb order by ts desc @@ -101,9 +101,9 @@ if $data01 != 40 then endi print ======== step5 stop dnode -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 3000 sql select * from tb diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index ab44841522..ce1c3fef12 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c clog -v 0 -system sh/cfg.sh -n dnode2 -c clog -v 0 -system sh/cfg.sh -n dnode3 -c clog -v 0 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 3 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 3 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 3 @@ -20,12 +20,12 @@ system sh/cfg.sh -n dnode2 -c cacheBlockSize -v 200 system sh/cfg.sh -n dnode3 -c cacheBlockSize -v 200 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 3000 print ======== step1 diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index 0a03bc558c..4e2808dbaa 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -9,10 +9,10 @@ system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 system sh/deploy.sh -n dnode4 -m 192.168.0.1 -i 192.168.0.4 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/cfg.sh -n dnode2 -c commitLog -v 0 -system sh/cfg.sh -n dnode3 -c commitLog -v 0 -system sh/cfg.sh -n dnode4 -c commitLog -v 0 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 +system sh/cfg.sh -n dnode4 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 @@ -35,10 +35,10 @@ system sh/cfg.sh -n dnode3 -c tables -v 4 system sh/cfg.sh -n dnode4 -c tables -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 3000 $loop = 0 @@ -59,13 +59,13 @@ begin: endw print ======== step2 - system sh/exec.sh -n dnode2 -s stop + system sh/exec_up.sh -n dnode2 -s stop sql drop database $db print ======== step3 sleep 3000 - system sh/exec.sh -n dnode2 -s start + system sh/exec_up.sh -n dnode2 -s start sleep 20000 print ===> test times : $loop diff --git a/tests/script/unique/db/replica_add12.sim b/tests/script/unique/db/replica_add12.sim index 6ef47132c2..29f9fdbea8 100644 --- a/tests/script/unique/db/replica_add12.sim +++ b/tests/script/unique/db/replica_add12.sim @@ -9,10 +9,10 @@ system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 system sh/deploy.sh -n dnode4 -m 192.168.0.1 -i 192.168.0.4 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 -system sh/cfg.sh -n dnode4 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 +system sh/cfg.sh -n dnode4 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 @@ -30,12 +30,12 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 3000 print ======== step1 @@ -76,41 +76,47 @@ endi sleep 2000 +sql show dnodes +print dnode192.168.0.1 ==> openVnodes: $data3_1 +print dnode192.168.0.2 ==> openVnodes: $data3_2 +print dnode192.168.0.3 ==> openVnodes: $data3_3 + +if $data3_1 != 0 then + return -1 +endi + +if $data3_2 != 2 then + return -1 +endi + +if $data3_3 != 2 then + return -1 +endi + print ======== step2 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 22000 +sleep 10000 print ======== step3 -$x = 0 -show3: - $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 - endi sql show dnodes -print dnode192.168.0.1 ==> openVnodes: $data2_192.168.0.1 freeVnodes: $data3_192.168.0.1 -print dnode192.168.0.2 ==> openVnodes: $data2_192.168.0.2 freeVnodes: $data3_192.168.0.2 -print dnode192.168.0.3 ==> openVnodes: $data2_192.168.0.3 freeVnodes: $data3_192.168.0.3 +print dnode192.168.0.1 ==> openVnodes: $data3_1 +print dnode192.168.0.2 ==> openVnodes: $data3_2 +print dnode192.168.0.3 ==> openVnodes: $data3_3 -if $data2_192.168.0.1 != 0 then - goto show3 +if $data3_1 != 0 then + return -1 endi -if $data3_192.168.0.1 != 4 then - goto show3 +if $data3_2 != 4 then + return -1 endi -if $data3_192.168.0.2 != 0 then - goto show3 -endi - -if $data3_192.168.0.3 != 0 then - goto show3 +if $data3_3 != 4 then + return -1 endi print ======== step4 @@ -139,16 +145,68 @@ if $rows != 2 then return -1 endi -sleep 20000 +sql reset query cache +sleep 2000 + print ========= step5 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 5000 -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql select * from d1.t1 +if $rows != 2 then + return -1 +endi +sql select * from d2.t2 +if $rows != 2 then + return -1 +endi + +sql select * from d3.t3 +if $rows != 2 then + return -1 +endi + +sql select * from d4.t4 +if $rows != 2 then + return -1 +endi + +print ===== insert data + +sql insert into d1.t1 values(now, 3) +# no master +sql_error insert into d2.t2 values(now, 3) +sql insert into d3.t3 values(now, 3) +# no master +sql_error insert into d4.t4 values(now, 3) + +sql select * from d1.t1 +if $rows != 3 then + return -1 +endi + +sql select * from d2.t2 +if $rows != 2 then + return -1 +endi + +sql select * from d3.t3 +if $rows != 3 then + return -1 +endi + +sql select * from d4.t4 +if $rows != 2 then + return -1 +endi + +print ========= step6 +system sh/exec_up.sh -n dnode2 -s start +sleep 5000 + +sql insert into d2.t2 values(now, 3) +sql insert into d4.t4 values(now, 3) sql select * from d1.t1 if $rows != 3 then return -1 @@ -169,42 +227,31 @@ if $rows != 3 then return -1 endi -print ========= step6 -system sh/exec.sh -n dnode2 -s start -sleep 10000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 10000 +print ========= step61 -sql insert into d1.t1 values(now, 4) +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT +sleep 5000 + +# no master +sql_error insert into d1.t1 values(now, 4) sql insert into d2.t2 values(now, 4) -sql insert into d3.t3 values(now, 4) +# no master +sql_error insert into d3.t3 values(now, 4) sql insert into d4.t4 values(now, 4) -sql select * from d1.t1 -if $rows != 4 then - return -1 -endi - sql select * from d2.t2 if $rows != 4 then return -1 endi -sql select * from d3.t3 -if $rows != 4 then - return -1 -endi - sql select * from d4.t4 if $rows != 4 then return -1 endi print ========= step7 -system sh/exec.sh -n dnode3 -s start -sleep 10000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 10000 +system sh/exec_up.sh -n dnode3 -s start +sleep 5000 sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -212,7 +259,7 @@ sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) sql select * from d1.t1 -if $rows != 5 then +if $rows != 4 then return -1 endi @@ -222,7 +269,7 @@ if $rows != 5 then endi sql select * from d3.t3 -if $rows != 5 then +if $rows != 4 then return -1 endi diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index ff8e73e97a..a4854adc53 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -9,10 +9,10 @@ system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 system sh/deploy.sh -n dnode4 -m 192.168.0.1 -i 192.168.0.4 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 -system sh/cfg.sh -n dnode4 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 +system sh/cfg.sh -n dnode4 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 @@ -30,14 +30,14 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sql create dnode 192.168.0.4 -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode4 -s start sleep 3000 print ======== step1 @@ -81,7 +81,7 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 12000 +sleep 10000 print ======== step3 $x = 0 @@ -93,29 +93,25 @@ show3: endi sql show dnodes -print dnode192.168.0.1 ==> openVnodes: $data2_192.168.0.1 freeVnodes: $data3_192.168.0.1 -print dnode192.168.0.2 ==> openVnodes: $data2_192.168.0.2 freeVnodes: $data3_192.168.0.2 -print dnode192.168.0.3 ==> openVnodes: $data2_192.168.0.3 freeVnodes: $data3_192.168.0.3 -print dnode192.168.0.4 ==> openVnodes: $data2_192.168.0.4 freeVnodes: $data3_192.168.0.4 +print dnode192.168.0.1 ==> openVnodes: $data3_1 +print dnode192.168.0.2 ==> openVnodes: $data3_2 +print dnode192.168.0.3 ==> openVnodes: $data3_3 +print dnode192.168.0.4 ==> openVnodes: $data3_4 -if $data2_192.168.0.1 != 0 then - goto show3 +if $data3_1 != 0 then + return -1 endi -if $data3_192.168.0.1 != 4 then - goto show3 +if $data3_2 != 4 then + return -1 endi -if $data3_192.168.0.2 != 0 then - goto show3 +if $data3_3 != 4 then + return -1 endi -if $data3_192.168.0.3 != 0 then - goto show3 -endi - -if $data3_192.168.0.4 != 0 then - goto show3 +if $data3_3 != 4 then + return -1 endi print ======== step4 @@ -145,9 +141,10 @@ if $rows != 2 then endi print ========= step5 -sleep 10000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +sql reset query cache +sleep 1000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -175,9 +172,9 @@ if $rows != 3 then endi print ========= step6 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 3000 sql insert into d1.t1 values(now, 4) @@ -206,9 +203,9 @@ if $rows != 4 then endi print ========= step7 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 5000 -system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec_up.sh -n dnode4 -s stop -x SIGINT sleep 3000 sql insert into d1.t1 values(now, 5) @@ -237,9 +234,9 @@ if $rows != 5 then endi print ========= step8 -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode4 -s start sleep 5000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 3000 sql insert into d1.t1 values(now, 6) diff --git a/tests/script/unique/db/replica_add23.sim b/tests/script/unique/db/replica_add23.sim index d839efa82b..0aebcce988 100644 --- a/tests/script/unique/db/replica_add23.sim +++ b/tests/script/unique/db/replica_add23.sim @@ -9,10 +9,10 @@ system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 system sh/deploy.sh -n dnode4 -m 192.168.0.1 -i 192.168.0.4 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 -system sh/cfg.sh -n dnode4 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 +system sh/cfg.sh -n dnode4 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 @@ -30,14 +30,14 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sql create dnode 192.168.0.4 -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode4 -s start sleep 3000 print ======== step1 @@ -81,7 +81,7 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 12000 +sleep 10000 print ======== step3 $x = 0 @@ -93,41 +93,25 @@ show3: endi sql show dnodes -print dnode192.168.0.1 ==> openVnodes: $data2_192.168.0.1 freeVnodes: $data3_192.168.0.1 -print dnode192.168.0.2 ==> openVnodes: $data2_192.168.0.2 freeVnodes: $data3_192.168.0.2 -print dnode192.168.0.3 ==> openVnodes: $data2_192.168.0.3 freeVnodes: $data3_192.168.0.3 -print dnode192.168.0.4 ==> openVnodes: $data2_192.168.0.4 freeVnodes: $data3_192.168.0.4 +print dnode192.168.0.1 ==> openVnodes: $data3_1 +print dnode192.168.0.2 ==> openVnodes: $data3_2 +print dnode192.168.0.3 ==> openVnodes: $data3_3 +print dnode192.168.0.4 ==> openVnodes: $data3_4 -if $data2_192.168.0.1 != 0 then - goto show3 +if $data3_1 != 0 then + return -1 endi -if $data2_192.168.0.2 != 4 then - goto show3 +if $data3_2 != 4 then + return -1 endi -if $data2_192.168.0.3 != 4 then - goto show3 +if $data3_3 != 4 then + return -1 endi -if $data2_192.168.0.4 != 4 then - goto show3 -endi - -if $data3_192.168.0.1 != 4 then - goto show3 -endi - -if $data3_192.168.0.2 != 0 then - goto show3 -endi - -if $data3_192.168.0.3 != 0 then - goto show3 -endi - -if $data3_192.168.0.4 != 0 then - goto show3 +if $data3_3 != 4 then + return -1 endi print ======== step4 @@ -156,10 +140,12 @@ if $rows != 2 then return -1 endi -sleep 10000 +sql reset query cache +sleep 1000 + print ========= step5 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -187,10 +173,10 @@ if $rows != 3 then endi print ========= step6 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start +sleep 5000 +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 sql insert into d1.t1 values(now, 4) sql insert into d2.t2 values(now, 4) @@ -218,10 +204,10 @@ if $rows != 4 then endi print ========= step7 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start +sleep 5000 +system sh/exec_up.sh -n dnode4 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 3000 sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -249,10 +235,10 @@ if $rows != 5 then endi print ========= step8 -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode4 -s start +sleep 5000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index 7753b47b4e..144c3f674f 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -6,12 +6,12 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c commitLog -v 0 -system sh/cfg.sh -n dnode2 -c commitLog -v 0 -system sh/cfg.sh -n dnode3 -c commitLog -v 0 -system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 +system sh/cfg.sh -n dnode1 -c numOfMPeers -v 2 +system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2 +system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2 system sh/cfg.sh -n dnode1 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 @@ -20,12 +20,12 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 3000 print ======== step1 @@ -65,16 +65,16 @@ if $rows != 1 then endi print ========= step2 alter db -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 3000 +sleep 5000 print ========= step3 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 10000 print ========= step4 @@ -104,70 +104,39 @@ if $rows != 2 then endi print ========= step5 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql reset query cache +sleep 1000 -sql select * from d1.t1 -if $rows != 3 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 3 then - return -1 -endi - -sql select * from d3.t3 -if $rows != 3 then - return -1 -endi - -sql select * from d4.t4 -if $rows != 3 then - return -1 -endi +sql insert into d1.t1 values(now, 3) -x s1 +s1: +sql insert into d2.t2 values(now, 3) -x s2 +s2: +sql insert into d3.t3 values(now, 3) -x s3 +s3: +sql insert into d4.t4 values(now, 3) -x s4 +s4: print ========= step6 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start +sleep 5000 +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 -sql insert into d1.t1 values(now, 4) -sql insert into d2.t2 values(now, 4) -sql insert into d3.t3 values(now, 4) -sql insert into d4.t4 values(now, 4) - -sql select * from d1.t1 -if $rows != 4 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 4 then - return -1 -endi - -sql select * from d3.t3 -if $rows != 4 then - return -1 -endi - -sql select * from d4.t4 -if $rows != 4 then - return -1 -endi +sql insert into d1.t1 values(now, 4) -x s5 +s5: +sql insert into d2.t2 values(now, 4) -x s6 +s6: +sql insert into d3.t3 values(now, 4) -x s7 +s7: +sql insert into d4.t4 values(now, 4) -x s8 +s8: print ========= step7 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 5000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -175,22 +144,6 @@ sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) sql select * from d1.t1 -if $rows != 5 then - return -1 -endi - sql select * from d2.t2 -if $rows != 5 then - return -1 -endi - sql select * from d3.t3 -if $rows != 5 then - return -1 -endi - sql select * from d4.t4 -if $rows != 5 then - return -1 -endi - diff --git a/tests/script/unique/db/replica_reduce21.sim b/tests/script/unique/db/replica_reduce21.sim index e79aedbb24..9f15cf3453 100644 --- a/tests/script/unique/db/replica_reduce21.sim +++ b/tests/script/unique/db/replica_reduce21.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 @@ -20,10 +20,10 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 3000 print ======== step1 @@ -70,7 +70,7 @@ error1: print ========= step3 alter d1 sql alter database d1 replica 1 -sleep 8000 +sleep 12000 print ========= step4 query d1 sql insert into d1.t1 values(now, 2) @@ -86,16 +86,12 @@ sql select * from d5.t5 if $rows != 1 then return -1 endi - -print ========= step6 alter d5 -sql alter database d5 replica 2 -x error2 - return -1 -error2: - +return print ========= step7 drop d1 sql drop database d1 -sleep 12000 +sleep 5000 +sql reset query cache print ========= step8 sql insert into d5.t5 values(now, 2) sql insert into d2.t2 values(now, 2) @@ -123,7 +119,7 @@ if $rows != 2 then endi print ======== step9 stop dnode2 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 3000 sql insert into d5.t5 values(now, 3) diff --git a/tests/script/unique/db/replica_reduce31.sim b/tests/script/unique/db/replica_reduce31.sim index 92da979b97..c34dccbd62 100644 --- a/tests/script/unique/db/replica_reduce31.sim +++ b/tests/script/unique/db/replica_reduce31.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 @@ -20,12 +20,12 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 3000 print ======== step1 @@ -39,8 +39,8 @@ sql create table d2.t2 (ts timestamp, i int) sql create table d3.t3 (ts timestamp, i int) sql create table d4.t4 (ts timestamp, i int) -sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) +sql insert into d2.t2 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) @@ -65,12 +65,25 @@ if $rows != 1 then endi print ========= step2 alter db +sql_error alter database d1 replica 1 +sql_error alter database d2 replica 1 +sql_error alter database d3 replica 1 +sql alter database d1 replica 2 +sql alter database d2 replica 2 +sql alter database d3 replica 2 + +sleep 8000 + sql alter database d1 replica 1 sql alter database d2 replica 1 sql alter database d3 replica 1 -sleep 12000 + +sleep 8000 print ========= step3 +sql reset query cache +sleep 1000 + sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) @@ -100,7 +113,7 @@ print ========= step4 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 12000 +sleep 8000 sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -128,93 +141,46 @@ if $rows != 3 then endi print ========= step4 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 -sql insert into d1.t1 values(now, 4) -sql insert into d2.t2 values(now, 4) -sql insert into d3.t3 values(now, 4) -sql insert into d4.t4 values(now, 4) - -sql select * from d1.t1 -if $rows != 4 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 4 then - return -1 -endi - -sql select * from d3.t3 -if $rows != 4 then - return -1 -endi - -sql select * from d4.t4 -if $rows != 4 then - return -1 -endi +sql reset query cache +sleep 1000 +sql insert into d1.t1 values(now, 4) -x step1 +step1: +sql insert into d2.t2 values(now, 4) -x step2 +step2: +sql insert into d3.t3 values(now, 4) -x step3 +step3: +sql insert into d4.t4 values(now, 4) -x step4 +step4: print ========= step5 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start +sleep 5000 +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 - -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) - -sql select * from d1.t1 -if $rows != 5 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 5 then - return -1 -endi - -sql select * from d3.t3 -if $rows != 5 then - return -1 -endi - -sql select * from d4.t4 -if $rows != 5 then - return -1 -endi +sql insert into d1.t1 values(now, 5) -x step5 +step5: +sql insert into d2.t2 values(now, 5) -x step6 +step6: +sql insert into d3.t3 values(now, 5) -x step7 +step7: +sql insert into d4.t4 values(now, 5) -x step8 +step8: print ========= step6 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 5000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 + sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) sql insert into d3.t3 values(now, 6) sql insert into d4.t4 values(now, 6) sql select * from d1.t1 -if $rows != 6 then - return -1 -endi - sql select * from d2.t2 -if $rows != 6 then - return -1 -endi - sql select * from d3.t3 -if $rows != 6 then - return -1 -endi - sql select * from d4.t4 -if $rows != 6 then - return -1 -endi diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index 86f99e72be..a9cda824b8 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2 system sh/deploy.sh -n dnode3 -m 192.168.0.1 -i 192.168.0.3 -system sh/cfg.sh -n dnode1 -c commitLog -v 1 -system sh/cfg.sh -n dnode2 -c commitLog -v 1 -system sh/cfg.sh -n dnode3 -c commitLog -v 1 +system sh/cfg.sh -n dnode1 -c clog -v 2 +system sh/cfg.sh -n dnode2 -c clog -v 2 +system sh/cfg.sh -n dnode3 -c clog -v 2 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 @@ -20,12 +20,12 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 print ========= start dnodes -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode 192.168.0.2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode 192.168.0.3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 3000 print ======== step1 @@ -68,6 +68,7 @@ print ========= step2 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 +sql alter database d4 replica 2 sleep 12000 print ========= step3 @@ -97,8 +98,8 @@ if $rows != 2 then endi print ========= step4 -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +sleep 5000 sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -126,40 +127,24 @@ if $rows != 3 then endi print ========= step5 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start + +sleep 5000 +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 -sql insert into d1.t1 values(now, 4) -sql insert into d2.t2 values(now, 4) -sql insert into d3.t3 values(now, 4) -sql insert into d4.t4 values(now, 4) +sql reset query cache +sleep 1000 -sql select * from d1.t1 -if $rows != 4 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 4 then - return -1 -endi - -sql select * from d3.t3 -if $rows != 4 then - return -1 -endi - -sql select * from d4.t4 -if $rows != 4 then - return -1 -endi +sql_error insert into d1.t1 values(now, 4) +sql_error insert into d2.t2 values(now, 4) +sql_error insert into d3.t3 values(now, 4) +sql_error insert into d4.t4 values(now, 4) print ========= step6 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start sleep 5000 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 3000 sql insert into d1.t1 values(now, 5) @@ -168,21 +153,21 @@ sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) sql select * from d1.t1 -if $rows != 5 then +if $rows != 4 then return -1 endi sql select * from d2.t2 -if $rows != 5 then +if $rows != 4 then return -1 endi sql select * from d3.t3 -if $rows != 5 then +if $rows != 4 then return -1 endi sql select * from d4.t4 -if $rows != 5 then +if $rows != 4 then return -1 endi diff --git a/tests/script/unique/db/testSuite.sim b/tests/script/unique/db/testSuite.sim index a370668860..4918850426 100644 --- a/tests/script/unique/db/testSuite.sim +++ b/tests/script/unique/db/testSuite.sim @@ -1,5 +1,5 @@ #run unique/db/commit.sim -run unique/db/delete.sim +#run unique/db/delete.sim run unique/db/replica_add12.sim run unique/db/replica_add13.sim run unique/db/replica_add23.sim @@ -7,4 +7,4 @@ run unique/db/replica_reduce21.sim run unique/db/replica_reduce32.sim run unique/db/replica_reduce31.sim run unique/db/replica_part.sim -run unique/db/delete_part.sim +#run unique/db/delete_part.sim diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 555ca3cd6b..493f712c57 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -262,6 +262,9 @@ endi system sh/exec_up.sh -n dnode3 -s stop -x SIGINT +sql reset query cache +sleep 1000 + print ========== step10 sql select * from d1.t1 order by t desc print $data01 $data11 $data21 $data31 $data41 diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index f6e54ffc2f..9081e50ef5 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -64,6 +64,8 @@ print error of no master print ============== step6 system sh/exec_up.sh -n dnode1 -s start +sql close +sql connect $x = 0 show6: diff --git a/tests/script/unique/mnode/mgmt24.sim b/tests/script/unique/mnode/mgmt24.sim index 9418fd3398..dac3ca4cc0 100644 --- a/tests/script/unique/mnode/mgmt24.sim +++ b/tests/script/unique/mnode/mgmt24.sim @@ -60,6 +60,9 @@ print ============== step5 system sh/exec_up.sh -n dnode1 -s start sql_error create dnode 192.168.0.1 +sql close +sql connect + $x = 0 step5: $x = $x + 1 diff --git a/tests/test-all.sh b/tests/test-all.sh index 6943dd47a7..907ef4bedd 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,7 +25,12 @@ if [ "$totalFailed" -ne "0" ]; then fi cd ../pytest -./simpletest.sh 2>&1 | tee pytest-out.txt + +if [ "$1" == "cron" ]; then + ./fulltest.sh 2>&1 | tee pytest-out.txt +else + ./smoketest.sh 2>&1 | tee pytest-out.txt +fi totalPySuccess=`grep 'successfully executed' pytest-out.txt | wc -l` if [ "$totalPySuccess" -gt "0" ]; then diff --git a/tests/test/c/importOneRow.c b/tests/test/c/importOneRow.c index babfdae277..25eeb0b392 100644 --- a/tests/test/c/importOneRow.c +++ b/tests/test/c/importOneRow.c @@ -104,7 +104,12 @@ void taos_error(TAOS *con) { void* taos_execute(void *param) { ThreadObj *pThread = (ThreadObj *)param; - void *taos = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + + void *taos = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (taos == NULL) taos_error(taos); char sql[1024] = {0}; diff --git a/tests/test/c/importPerTabe.c b/tests/test/c/importPerTabe.c index 8a3ff048d3..e53707c55f 100644 --- a/tests/test/c/importPerTabe.c +++ b/tests/test/c/importPerTabe.c @@ -68,7 +68,12 @@ void createDbAndTable() { int64_t st, et; char qstr[64000]; - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("failed to connect to DB, reason:%s", taos_errstr(con)); exit(1); @@ -190,8 +195,12 @@ void *syncTest(void *param) { int maxBytes = 60000; pPrint("thread:%d, start to run", pInfo->threadIndex); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); exit(1); diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index de611eeec1..7d17283b88 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -68,7 +68,10 @@ void createDbAndTable() { int64_t st, et; char qstr[64000]; - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("failed to connect to DB, reason:%s", taos_errstr(con)); exit(1); @@ -191,7 +194,11 @@ void *syncTest(void *param) { pPrint("thread:%d, start to run", pInfo->threadIndex); - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); exit(1); diff --git a/tests/test/c/insertPerTable.c b/tests/test/c/insertPerTable.c index 0edae323fa..e280a72854 100644 --- a/tests/test/c/insertPerTable.c +++ b/tests/test/c/insertPerTable.c @@ -68,7 +68,12 @@ void createDbAndTable() { int64_t st, et; char qstr[64000]; - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("failed to connect to DB, reason:%s", taos_errstr(con)); exit(1); @@ -191,7 +196,12 @@ void *syncTest(void *param) { pPrint("thread:%d, start to run", pInfo->threadIndex); - con = taos_connect(tsMasterIp, tsDefaultUser, tsDefaultPass, NULL, 0); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsMaster, fqdn, &port); + + con = taos_connect(fqdn, tsDefaultUser, tsDefaultPass, NULL, port); if (con == NULL) { pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); exit(1); diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 304c488947..ae8848e1ac 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -160,7 +160,7 @@ extern SCommand simCmdList[]; extern int simScriptPos; extern int simScriptSucced; extern int simDebugFlag; -extern char scriptDir[]; +extern char tsScriptDir[]; extern bool simAsyncQuery; SScript *simParseScript(char *fileName); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 23f8f9173f..dd7f268f3e 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -24,7 +24,7 @@ void simLogSql(char *sql) { static FILE *fp = NULL; char filename[256]; - sprintf(filename, "%s/sim.sql", scriptDir); + sprintf(filename, "%s/sim.sql", tsScriptDir); if (fp == NULL) { fp = fopen(filename, "w"); if (fp == NULL) { @@ -270,7 +270,7 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; - sprintf(buf, "cd %s; ", scriptDir); + sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); int code = system(buf); @@ -306,9 +306,9 @@ void simStoreSystemContentResult(SScript *script, char *filename) { bool simExecuteSystemContentCmd(SScript *script, char *option) { char buf[4096] = {0}; char filename[400] = {0}; - sprintf(filename, "%s/%s.tmp", scriptDir, script->fileName); + sprintf(filename, "%s/%s.tmp", tsScriptDir, script->fileName); - sprintf(buf, "cd %s; ", scriptDir); + sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); sprintf(buf, "%s > %s 2>/dev/null", buf, filename); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 62f5c1efb8..fddda679c6 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -186,7 +186,7 @@ SScript *simParseScript(char *fileName) { if ((fileName[0] == '.') || (fileName[0] == '/')) { strcpy(name, fileName); } else { - sprintf(name, "%s/%s", scriptDir, fileName); + sprintf(name, "%s/%s", tsScriptDir, fileName); } if ((fd = fopen(name, "r")) == NULL) {