diff --git a/.gitmodules b/.gitmodules index 7c84eac8a4..049b39abfb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin url = https://github.com/taosdata/grafanaplugin +[submodule "src/connector/hivemq-tdengine-extension"] + path = src/connector/hivemq-tdengine-extension + url = https://github.com/huskar-t/hivemq-tdengine-extension.git diff --git a/Jenkinsfile b/Jenkinsfile index 3968451d87..8bf7e435fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,6 +86,13 @@ pipeline { ./crash_gen.sh -a -p -t 4 -s 2000 ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 + ./handle_crash_gen_val_log.sh + ''' + } sh ''' date cd ${WKC}/tests @@ -131,14 +138,33 @@ pipeline { sh''' cd ${WORKSPACE} git checkout develop - cd tests/gotest - bash batchtest.sh - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } + } } @@ -146,5 +172,82 @@ pipeline { } } - + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } } \ No newline at end of file diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 450c6a8f55..a7bb22f345 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -58,7 +58,7 @@ cp -r ${top_dir}/src/connector/grafanaplugin ${pkg_dir}${install_home_pat cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector cp -r ${top_dir}/src/connector/nodejs ${pkg_dir}${install_home_path}/connector -cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector +cp ${compile_dir}/build/lib/taos-jdbcdriver*dist.* ${pkg_dir}${install_home_path}/connector ||: cp -r ${compile_dir}/../packaging/deb/DEBIAN ${pkg_dir}/ chmod 755 ${pkg_dir}/DEBIAN/* diff --git a/packaging/release.sh b/packaging/release.sh index 7542a5b4ca..68f947ccab 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -156,9 +156,15 @@ build_time=$(date +"%F %R") # get commint id from git gitinfo=$(git rev-parse --verify HEAD) -enterprise_dir="${top_dir}/../enterprise" -cd ${enterprise_dir} -gitinfoOfInternal=$(git rev-parse --verify HEAD) + +if [[ "$verMode" == "cluster" ]]; then + enterprise_dir="${top_dir}/../enterprise" + cd ${enterprise_dir} + gitinfoOfInternal=$(git rev-parse --verify HEAD) +else + gitinfoOfInternal=NULL +fi + cd ${curr_dir} # 2. cmake executable file @@ -193,23 +199,35 @@ cd ${curr_dir} # 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} + ret='0' + command -v dpkg >/dev/null 2>&1 || { ret='1'; } + if [ "$ret" -eq 0 ]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + else + echo "==========dpkg command not exist, so not release deb package!!!" fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} + ret='0' + command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } + if [ "$ret" -eq 0 ]; then + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + else + echo "==========rpmbuild command not exist, so not release rpm package!!!" fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} fi echo "====do tar.gz package for all systems====" diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 4e40263dc4..54c3c9b279 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -2,7 +2,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true -Name: TDengine +Name: tdengine Version: %{_version} Release: 3%{?dist} Summary: tdengine from taosdata @@ -65,7 +65,7 @@ cp -r %{_compiledir}/../src/connector/grafanaplugin %{buildroot}%{homepath}/conn cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector -cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector +cp %{_compiledir}/build/lib/taos-jdbcdriver*dist.* %{buildroot}%{homepath}/connector ||: cp -r %{_compiledir}/../tests/examples/* %{buildroot}%{homepath}/examples #Scripts executed before installation diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index eff70d8035..831012851a 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -278,11 +278,11 @@ function install_service_on_sysvinit() { # Install taosd service if ((${os_type}==1)); then - ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d - ${csudo} cp ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/../deb/taosd ${install_main_dir}/init.d + ${csudo} cp ${script_dir}/../deb/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd elif ((${os_type}==2)); then - ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d - ${csudo} cp ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/../rpm/taosd ${install_main_dir}/init.d + ${csudo} cp ${script_dir}/../rpm/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd fi #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index e17c678f26..69fa53c087 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -110,7 +110,7 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index faa5a03f52..7e8bef0dff 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -135,7 +135,7 @@ mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: fi cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 75b45b544e..a6d868ed1d 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -124,7 +124,7 @@ cp ${lib_files} ${install_dir}/driver connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 2c02b99787..0ffcb63e3f 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -156,7 +156,7 @@ cp ${lib_files} ${install_dir}/driver connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - cp ${build_dir}/lib/*.jar ${install_dir}/connector + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: cp -r ${connector_dir}/grafanaplugin ${install_dir}/connector/ cp -r ${connector_dir}/python ${install_dir}/connector/ cp -r ${connector_dir}/go ${install_dir}/connector diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b0d8d8983b..233b7a15b4 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.5.1' +version: '2.0.6.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.5.1 + - usr/lib/libtaos.so.2.0.6.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index d86e1aa0fb..39299d3308 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -75,6 +75,7 @@ typedef struct SJoinSupporter { SArray* exprList; SFieldInfo fieldsInfo; STagCond tagCond; + SSqlGroupbyExpr groupInfo; // group by info struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array FILE* f; // temporary file in order to create TSBuf char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory @@ -225,7 +226,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); -SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables); +SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); @@ -265,6 +266,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t sub void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid); +int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId); void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index 67942ad42a..e5bdcecfa8 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -77,7 +77,7 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); * @param colId * @return */ -SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId); +SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); /** * check if the schema is valid or not, including following aspects: diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index fa215db270..bbd029d675 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,6 +30,7 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" #include "tcache.h" +#include "tref.h" #include "qExecutor.h" #include "qSqlparser.h" @@ -127,7 +128,7 @@ typedef struct STableMetaInfo { typedef struct SSqlExpr { char aliasName[TSDB_COL_NAME_LEN]; // as aliasName SColIndex colInfo; - int64_t uid; // refactor use the pointer + uint64_t uid; // refactor use the pointer int16_t functionId; // function id in aAgg array int16_t resType; // return value type int16_t resBytes; // length of return value @@ -446,7 +447,7 @@ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); void tscFreeTableMetaHelper(void *pTableMeta); -void tscCloseTscObj(STscObj *pObj); +void tscCloseTscObj(void *pObj); // todo move to taos? or create a new file: taos_internal.h TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), @@ -516,6 +517,7 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; +extern int tscRefId; extern SRpcCorEpSet tscMgmtEpSet; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index c996bb2a76..a85a0ea570 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -405,7 +405,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlRes *pRes = &pSql->res; pRes->code = code; - const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + SSqlObj *sub = (SSqlObj*) res; + const char* msg = (sub->cmd.command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; if (code != TSDB_CODE_SUCCESS) { tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); goto _error; @@ -427,8 +428,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { assert(code == TSDB_CODE_SUCCESS); } - - assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pSql->param != NULL); + // param already freed by other routine and pSql in tscCache when ctrl + c + if (atomic_load_ptr(&pSql->param) == NULL) { + return; + } + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; SSqlObj * pParObj = trs->pParentSql; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 18d72e2d1e..ea073e8192 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -1253,10 +1253,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur return true; } -void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pLocalReducer->pCtx[i].aOutputBuf = - pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; +void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning + size_t t = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < t; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; } memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); @@ -1501,8 +1502,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { if (pLocalReducer->discard && sameGroup) { pLocalReducer->hasUnprocessedRow = false; tmpBuffer->num = 0; - } else { - // current row does not belongs to the previous group, so it is not be handled yet. + } else { // current row does not belongs to the previous group, so it is not be handled yet. pLocalReducer->hasUnprocessedRow = true; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ac34d26d2f..5e65126ef6 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -702,7 +702,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st } int32_t code = TSDB_CODE_TSC_INVALID_SQL; - char * tmpTokenBuf = calloc(1, 4096); // used for deleting Escape character: \\, \', \" + char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 815af79d8f..8716b5cb02 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2728,7 +2728,6 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; -// const char* msg3 = "group by columns must belong to one table"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2803,7 +2802,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* tscColumnListInsert(pTableMetaInfo->tagColList, &index); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by - if (pSchema->type > TSDB_DATA_TYPE_BINARY) { + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); } @@ -5281,20 +5280,26 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); + SSqlExpr* pExpr = NULL; + size_t size = taosArrayGetSize(pQueryInfo->exprList); - - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1); + if (size > 0) { + pExpr = tscSqlExprGet(pQueryInfo, (int32_t)size - 1); + } - if (pExpr->functionId != TSDB_FUNC_TAG) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo}; - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + if (pExpr == NULL || pExpr->functionId != TSDB_FUNC_TAG) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pParentQueryInfo, tableIndex); + + int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + + SSchema* pTagSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, colId); + int16_t colIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, colId); + SColumnIndex index = {.tableIndex = 0, .columnIndex = colIndex}; + + char* name = pTagSchema->name; + int16_t type = pTagSchema->type; + int16_t bytes = pTagSchema->bytes; - int16_t type = pSchema[index.columnIndex].type; - int16_t bytes = pSchema[index.columnIndex].bytes; - char* name = pSchema[index.columnIndex].name; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); pExpr->colInfo.flag = TSDB_COL_TAG; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index ac740555af..2a9a9de84b 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -118,7 +118,7 @@ SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) } // TODO for large number of columns, employ the binary search method -SSchema* tscGetTableColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { +SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { STableComInfo tinfo = tscGetTableInfo(pTableMeta); for(int32_t i = 0; i < tinfo.numOfColumns + tinfo.numOfTags; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 581271f845..2e7a99f2fb 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -150,7 +150,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pObj == NULL) return; if (pObj != pObj->signature) { - tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); return; } @@ -175,12 +175,12 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); } } else { - tscDebug("heartbeat failed, code:%s", tstrerror(code)); + tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code)); } if (pObj->pHb != NULL) { int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("%p start heartbeat in %dms", pSql, waitingDuring); + tscDebug("%p send heartbeat in %dms", pSql, waitingDuring); taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer); } else { @@ -190,29 +190,33 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { void tscProcessActivityTimer(void *handle, void *tmrId) { STscObj *pObj = (STscObj *)handle; - if (pObj == NULL || pObj->signature != pObj) { + + int ret = taosAcquireRef(tscRefId, pObj); + if (ret < 0) { + tscTrace("%p failed to acquire TSC obj, reason:%s", pObj, tstrerror(ret)); return; } SSqlObj* pHB = pObj->pHb; - if (pObj->pTimer != tmrId || pHB == NULL) { - return; - } void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE)); if (p == NULL) { tscWarn("%p HB object has been released already", pHB); + taosReleaseRef(tscRefId, pObj); return; } assert(*pHB->self == pHB); + pHB->retry = 0; int32_t code = tscProcessSql(pHB); taosCacheRelease(tscObjCache, (void**) &p, false); if (code != TSDB_CODE_SUCCESS) { tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); } + + taosReleaseRef(tscRefId, pObj); } int tscSendMsgToServer(SSqlObj *pSql) { @@ -549,11 +553,28 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs); - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096; + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + + tableSerialize + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -1638,11 +1659,14 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SCMHeartBeatMsg) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); - tscError("%p failed to malloc for heartbeat msg", pSql); + tscError("%p failed to create heartbeat msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + // TODO the expired hb and client can not be identified by server till now. SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + pHeartbeat->numOfQueries = numOfQueries; pHeartbeat->numOfStreams = numOfStreams; @@ -1995,10 +2019,11 @@ static void createHBObj(STscObj* pObj) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_TABLE_FNAME_LEN * 2]; STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + SCMConnectRsp *pConnect = (SCMConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); @@ -2017,6 +2042,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pObj->connId = htonl(pConnect->connId); createHBObj(pObj); + + //launch a timer to send heartbeat to maintain the connection and send status to mnode taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); return 0; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 8cac9b3398..89dfa24e8f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -161,6 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa registerSqlObj(pSql); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + taosAddRef(tscRefId, pObj); return pSql; } @@ -296,7 +297,8 @@ void taos_close(TAOS *taos) { } tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - tscCloseTscObj(pObj); + + taosRemoveRef(tscRefId, pObj); } void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 794b7a068b..c747eb2fe8 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -320,11 +320,8 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pQueryInfo->colList = pSupporter->colList; pQueryInfo->exprList = pSupporter->exprList; pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; + pQueryInfo->groupbyExpr = pSupporter->groupInfo; - pSupporter->exprList = NULL; - pSupporter->colList = NULL; - memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); @@ -332,7 +329,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; + + pSupporter->exprList = NULL; + pSupporter->colList = NULL; pSupporter->pVgroupTables = NULL; + memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo)); + memset(&pSupporter->groupInfo, 0, sizeof(SSqlGroupbyExpr)); /* * When handling the projection query, the offset value will be modified for table-table join, which is changed @@ -460,18 +462,36 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { } -int32_t tscCompareTidTags(const void* p1, const void* p2) { - const STidTags* t1 = (const STidTags*) varDataVal(p1); - const STidTags* t2 = (const STidTags*) varDataVal(p2); +int32_t tidTagsCompar(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) (p1); + const STidTags* t2 = (const STidTags*) (p2); if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } - if (t1->tid != t2->tid) { - return (t1->tid > t2->tid) ? 1 : -1; + tstr* tag1 = (tstr*) t1->tag; + tstr* tag2 = (tstr*) t2->tag; + + if (tag1->len != tag2->len) { + return (tag1->len > tag2->len)? 1: -1; } - return 0; + + return strncmp(tag1->data, tag2->data, tag1->len); +} + +int32_t tagValCompar(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) varDataVal(p1); + const STidTags* t2 = (const STidTags*) varDataVal(p2); + + tstr* tag1 = (tstr*) t1->tag; + tstr* tag2 = (tstr*) t2->tag; + + if (tag1->len != tag2->len) { + return (tag1->len > tag2->len)? 1: -1; + } + + return strncmp(tag1->data, tag2->data, tag1->len); } void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) { @@ -587,17 +607,19 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p2 = pParentSql->pSubs[1]->param; - qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags); - qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags); + // sort according to the tag value + qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar); + qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); // int16_t for padding - *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t)); - *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t)); + int32_t size = p1->tagSize - sizeof(int16_t); + *s1 = taosArrayInit(p1->num, size); + *s2 = taosArrayInit(p2->num, size); if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) { return TSDB_CODE_QRY_DUP_JOIN_KEY; @@ -625,6 +647,14 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar } } + // reorganize the tid-tag value according to both the vgroup id and tag values + // sort according to the tag value + size_t t1 = taosArrayGetSize(*s1); + size_t t2 = taosArrayGetSize(*s2); + + qsort((*s1)->pData, t1, size, tidTagsCompar); + qsort((*s2)->pData, t2, size, tidTagsCompar); + return TSDB_CODE_SUCCESS; } @@ -744,10 +774,10 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2); SSqlObj* psub1 = pParentSql->pSubs[0]; - ((SJoinSupporter*)psub1->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo1->pVgroupTables); + ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables); SSqlObj* psub2 = pParentSql->pSubs[1]; - ((SJoinSupporter*)psub2->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo2->pVgroupTables); + ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables); pParentSql->subState.numOfSub = 2; pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; @@ -1313,6 +1343,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter return TSDB_CODE_TSC_OUT_OF_MEMORY; } + pSupporter->groupInfo = pNewQueryInfo->groupbyExpr; + memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); + pNew->cmd.numOfCols = 0; pNewQueryInfo->interval.interval = 0; pSupporter->limit = pNewQueryInfo->limit; @@ -1333,17 +1366,9 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter assert(pTagCond->joinInfo.hasJoin); int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); - SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + SSchema* s = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); - // get the tag colId column index - int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - for(int32_t i = 0; i < numOfTags; ++i) { - if (pSchema[i].colId == tagColId) { - colIndex.columnIndex = i; - break; - } - } + colIndex.columnIndex = tscGetTagColIndexById(pTableMetaInfo->pTableMeta, tagColId); int16_t bytes = 0; int16_t type = 0; @@ -1424,6 +1449,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + SJoinSupporter *pSupporter = tscCreateJoinSupporter(pSql, i); if (pSupporter == NULL) { // failed to create support struct, abort current query @@ -2165,7 +2191,8 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { numOfRes = (int32_t)(MIN(numOfRes, remain)); } - if (numOfRes == 0) { + if (numOfRes == 0) { // no result any more, free all subquery objects + freeJoinSubqueryObj(pSql); return; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 47c2d35a75..77d668c5af 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -36,6 +36,7 @@ void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; +int tscRefId; int tscNumOfThreads; @@ -79,7 +80,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon void taos_init_imp(void) { - char temp[128]; + char temp[128] = {0}; errno = TSDB_CODE_SUCCESS; srand(taosGetTimestampSec()); @@ -146,29 +147,44 @@ void taos_init_imp(void) { tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj"); } + tscRefId = taosOpenRef(200, tscCloseTscObj); + + // in other language APIs, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated + // resource to suppress the valgrind warning. + atexit(taos_cleanup); tscDebug("client is initialized successfully"); } void taos_init() { pthread_once(&tscinit, taos_init_imp); } -void taos_cleanup() { - if (tscMetaCache != NULL) { - taosCacheCleanup(tscMetaCache); - tscMetaCache = NULL; +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); - taosCacheCleanup(tscObjCache); - tscObjCache = NULL; - } - - if (tscQhandle != NULL) { - taosCleanUpScheduler(tscQhandle); - tscQhandle = NULL; + void* m = tscMetaCache; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { + taosCacheCleanup(m); } + m = tscObjCache; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) { + taosCacheCleanup(m); + } + + m = tscQhandle; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { + taosCleanUpScheduler(m); + } + + taosCloseRef(tscRefId); taosCleanupKeywordsTable(); taosCloseLog(); - - taosTmrCleanUp(tscTmr); + + m = tscTmr; + if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { + taosTmrCleanUp(m); + } } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 07bd9d1b07..117874475b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -404,7 +404,7 @@ void tscFreeRegisteredSqlObj(void *pSql) { tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); if (ref == 0) { tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); - tscCloseTscObj(pTscObj); + taosRemoveRef(tscRefId, pTscObj); } } @@ -786,8 +786,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { } // TODO: all subqueries should be freed correctly before close this connection. -void tscCloseTscObj(STscObj* pObj) { - assert(pObj != NULL); +void tscCloseTscObj(void *param) { + STscObj *pObj = param; pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); @@ -1665,6 +1665,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { if (pQueryInfo->groupbyExpr.columnInfo != NULL) { taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); pQueryInfo->groupbyExpr.columnInfo = NULL; + pQueryInfo->groupbyExpr.numOfGroupCols = 0; } pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); @@ -1713,7 +1714,7 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { taosArrayRemove(pVgroupTable, index); } -SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { +SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return NULL; } @@ -1739,7 +1740,7 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { } void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); + tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); @@ -1779,6 +1780,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); } + // TODO handle malloc failure pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); if (pTableMetaInfo->tagColList == NULL) { return NULL; @@ -1788,7 +1790,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } - pTableMetaInfo->pVgroupTables = tscCloneVgroupTableInfo(pVgroupTables); + pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables); pQueryInfo->numOfTables += 1; return pTableMetaInfo; @@ -2155,6 +2157,21 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { } } +int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) { + int32_t numOfTags = tscGetNumOfTags(pTableMeta); + + SSchema* pSchema = tscGetTableTagSchema(pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == colId) { + return i; + } + } + + // can not reach here + assert(0); + return INT16_MIN; +} + bool tscIsUpdateQuery(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -2469,6 +2486,7 @@ void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) { dst->vgId = src->vgId; dst->numOfEps = src->numOfEps; for(int32_t i = 0; i < dst->numOfEps; ++i) { + taosTFree(dst->epAddr[i].fqdn); dst->epAddr[i].port = src->epAddr[i].port; dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); } diff --git a/src/connector/hivemq-tdengine-extension b/src/connector/hivemq-tdengine-extension new file mode 160000 index 0000000000..b62a26ecc1 --- /dev/null +++ b/src/connector/hivemq-tdengine-extension @@ -0,0 +1 @@ +Subproject commit b62a26ecc164a310104df57691691b237e091c89 diff --git a/src/inc/taos.h b/src/inc/taos.h index 3153137347..2c6454ced1 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -64,7 +64,7 @@ typedef struct taosField { #endif DLL_EXPORT void taos_init(); -DLL_EXPORT void taos_cleanup(); +DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 4b40d3919b..5330ca0bad 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -28,7 +28,7 @@ extern "C" { #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (int32_t)((0x80000000 | ((mod)<<16) | (code))); #endif - + #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) @@ -37,7 +37,7 @@ const char* tstrerror(int32_t err); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) - + #define TSDB_CODE_SUCCESS 0 #ifdef TAOS_ERROR_C @@ -74,6 +74,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory cor TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement") @@ -182,7 +188,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") -// vnode +// vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 600347c44f..ddc72eab14 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -106,6 +106,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) + +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "network-test" ) + + #ifndef TAOS_MESSAGE_C TSDB_MSG_TYPE_MAX // 105 #endif @@ -781,6 +785,7 @@ typedef struct { } SStreamDesc; typedef struct { + char clientVer[TSDB_VERSION_LEN]; uint32_t connId; int32_t pid; int32_t numOfQueries; diff --git a/src/inc/twal.h b/src/inc/twal.h index 92204abd7d..1ce7b132b0 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -22,22 +22,22 @@ extern "C" { #define TAOS_WAL_NOLOG 0 #define TAOS_WAL_WRITE 1 #define TAOS_WAL_FSYNC 2 - + typedef struct { - int8_t msgType; - int8_t reserved[3]; - int32_t len; - uint64_t version; - uint32_t signature; - uint32_t cksum; - char cont[]; + int8_t msgType; + int8_t reserved[3]; + int32_t len; + uint64_t version; + uint32_t signature; + uint32_t cksum; + char cont[]; } SWalHead; typedef struct { - int8_t walLevel; // wal level - int32_t fsyncPeriod; // millisecond - int8_t wals; // number of WAL files; - int8_t keep; // keep the wal file when closed + int8_t walLevel; // wal level + int32_t fsyncPeriod; // millisecond + int8_t wals; // number of WAL files; + int8_t keep; // keep the wal file when closed } SWalCfg; typedef void* twalh; // WAL HANDLE @@ -53,9 +53,6 @@ int walRestore(twalh, void *pVnode, FWalWrite writeFp); int walGetWalFile(twalh, char *name, uint32_t *index); int64_t walGetVersion(twalh); -extern int wDebugFlag; - - #ifdef __cplusplus } #endif diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 6cb7c669cc..2083ad3e9b 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -80,7 +80,10 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); if (args.netTestRole && args.netTestRole[0] != 0) { - taosNetTest(args.host, (uint16_t)args.port, (uint16_t)args.endPort, args.pktLen, args.netTestRole); + taos_init(); + CmdArguments cmdArgs; + memcpy(&cmdArgs, &args, sizeof(SShellArguments)); + taosNetTest(&cmdArgs); exit(0); } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 14558485aa..8c61c61a10 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -281,7 +281,14 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont), pHead->version, action, tstrerror(pOper->retCode)); if (action == SDB_ACTION_INSERT) { - sdbDeleteHash(pOper->table, pOper); + // It's better to create a table in two stages, create it first and then set it success + //sdbDeleteHash(pOper->table, pOper); + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = pOper->table, + .pObj = pOper->pObj + }; + sdbDeleteRow(&oper); } } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 80909e99ae..655fe71259 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -232,12 +232,16 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { - SCMHeartBeatRsp *pHBRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp)); - if (pHBRsp == NULL) { + SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *) rpcMallocCont(sizeof(SCMHeartBeatRsp)); + if (pRsp == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; + if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code + } + SRpcConnInfo connInfo = {0}; rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); @@ -251,33 +255,33 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { if (pConn == NULL) { // do not close existing links, otherwise // mError("failed to create connId, close connect"); - // pHBRsp->killConnection = 1; + // pRsp->killConnection = 1; } else { - pHBRsp->connId = htonl(pConn->connId); + pRsp->connId = htonl(pConn->connId); mnodeSaveQueryStreamList(pConn, pHBMsg); if (pConn->killed != 0) { - pHBRsp->killConnection = 1; + pRsp->killConnection = 1; } if (pConn->streamId != 0) { - pHBRsp->streamId = htonl(pConn->streamId); + pRsp->streamId = htonl(pConn->streamId); pConn->streamId = 0; } if (pConn->queryId != 0) { - pHBRsp->queryId = htonl(pConn->queryId); + pRsp->queryId = htonl(pConn->queryId); pConn->queryId = 0; } } - pHBRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); - pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeEpSetForShell(&pHBRsp->epSet); + pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); + pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); + mnodeGetMnodeEpSetForShell(&pRsp->epSet); - pMsg->rpcRsp.rsp = pHBRsp; + pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); - + mnodeReleaseConn(pConn); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5084f1276a..df703241c2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index f392644e67..b474bea987 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,15 +33,11 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SPosInfo { - int32_t pageId:20; - int32_t rowId:12; -} SPosInfo; - typedef struct SGroupResInfo { int32_t groupId; int32_t numOfDataPages; - SPosInfo pos; + int32_t pageId; + int32_t rowId; } SGroupResInfo; typedef struct SSqlGroupbyExpr { @@ -53,9 +49,10 @@ typedef struct SSqlGroupbyExpr { } SSqlGroupbyExpr; typedef struct SWindowResult { - SPosInfo pos; // Position of current result in disk-based output buffer + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t rowId:15; + bool closed:1; // this result status: closed or opened uint16_t numOfRows; // number of rows of current time window - bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo union {STimeWindow win; char* key;}; // start key of current time window } SWindowResult; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 5320e5622e..32f26f66f5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -51,7 +51,7 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); + int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d46beab2cb..7a7c1ee2d0 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -557,7 +557,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, int32_t numOfRowsPerPage) { - if (pWindowRes->pos.pageId != -1) { + if (pWindowRes->pageId != -1) { return 0; } @@ -590,11 +590,11 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult } // set the number of rows in current disk page - if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pos.pageId = pageId; - pWindowRes->pos.rowId = (int32_t)(pData->num++); + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->rowId = (int32_t)(pData->num++); - assert(pWindowRes->pos.pageId >= 0); + assert(pWindowRes->pageId >= 0); } return 0; @@ -616,7 +616,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes *newWind = true; // not assign result buffer yet, add new result buffer - if (pWindowRes->pos.pageId == -1) { + if (pWindowRes->pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; @@ -940,7 +940,6 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); if (sas->data == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1003,7 +1002,6 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1143,7 +1141,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat assert(pRuntimeEnv->windowResInfo.interval == 0); - if (pWindowRes->pos.pageId == -1) { + if (pWindowRes->pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; @@ -1277,7 +1275,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); if (sasArray == NULL) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1857,8 +1854,11 @@ static bool needReverseScan(SQuery *pQuery) { } if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) { + // the scan order to acquire the last result of the specified column int32_t order = (int32_t)pQuery->pSelectExpr[i].base.arg->argValue.i64; - return order != pQuery->order.order; + if (order != pQuery->order.order) { + return true; + } } } @@ -2652,7 +2652,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; @@ -2823,14 +2823,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId); + tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId); + tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); TSKEY rightTimestamp = GET_INT64_VAL(b2); @@ -2867,7 +2867,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { } SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { + if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -2883,10 +2883,10 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; // all results have been return to client, try next group - if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pos.pageId = 0; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId = 0; + pGroupResInfo->rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2914,22 +2914,22 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(size == pGroupResInfo->numOfDataPages); bool done = false; - for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { + for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); if (numOfRes > pQuery->rec.capacity - offset) { numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->pos.rowId += numOfCopiedRows; + pGroupResInfo->rowId += numOfCopiedRows; done = true; } else { numOfCopiedRows = (int32_t)pData->num; - pGroupResInfo->pos.pageId += 1; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId += 1; + pGroupResInfo->rowId = 0; } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -3020,8 +3020,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); pGroupResInfo->groupId = tid; - pGroupResInfo->pos.pageId = 0; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->pageId = 0; + pGroupResInfo->rowId = 0; return pGroupResInfo->numOfDataPages; } @@ -3067,7 +3067,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); @@ -3104,7 +3104,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { lastTimestamp = ts; // move to the next element of current entry - int32_t currentPageId = pWindowRes->pos.pageId; + int32_t currentPageId = pWindowRes->pageId; cs.position[pos] += 1; if (cs.position[pos] >= pWindowResInfo->size) { @@ -3117,7 +3117,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } else { // current page is not needed anymore SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pos.pageId != currentPageId) { + if (pNextWindowRes->pageId != currentPageId) { releaseResBufPage(pRuntimeEnv->pResultBuf, page); } } @@ -3329,7 +3329,8 @@ int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool is return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pResultRow->pos = (SPosInfo) {-1, -1}; + pResultRow->pageId = -1; + pResultRow->rowId = -1; char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo); @@ -3666,7 +3667,6 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // check if query is killed or not if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } } @@ -3796,7 +3796,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one */ - if (pWindowRes->pos.pageId == -1) { + if (pWindowRes->pageId == -1) { if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; @@ -3813,7 +3813,7 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3840,7 +3840,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -4019,12 +4019,12 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { pQInfo->groupIndex += 1; - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; - int32_t oldOffset = pGroupResInfo->pos.rowId; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->rowId; + int32_t oldOffset = pGroupResInfo->rowId; /* * current output space is not enough to accommodate all data of this page, only partial results @@ -4032,13 +4032,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; - pGroupResInfo->pos.rowId += numOfRowsToCopy; + pGroupResInfo->rowId += numOfRowsToCopy; } else { - pGroupResInfo->pos.rowId = 0; + pGroupResInfo->rowId = 0; pQInfo->groupIndex += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; @@ -4309,7 +4309,6 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; while (tsdbNextDataBlock(pQueryHandle)) { if (IS_QUERY_KILLED(GET_QINFO_ADDR(pRuntimeEnv))) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5173,7 +5172,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pQuery->order.order); if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; + SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } STsdbQueryCond cond = { @@ -5266,7 +5265,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5288,7 +5286,8 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query + //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead +// finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -5328,7 +5327,6 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) finalizeQueryResult(pRuntimeEnv); if (IS_QUERY_KILLED(pQInfo)) { - finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 0c9f92786f..d8159a27a5 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -251,16 +251,16 @@ static const char isIdChar[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; -static void* KeywordHashTable = NULL; +static void* keywordHashTable = NULL; static void doInitKeywordsTable(void) { int numOfEntries = tListLen(keywordTable); - KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); + keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); for (int32_t i = 0; i < numOfEntries; i++) { keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name); void* ptr = &keywordTable[i]; - taosHashPut(KeywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES); + taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES); } } @@ -282,7 +282,7 @@ int tSQLKeywordCode(const char* z, int n) { } } - SKeyword** pKey = (SKeyword**)taosHashGet(KeywordHashTable, key, n); + SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n); return (pKey != NULL)? (*pKey)->type:TK_ID; } @@ -660,5 +660,8 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } void taosCleanupKeywordsTable() { - taosHashCleanup(KeywordHashTable); + void* m = keywordHashTable; + if (m != NULL && atomic_val_compare_exchange_ptr(&keywordHashTable, m, 0) == m) { + taosHashCleanup(m); + } } \ No newline at end of file diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2bd92c74a4..ac95afffb1 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -266,7 +266,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow return; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; @@ -279,7 +279,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow } pWindowRes->numOfRows = 0; - pWindowRes->pos = (SPosInfo){-1, -1}; + pWindowRes->pageId = -1; + pWindowRes->rowId = -1; pWindowRes->closed = false; pWindowRes->win = TSWINDOW_INITIALIZER; } @@ -308,10 +309,10 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); // copy the output buffer data from src to dst, the position info keep unchanged - tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId); + tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); - tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId); + tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage); size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 6e9088d9fb..1dfd87ff3a 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -20,6 +20,7 @@ #include "ttimer.h" #include "tutil.h" #include "lz4.h" +#include "tref.h" #include "taoserror.h" #include "tsocket.h" #include "tglobal.h" @@ -72,7 +73,6 @@ typedef struct { SRpcInfo *pRpc; // associated SRpcInfo SRpcEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app - void *signature; // for validation struct SRpcConn *pConn; // pConn allocated char msgType; // message type uint8_t *pCont; // content provided by app @@ -132,6 +132,10 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +static int tsRpcRefId = -1; +static int32_t tsRpcNum = 0; +static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; + // server:0 client:1 tcp:2 udp:0 #define RPC_CONN_UDPS 0 #define RPC_CONN_UDPC 1 @@ -211,14 +215,26 @@ static void rpcUnlockConn(SRpcConn *pConn); static void rpcAddRef(SRpcInfo *pRpc); static void rpcDecRef(SRpcInfo *pRpc); -void *rpcOpen(const SRpcInit *pInit) { - SRpcInfo *pRpc; +static void rpcFree(void *p) { + tTrace("free mem: %p", p); + free(p); +} + +static void rpcInit(void) { tsProgressTimer = tsRpcTimer/2; tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; tsRpcHeadSize = RPC_MSG_OVERHEAD; tsRpcOverhead = sizeof(SRpcReqContext); + tsRpcRefId = taosOpenRef(200, rpcFree); +} + +void *rpcOpen(const SRpcInit *pInit) { + SRpcInfo *pRpc; + + pthread_once(&tsRpcInit, rpcInit); + pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; @@ -237,6 +253,8 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc->afp = pInit->afp; pRpc->refCount = 1; + atomic_add_fetch_32(&tsRpcNum, 1); + size_t size = sizeof(SRpcConn) * pRpc->sessions; pRpc->connList = (SRpcConn *)calloc(1, size); if (pRpc->connList == NULL) { @@ -363,7 +381,6 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; - pContext->signature = pContext; pContext->pRpc = (SRpcInfo *)shandle; pContext->epSet = *pEpSet; pContext->contLen = contLen; @@ -386,6 +403,7 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { // set the handle to pContext, so app can cancel the request if (pMsg->handle) *((void **)pMsg->handle) = pContext; + taosAddRef(tsRpcRefId, pContext); rpcSendReqToServer(pRpc, pContext); return; @@ -536,14 +554,15 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { void rpcCancelRequest(void *handle) { SRpcReqContext *pContext = handle; - // signature is used to check if pContext is freed. - // pContext may have been released just before app calls the rpcCancelRequest - if (pContext == NULL || pContext->signature != pContext) return; + int code = taosAcquireRef(tsRpcRefId, pContext); + if (code < 0) return; if (pContext->pConn) { tDebug("%s, app tries to cancel request", pContext->pConn->info); rpcCloseConn(pContext->pConn); } + + taosReleaseRef(tsRpcRefId, pContext); } static void rpcFreeMsg(void *msg) { @@ -612,7 +631,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { if (pConn->pContext) pConn->pContext->pConn = NULL; - rpcFreeMsg(pConn->pReqMsg); + taosRemoveRef(tsRpcRefId, pConn->pContext); } } @@ -1057,6 +1076,13 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); } } else { // msg is passed to app only parsing is ok + + if (pHead->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + rpcSendQuickRsp(pConn, TSDB_CODE_SUCCESS); + rpcFreeMsg(pRecv->msg); + return pConn; + } + rpcProcessIncomingMsg(pConn, pHead, pContext); } } @@ -1068,7 +1094,6 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { SRpcInfo *pRpc = pContext->pRpc; - pContext->signature = NULL; pContext->pConn = NULL; if (pContext->pRsp) { // for synchronous API @@ -1085,7 +1110,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { } // free the request message - rpcFreeCont(pContext->pCont); + taosRemoveRef(tsRpcRefId, pContext); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { @@ -1593,6 +1618,12 @@ static void rpcDecRef(SRpcInfo *pRpc) pthread_mutex_destroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); taosTFree(pRpc); + + int count = atomic_sub_fetch_32(&tsRpcNum, 1); + if (count == 0) { + taosCloseRef(tsRpcRefId); + // tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error + } } } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ef635e6efc..6f5e3be8ab 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -20,6 +20,7 @@ #include "tlog.h" #include "tutil.h" #include "ttimer.h" +#include "tref.h" #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" @@ -43,6 +44,7 @@ char tsNodeFqdn[TSDB_FQDN_LEN]; static ttpool_h tsTcpPool; static void * syncTmrCtrl = NULL; static void * vgIdHash; +static int tsSyncRefId = -1; // local functions static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); @@ -54,13 +56,13 @@ static int syncProcessPeerMsg(void *param, void *buffer); static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); -static void syncAddNodeRef(SSyncNode *pNode); -static void syncDecNodeRef(SSyncNode *pNode); +static void syncFreeNode(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtyp); static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); char* syncRole[] = { @@ -106,6 +108,12 @@ int32_t syncInit() { return -1; } + tsSyncRefId = taosOpenRef(200, syncFreeNode); + if (tsSyncRefId < 0) { + syncCleanUp(); + return -1; + } + tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn)); sInfo("sync module initialized successfully"); @@ -128,6 +136,9 @@ void syncCleanUp() { vgIdHash = NULL; } + taosCloseRef(tsSyncRefId); + tsSyncRefId = -1; + sInfo("sync module is cleaned up"); } @@ -159,6 +170,12 @@ void *syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; + int ret = taosAddRef(tsSyncRefId, pNode); + if (ret < 0) { + syncFreeNode(pNode); + return NULL; + } + for (int i = 0; i < pCfg->replica; ++i) { const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); @@ -167,8 +184,6 @@ void *syncStart(const SSyncInfo *pInfo) { } } - syncAddNodeRef(pNode); - if (pNode->selfIndex < 0) { sInfo("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; @@ -210,7 +225,9 @@ void syncStop(void *param) { SSyncNode *pNode = param; SSyncPeer *pPeer; - if (pNode == NULL) return; + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return; + sInfo("vgId:%d, cleanup sync", pNode->vgId); pthread_mutex_lock(&(pNode->mutex)); @@ -228,14 +245,17 @@ void syncStop(void *param) { pthread_mutex_unlock(&(pNode->mutex)); - syncDecNodeRef(pNode); + taosReleaseRef(tsSyncRefId, pNode); + taosRemoveRef(tsSyncRefId, pNode); } int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { SSyncNode *pNode = param; int i, j; - if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG; + sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, pNode->replica); @@ -298,105 +318,63 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { syncRole[nodeRole]); syncBroadcastStatus(pNode); + taosReleaseRef(tsSyncRefId, pNode); + return 0; } int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { SSyncNode *pNode = param; - SSyncPeer *pPeer; - SSyncHead *pSyncHead; - SWalHead * pWalHead = data; - int fwdLen; - int code = 0; - if (pNode == NULL) return 0; + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return 0; - if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) { - sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId, - pWalHead->version, nodeVersion); - for (int i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - syncRestartConnection(pPeer); - } - return TSDB_CODE_SYN_INVALID_VERSION; - } + int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - // always update version - nodeVersion = pWalHead->version; - sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole], - qtype, pWalHead->version); - - if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; - - // only pkt from RPC or CQ can be forwarded - if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; - - // a hacker way to improve the performance - pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); - pSyncHead->type = TAOS_SMSG_FORWARD; - pSyncHead->pversion = 0; - pSyncHead->len = sizeof(SWalHead) + pWalHead->len; - fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head - - pthread_mutex_lock(&(pNode->mutex)); - - for (int i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; - if (pPeer == NULL || pPeer->peerFd < 0) continue; - if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; - - if (pNode->quorum > 1 && code == 0) { - syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - code = 1; - } - - int retLen = write(pPeer->peerFd, pSyncHead, fwdLen); - if (retLen == fwdLen) { - sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); - } else { - sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); - syncRestartConnection(pPeer); - } - } - - pthread_mutex_unlock(&(pNode->mutex)); + taosReleaseRef(tsSyncRefId, pNode); return code; } void syncConfirmForward(void *param, uint64_t version, int32_t code) { SSyncNode *pNode = param; - if (pNode == NULL) return; - if (pNode->quorum <= 1) return; + + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return; SSyncPeer *pPeer = pNode->pMaster; - if (pPeer == NULL) return; + if (pPeer && pNode->quorum > 1) { + char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; - char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; + SSyncHead *pHead = (SSyncHead *)msg; + pHead->type = TAOS_SMSG_FORWARD_RSP; + pHead->len = sizeof(SFwdRsp); - SSyncHead *pHead = (SSyncHead *)msg; - pHead->type = TAOS_SMSG_FORWARD_RSP; - pHead->len = sizeof(SFwdRsp); + SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); + pFwdRsp->version = version; + pFwdRsp->code = code; - SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); - pFwdRsp->version = version; - pFwdRsp->code = code; + int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); + int retLen = write(pPeer->peerFd, msg, msgLen); - int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int retLen = write(pPeer->peerFd, msg, msgLen); - - if (retLen == msgLen) { - sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version); - } else { - sDebug("%s, failed to send forward ack, restart", pPeer->id); - syncRestartConnection(pPeer); + if (retLen == msgLen) { + sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version); + } else { + sDebug("%s, failed to send forward ack, restart", pPeer->id); + syncRestartConnection(pPeer); + } } + + taosReleaseRef(tsSyncRefId, pNode); } void syncRecover(void *param) { SSyncNode *pNode = param; SSyncPeer *pPeer; + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return; + // to do: add a few lines to check if recover is OK // if take this node to unsync state, the whole system may not work @@ -414,17 +392,24 @@ void syncRecover(void *param) { } pthread_mutex_unlock(&(pNode->mutex)); + + taosReleaseRef(tsSyncRefId, pNode); } int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { SSyncNode *pNode = param; + int ret = taosAcquireRef(tsSyncRefId, pNode); + if (ret < 0) return -1; + pNodesRole->selfIndex = pNode->selfIndex; for (int i = 0; i < pNode->replica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; pNodesRole->role[i] = pNode->peerInfo[i]->role; } + taosReleaseRef(tsSyncRefId, pNode); + return 0; } @@ -457,22 +442,20 @@ static void syncAddArbitrator(SSyncNode *pNode) { pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo); } -static void syncAddNodeRef(SSyncNode *pNode) { atomic_add_fetch_8(&pNode->refCount, 1); } +static void syncFreeNode(void *param) { + SSyncNode *pNode = param; -static void syncDecNodeRef(SSyncNode *pNode) { - if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) { - pthread_mutex_destroy(&pNode->mutex); - taosTFree(pNode->pRecv); - taosTFree(pNode->pSyncFwds); - taosTFree(pNode); - } + pthread_mutex_destroy(&pNode->mutex); + taosTFree(pNode->pRecv); + taosTFree(pNode->pSyncFwds); + taosTFree(pNode); } void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { - syncDecNodeRef(pPeer->pSyncNode); + taosReleaseRef(tsSyncRefId, pPeer->pSyncNode); sDebug("%s, resource is freed", pPeer->id); taosTFree(pPeer->watchFd); @@ -529,7 +512,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer); } - syncAddNodeRef(pNode); + taosAcquireRef(tsSyncRefId, pNode); return pPeer; } @@ -1122,7 +1105,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - syncAddNodeRef(pNode); + if (taosAcquireRef(tsSyncRefId, pNode) < 0) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); @@ -1133,7 +1116,7 @@ static void syncProcessBrokenLink(void *param) { } pthread_mutex_unlock(&(pNode->mutex)); - syncDecNodeRef(pNode); + taosReleaseRef(tsSyncRefId, pNode); } static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { @@ -1202,22 +1185,90 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncNode *pNode = param; + + int ret = taosAcquireRef(tsSyncRefId, pNode); + if ( ret < 0) return; + SSyncFwds *pSyncFwds = pNode->pSyncFwds; - if (pSyncFwds == NULL) return; - uint64_t time = taosGetTimestampMs(); + if (pSyncFwds) {; + uint64_t time = taosGetTimestampMs(); - if (pSyncFwds->fwds > 0) { - pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pSyncFwds->fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; - if (time - pFwdInfo->time < 2000) break; - syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + if (pSyncFwds->fwds > 0) { + pthread_mutex_lock(&(pNode->mutex)); + for (int i = 0; i < pSyncFwds->fwds; ++i) { + SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; + if (time - pFwdInfo->time < 2000) break; + syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + syncRemoveConfirmedFwdInfo(pNode); + pthread_mutex_unlock(&(pNode->mutex)); } - syncRemoveConfirmedFwdInfo(pNode); - pthread_mutex_unlock(&(pNode->mutex)); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); + taosReleaseRef(tsSyncRefId, pNode); } + +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) { + SSyncPeer *pPeer; + SSyncHead *pSyncHead; + SWalHead * pWalHead = data; + int fwdLen; + int32_t code = 0; + + if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) { + sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId, + pWalHead->version, nodeVersion); + for (int i = 0; i < pNode->replica; ++i) { + pPeer = pNode->peerInfo[i]; + syncRestartConnection(pPeer); + } + return TSDB_CODE_SYN_INVALID_VERSION; + } + + // always update version + nodeVersion = pWalHead->version; + sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole], + qtype, pWalHead->version); + + if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; + + // only pkt from RPC or CQ can be forwarded + if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; + + // a hacker way to improve the performance + pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); + pSyncHead->type = TAOS_SMSG_FORWARD; + pSyncHead->pversion = 0; + pSyncHead->len = sizeof(SWalHead) + pWalHead->len; + fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head + + pthread_mutex_lock(&(pNode->mutex)); + + for (int i = 0; i < pNode->replica; ++i) { + pPeer = pNode->peerInfo[i]; + if (pPeer == NULL || pPeer->peerFd < 0) continue; + if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; + + if (pNode->quorum > 1 && code == 0) { + syncSaveFwdInfo(pNode, pWalHead->version, mhandle); + code = 1; + } + + int retLen = write(pPeer->peerFd, pSyncHead, fwdLen); + if (retLen == fwdLen) { + sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + } else { + sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); + syncRestartConnection(pPeer); + } + } + + pthread_mutex_unlock(&(pNode->mutex)); + + return code; +} + diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ebb6c3a0a9..19a5d3ba41 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -56,6 +56,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { int code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore + bool fileChanged = false; *fversion = 0; sinfo.index = 0; @@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { close(dfd); if (ret < 0) break; + fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } - if (code == 0 && (minfo.fversion != sinfo.fversion)) { + if (code == 0 && fileChanged) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 626ad77da2..5633e97cdf 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -79,7 +79,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; int vid = 0; - regex_t regex1, regex2; + regex_t regex1 = {0}, regex2 = {0}; int code = 0; char fname[TSDB_FILENAME_LEN] = "\0"; @@ -95,9 +95,27 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { dir = opendir(tDataDir); if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (errno == ENOENT) { + tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); + terrno = TAOS_SYSTEM_ERROR(errno); + + if (taosMkDir(tDataDir, 0755) < 0) { + tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + dir = opendir(tDataDir); + if (dir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } else { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index f3bd91f038..a2affb8c4f 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -562,12 +562,12 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { void tsdbRefTable(STable *pTable) { int32_t ref = T_REF_INC(pTable); UNUSED(ref); - // tsdbDebug("ref table %"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); } void tsdbUnRefTable(STable *pTable) { int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table uid:%"PRIu64", tid:%d, refCount:%d", TABLE_UID(pTable), TABLE_TID(pTable), ref); + tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); if (ref == 0) { // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); @@ -745,7 +745,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { T_REF_INC(pTable); - tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return pTable; @@ -889,7 +889,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro } if (lock) tsdbUnlockRepoMeta(pRepo); - tsdbDebug("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); + tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable)); tsdbUnRefTable(pTable); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac6c2e0c5a..90e16aee53 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1263,7 +1263,6 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl int32_t end = endPos; if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(start >= end); SWAP(start, end, int32_t); } @@ -2122,7 +2121,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { } } - // clear current group + // clear current group, unref unused table + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); + + // keyInfo.pTable may be NULL here. + if (pKeyInfo->pTable != keyInfo.pTable) { + tsdbUnRefTable(pKeyInfo->pTable); + } + } + taosArrayClear(pGroup); // more than one table in each group, only one table left for each group diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 89c8e3dc39..b8f568f82c 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,6 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) diff --git a/src/util/inc/tnettest.h b/src/util/inc/tnettest.h index 3fe1dfa920..426df5cbb2 100644 --- a/src/util/inc/tnettest.h +++ b/src/util/inc/tnettest.h @@ -20,7 +20,27 @@ extern "C" { #endif -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole); +typedef struct CmdArguments { + char* host; + char* password; + char* user; + char* auth; + char* database; + char* timezone; + bool is_raw_time; + bool is_use_passwd; + char file[TSDB_FILENAME_LEN]; + char dir[TSDB_FILENAME_LEN]; + int threadNum; + char* commands; + int abort; + int port; + int endPort; + int pktLen; + char* netTestRole; +} CmdArguments; + +void taosNetTest(CmdArguments* args); #ifdef __cplusplus } diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h new file mode 100644 index 0000000000..ead8e2eb90 --- /dev/null +++ b/src/util/inc/tref.h @@ -0,0 +1,65 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TREF_H +#define TDENGINE_TREF_H + +#ifdef __cplusplus +extern "C" { +#endif + +// open an instance, return refId which will be used by other APIs +int taosOpenRef(int max, void (*fp)(void *)); + +// close the Ref instance +void taosCloseRef(int refId); + +// add ref, p is the pointer to resource or pointer ID +int taosAddRef(int refId, void *p); +#define taosRemoveRef taosReleaseRef + +// acquire ref, p is the pointer to resource or pointer ID +int taosAcquireRef(int refId, void *p); + +// release ref, p is the pointer to resource or pinter ID +void taosReleaseRef(int refId, void *p); + +// return the first if p is null, otherwise return the next after p +void *taosIterateRef(int refId, void *p); + +// return the number of references in system +int taosListRef(); + +/* sample code to iterate the refs + +void demoIterateRefs(int refId) { + + void *p = taosIterateRef(refId, NULL); + while (p) { + + // process P + + p = taosIterateRef(refId, p); + } +} + +*/ + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TREF_H diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 3793f3d3a9..6fd5265983 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -15,11 +15,16 @@ #include "os.h" #include "taosdef.h" +#include "taosmsg.h" #include "taoserror.h" #include "tulog.h" #include "tconfig.h" #include "tglobal.h" #include "tsocket.h" +#include "trpc.h" +#include "rpcHead.h" +#include "tutil.h" +#include "tnettest.h" #define MAX_PKG_LEN (64*1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) @@ -30,9 +35,15 @@ typedef struct { uint16_t pktLen; } info_s; -static char serverFqdn[TSDB_FQDN_LEN]; +extern int tsRpcMaxUdpSize; + +static char g_user[TSDB_USER_LEN+1] = {0}; +static char g_pass[TSDB_PASSWORD_LEN+1] = {0}; +static char g_serverFqdn[TSDB_FQDN_LEN] = {0}; static uint16_t g_startPort = 0; static uint16_t g_endPort = 6042; +static uint32_t g_pktLen = 0; + static void *bindUdpPort(void *sarg) { info_s *pinfo = (info_s *)sarg; @@ -321,20 +332,146 @@ static void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uin return ; } -static void taosNetTestClient(const char* serverFqdn, uint16_t startPort, uint16_t endPort, int pktLen) { - uint32_t serverIp = taosGetIpFromFqdn(serverFqdn); +void* tnetInitRpc(char* secretEncrypt, char spi) { + SRpcInit rpcInit; + void* pRpcConn = NULL; + + taosEncryptPass((uint8_t *)g_pass, strlen(g_pass), secretEncrypt); + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "NET-TEST"; + rpcInit.numOfThreads = 1; // every DB connection has only one thread + rpcInit.cfp = NULL; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = g_user; + rpcInit.idleTime = 2000; + rpcInit.ckey = "key"; + rpcInit.spi = spi; + rpcInit.secret = secretEncrypt; + + pRpcConn = rpcOpen(&rpcInit); + return pRpcConn; +} + +static int rpcCheckPortImpl(const char* serverFqdn, uint16_t port, uint16_t pktLen, char spi) { + SRpcEpSet epSet; + SRpcMsg reqMsg; + SRpcMsg rspMsg; + void* pRpcConn; + + char secretEncrypt[32] = {0}; + + pRpcConn = tnetInitRpc(secretEncrypt, spi); + if (NULL == pRpcConn) { + return -1; + } + + memset(&epSet, 0, sizeof(SRpcEpSet)); + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + strcpy(epSet.fqdn[0], serverFqdn); + + reqMsg.msgType = TSDB_MSG_TYPE_NETWORK_TEST; + reqMsg.pCont = rpcMallocCont(pktLen); + reqMsg.contLen = pktLen; + reqMsg.code = 0; + reqMsg.handle = NULL; // rpc handle returned to app + reqMsg.ahandle = NULL; // app handle set by client + + rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); + + // handle response + if ((rspMsg.code != 0) || (rspMsg.msgType != TSDB_MSG_TYPE_NETWORK_TEST + 1)) { + //printf("code:%d[%s]\n", rspMsg.code, tstrerror(rspMsg.code)); + return -1; + } + + rpcFreeCont(rspMsg.pCont); + + rpcClose(pRpcConn); + + return 0; +} + +static void rpcCheckPort(uint32_t hostIp) { + int ret; + char spi; + + for (uint16_t port = g_startPort; port <= g_endPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + //================ check tcp port ================ + int32_t pktLen; + if (g_pktLen <= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize + 1000; + } else { + pktLen = g_pktLen; + } + + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("TCP port:%d test fail.\t\t", port); + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + } else { + //printf("tcp port:%d test ok.\t\t", port); + printf("TCP port:\033[32m%d test OK\033[0m\t\t", port); + } + + //================ check udp port ================ + if (g_pktLen >= tsRpcMaxUdpSize) { + pktLen = tsRpcMaxUdpSize - 1000; + } else { + pktLen = g_pktLen; + } + + spi = 0; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + spi = 1; + ret = rpcCheckPortImpl(g_serverFqdn, port, pktLen, spi); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } else { + //printf("udp port:%d test ok.\t\n", port); + printf("UDP port:\033[32m%d test OK\033[0m\t\n", port); + } + } + + printf("\n"); + return ; +} + +static void taosNetTestClient(int flag) { + uint32_t serverIp = taosGetIpFromFqdn(g_serverFqdn); if (serverIp == 0xFFFFFFFF) { - printf("Failed to resolve FQDN:%s", serverFqdn); + printf("Failed to resolve FQDN:%s", g_serverFqdn); exit(-1); } - checkPort(serverIp, startPort, endPort, pktLen); - + if (0 == flag) { + checkPort(serverIp, g_startPort, g_endPort, g_pktLen); + } else { + rpcCheckPort(serverIp); + } + return; } - - static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) { int port = startPort; @@ -375,49 +512,66 @@ static void taosNetTestServer(uint16_t startPort, uint16_t endPort, int pktLen) } -void taosNetTest(const char* host, uint16_t port, uint16_t endPort, int pktLen, const char* netTestRole) { - if (pktLen > MAX_PKG_LEN) { - printf("test packet len overflow: %d, max len not greater than %d bytes\n", pktLen, MAX_PKG_LEN); - exit(-1); +void taosNetTest(CmdArguments *args) { + if (0 == args->pktLen) { + g_pktLen = 1000; + } else { + g_pktLen = args->pktLen; } - if (port && endPort) { - if (port > endPort) { - printf("endPort[%d] must not lesss port[%d]\n", endPort, port); + if (args->port && args->endPort) { + if (args->port > args->endPort) { + printf("endPort[%d] must not lesss port[%d]\n", args->endPort, args->port); exit(-1); } } - if (host && host[0] != 0) { - if (strlen(host) >= TSDB_EP_LEN) { - printf("host invalid: %s\n", host); + if (args->host && args->host[0] != 0) { + if (strlen(args->host) >= TSDB_EP_LEN) { + printf("host invalid: %s\n", args->host); exit(-1); } - taosGetFqdnPortFromEp(host, serverFqdn, &g_startPort); + taosGetFqdnPortFromEp(args->host, g_serverFqdn, &g_startPort); } else { - tstrncpy(serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); + tstrncpy(g_serverFqdn, "127.0.0.1", TSDB_IPv4ADDR_LEN); g_startPort = tsServerPort; } - if (port) { - g_startPort = port; + if (args->port) { + g_startPort = args->port; } - if (endPort) { - g_endPort = endPort; + if (args->endPort) { + g_endPort = args->endPort; } - if (port > endPort) { + if (g_startPort > g_endPort) { printf("endPort[%d] must not lesss port[%d]\n", g_endPort, g_startPort); exit(-1); } + - if (0 == strcmp("client", netTestRole)) { - printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", serverFqdn, g_startPort, g_endPort, pktLen); - taosNetTestClient(serverFqdn, g_startPort, g_endPort, pktLen); - } else if (0 == strcmp("server", netTestRole)) { - taosNetTestServer(g_startPort, g_endPort, pktLen); + if (args->is_use_passwd) { + if (args->password == NULL) args->password = getpass("Enter password: "); + } else { + args->password = TSDB_DEFAULT_PASS; + } + tstrncpy(g_pass, args->password, TSDB_PASSWORD_LEN); + + if (args->user == NULL) { + args->user = TSDB_DEFAULT_USER; + } + tstrncpy(g_user, args->user, TSDB_USER_LEN); + + if (0 == strcmp("client", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(0); + } else if (0 == strcmp("clients", args->netTestRole)) { + printf("host: %s\tstart port: %d\tend port: %d\tpacket len: %d\n", g_serverFqdn, g_startPort, g_endPort, g_pktLen); + taosNetTestClient(1); + } else if (0 == strcmp("server", args->netTestRole)) { + taosNetTestServer(g_startPort, g_endPort, g_pktLen); } } diff --git a/src/util/src/tref.c b/src/util/src/tref.c new file mode 100644 index 0000000000..23a7210e99 --- /dev/null +++ b/src/util/src/tref.c @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taoserror.h" +#include "tulog.h" +#include "tutil.h" + +#define TSDB_REF_OBJECTS 50 +#define TSDB_REF_STATE_EMPTY 0 +#define TSDB_REF_STATE_ACTIVE 1 +#define TSDB_REF_STATE_DELETED 2 + +typedef struct SRefNode { + struct SRefNode *prev; + struct SRefNode *next; + void *p; + int32_t count; +} SRefNode; + +typedef struct { + SRefNode **nodeList; + int state; // 0: empty, 1: active; 2: deleted + int refId; + int max; + int32_t count; // total number of SRefNodes in this set + int64_t *lockedBy; + void (*fp)(void *); +} SRefSet; + +static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; +static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t tsRefMutex; +static int tsRefSetNum = 0; +static int tsNextId = 0; + +static void taosInitRefModule(void); +static int taosHashRef(SRefSet *pSet, void *p); +static void taosLockList(int64_t *lockedBy); +static void taosUnlockList(int64_t *lockedBy); +static void taosIncRefCount(SRefSet *pSet); +static void taosDecRefCount(SRefSet *pSet); + +int taosOpenRef(int max, void (*fp)(void *)) +{ + SRefNode **nodeList; + SRefSet *pSet; + int64_t *lockedBy; + int i, refId; + + pthread_once(&tsRefModuleInit, taosInitRefModule); + + nodeList = calloc(sizeof(SRefNode *), (size_t)max); + if (nodeList == NULL) { + return TSDB_CODE_REF_NO_MEMORY; + } + + lockedBy = calloc(sizeof(int64_t), (size_t)max); + if (lockedBy == NULL) { + free(nodeList); + return TSDB_CODE_REF_NO_MEMORY; + } + + pthread_mutex_lock(&tsRefMutex); + + for (i = 0; i < TSDB_REF_OBJECTS; ++i) { + tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; + if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; + } + + if (i < TSDB_REF_OBJECTS) { + refId = tsNextId; + pSet = tsRefSetList + refId; + taosIncRefCount(pSet); + pSet->max = max; + pSet->nodeList = nodeList; + pSet->lockedBy = lockedBy; + pSet->fp = fp; + pSet->state = TSDB_REF_STATE_ACTIVE; + pSet->refId = refId; + + tsRefSetNum++; + uTrace("refId:%d is opened, max:%d, fp:%p refSetNum:%d", refId, max, fp, tsRefSetNum); + } else { + refId = TSDB_CODE_REF_FULL; + free (nodeList); + free (lockedBy); + uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); + } + + pthread_mutex_unlock(&tsRefMutex); + + return refId; +} + +void taosCloseRef(int refId) +{ + SRefSet *pSet; + int deleted = 0; + + if (refId < 0 || refId >= TSDB_REF_OBJECTS) { + uTrace("refId:%d is invalid, out of range", refId); + return; + } + + pSet = tsRefSetList + refId; + + pthread_mutex_lock(&tsRefMutex); + + if (pSet->state == TSDB_REF_STATE_ACTIVE) { + pSet->state = TSDB_REF_STATE_DELETED; + deleted = 1; + uTrace("refId:%d is closed, count:%d", refId, pSet->count); + } else { + uTrace("refId:%d is already closed, count:%d", refId, pSet->count); + } + + pthread_mutex_unlock(&tsRefMutex); + + if (deleted) taosDecRefCount(pSet); +} + +int taosAddRef(int refId, void *p) +{ + int hash; + SRefNode *pNode; + SRefSet *pSet; + + if (refId < 0 || refId >= TSDB_REF_OBJECTS) { + uTrace("refId:%d p:%p failed to add, refId not valid", refId, p); + return TSDB_CODE_REF_INVALID_ID; + } + + pSet = tsRefSetList + refId; + taosIncRefCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + taosDecRefCount(pSet); + uTrace("refId:%d p:%p failed to add, not active", refId, p); + return TSDB_CODE_REF_ID_REMOVED; + } + + int code = 0; + hash = taosHashRef(pSet, p); + + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->p == p) + break; + + pNode = pNode->next; + } + + if (pNode) { + code = TSDB_CODE_REF_ALREADY_EXIST; + uTrace("refId:%d p:%p is already there, faild to add", refId, p); + } else { + pNode = calloc(sizeof(SRefNode), 1); + if (pNode) { + pNode->p = p; + pNode->count = 1; + pNode->prev = 0; + pNode->next = pSet->nodeList[hash]; + if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode; + pSet->nodeList[hash] = pNode; + uTrace("refId:%d p:%p is added, count:%d malloc mem: %p", refId, p, pSet->count, pNode); + } else { + code = TSDB_CODE_REF_NO_MEMORY; + uTrace("refId:%d p:%p is not added, since no memory", refId, p); + } + } + + if (code < 0) taosDecRefCount(pSet); + + taosUnlockList(pSet->lockedBy+hash); + + return code; +} + +int taosAcquireRef(int refId, void *p) +{ + int hash, code = 0; + SRefNode *pNode; + SRefSet *pSet; + + if (refId < 0 || refId >= TSDB_REF_OBJECTS) { + uTrace("refId:%d p:%p failed to acquire, refId not valid", refId, p); + return TSDB_CODE_REF_INVALID_ID; + } + + pSet = tsRefSetList + refId; + taosIncRefCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + uTrace("refId:%d p:%p failed to acquire, not active", refId, p); + taosDecRefCount(pSet); + return TSDB_CODE_REF_ID_REMOVED; + } + + hash = taosHashRef(pSet, p); + + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + + while (pNode) { + if (pNode->p == p) { + break; + } + + pNode = pNode->next; + } + + if (pNode) { + pNode->count++; + uTrace("refId:%d p:%p is acquired", refId, p); + } else { + code = TSDB_CODE_REF_NOT_EXIST; + uTrace("refId:%d p:%p is not there, failed to acquire", refId, p); + } + + taosUnlockList(pSet->lockedBy+hash); + + taosDecRefCount(pSet); + + return code; +} + +void taosReleaseRef(int refId, void *p) +{ + int hash; + SRefNode *pNode; + SRefSet *pSet; + int released = 0; + + if (refId < 0 || refId >= TSDB_REF_OBJECTS) { + uTrace("refId:%d p:%p failed to release, refId not valid", refId, p); + return; + } + + pSet = tsRefSetList + refId; + if (pSet->state == TSDB_REF_STATE_EMPTY) { + uTrace("refId:%d p:%p failed to release, cleaned", refId, p); + return; + } + + hash = taosHashRef(pSet, p); + + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->p == p) + break; + + pNode = pNode->next; + } + + if (pNode) { + pNode->count--; + + if (pNode->count == 0) { + if ( pNode->prev ) { + pNode->prev->next = pNode->next; + } else { + pSet->nodeList[hash] = pNode->next; + } + + if ( pNode->next ) { + pNode->next->prev = pNode->prev; + } + + (*pSet->fp)(pNode->p); + + free(pNode); + released = 1; + uTrace("refId:%d p:%p is removed, count:%d, free mem: %p", refId, p, pSet->count, pNode); + } else { + uTrace("refId:%d p:%p is released", refId, p); + } + } else { + uTrace("refId:%d p:%p is not there, failed to release", refId, p); + } + + taosUnlockList(pSet->lockedBy+hash); + + if (released) taosDecRefCount(pSet); +} + +// if p is NULL, return the first p in hash list, otherwise, return the next after p +void *taosIterateRef(int refId, void *p) { + SRefNode *pNode = NULL; + SRefSet *pSet; + + if (refId < 0 || refId >= TSDB_REF_OBJECTS) { + uTrace("refId:%d p:%p failed to iterate, refId not valid", refId, p); + return NULL; + } + + pSet = tsRefSetList + refId; + taosIncRefCount(pSet); + if (pSet->state != TSDB_REF_STATE_ACTIVE) { + uTrace("refId:%d p:%p failed to iterate, not active", refId, p); + taosDecRefCount(pSet); + return NULL; + } + + int hash = 0; + if (p) { + hash = taosHashRef(pSet, p); + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->p == p) break; + pNode = pNode->next; + } + + if (pNode == NULL) { + uError("refId:%d p:%p not there, quit", refId, p); + taosUnlockList(pSet->lockedBy+hash); + return NULL; + } + + // p is there + pNode = pNode->next; + if (pNode == NULL) { + taosUnlockList(pSet->lockedBy+hash); + hash++; + } + } + + if (pNode == NULL) { + for (; hash < pSet->max; ++hash) { + taosLockList(pSet->lockedBy+hash); + pNode = pSet->nodeList[hash]; + if (pNode) break; + taosUnlockList(pSet->lockedBy+hash); + } + } + + void *newP = NULL; + if (pNode) { + pNode->count++; // acquire it + newP = pNode->p; + taosUnlockList(pSet->lockedBy+hash); + uTrace("refId:%d p:%p is returned", refId, p); + } else { + uTrace("refId:%d p:%p the list is over", refId, p); + } + + if (p) taosReleaseRef(refId, p); // release the current one + + taosDecRefCount(pSet); + + return newP; +} + +int taosListRef() { + SRefSet *pSet; + SRefNode *pNode; + int num = 0; + + pthread_mutex_lock(&tsRefMutex); + + for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { + pSet = tsRefSetList + i; + + if (pSet->state == TSDB_REF_STATE_EMPTY) + continue; + + uInfo("refId:%d state:%d count::%d", i, pSet->state, pSet->count); + + for (int j=0; j < pSet->max; ++j) { + pNode = pSet->nodeList[j]; + + while (pNode) { + uInfo("refId:%d p:%p count:%d", i, pNode->p, pNode->count); + pNode = pNode->next; + num++; + } + } + } + + pthread_mutex_unlock(&tsRefMutex); + + return num; +} + +static int taosHashRef(SRefSet *pSet, void *p) +{ + int hash = 0; + int64_t v = (int64_t)p; + + for (int i = 0; i < sizeof(v); ++i) { + hash += (int)(v & 0xFFFF); + v = v >> 16; + i = i + 2; + } + + hash = hash % pSet->max; + + return hash; +} + +static void taosLockList(int64_t *lockedBy) { + int64_t tid = taosGetPthreadId(); + int i = 0; + while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { + if (++i % 100 == 0) { + sched_yield(); + } + } +} + +static void taosUnlockList(int64_t *lockedBy) { + int64_t tid = taosGetPthreadId(); + if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { + assert(false); + } +} + +static void taosInitRefModule(void) { + pthread_mutex_init(&tsRefMutex, NULL); +} + +static void taosIncRefCount(SRefSet *pSet) { + atomic_add_fetch_32(&pSet->count, 1); + uTrace("refId:%d inc count:%d", pSet->refId, pSet->count); +} + +static void taosDecRefCount(SRefSet *pSet) { + int32_t count = atomic_sub_fetch_32(&pSet->count, 1); + uTrace("refId:%d dec count:%d", pSet->refId, pSet->count); + + if (count > 0) return; + + pthread_mutex_lock(&tsRefMutex); + + if (pSet->state != TSDB_REF_STATE_EMPTY) { + pSet->state = TSDB_REF_STATE_EMPTY; + pSet->max = 0; + pSet->fp = NULL; + + taosTFree(pSet->nodeList); + taosTFree(pSet->lockedBy); + + tsRefSetNum--; + uTrace("refId:%d is cleaned, refSetNum:%d count:%d", pSet->refId, tsRefSetNum, pSet->count); + } + + pthread_mutex_unlock(&tsRefMutex); +} + diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 6c4af437b2..099b9d9530 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -326,6 +326,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { return 0; } +// TODO move to comm module bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { if (versionStr == NULL || versionNubmer == NULL) { return false; diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 8687a8005d..0c96ed2a2f 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -9,7 +9,22 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - + + LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(utilTest tutil common osdetail gtest pthread gcov) + + LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) + ADD_EXECUTABLE(trefTest ${BIN_SRC}) + TARGET_LINK_LIBRARIES(trefTest common tutil) + ENDIF() + +#IF (TD_LINUX) +# ADD_EXECUTABLE(trefTest ./trefTest.c) +# TARGET_LINK_LIBRARIES(trefTest tutil common) +#ENDIF () + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) + + diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c new file mode 100644 index 0000000000..09ffccd7b5 --- /dev/null +++ b/src/util/tests/trefTest.c @@ -0,0 +1,182 @@ +#include +#include +#include +#include +#include +#include "os.h" +#include "tref.h" +#include "tlog.h" +#include "tglobal.h" +#include "taoserror.h" +#include "tulog.h" + +typedef struct { + int refNum; + int steps; + int refId; + void **p; +} SRefSpace; + +void iterateRefs(int refId) { + int count = 0; + + void *p = taosIterateRef(refId, NULL); + while (p) { + // process P + count++; + p = taosIterateRef(refId, p); + } + + printf(" %d ", count); +} + +void *takeRefActions(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int code, id; + + for (int i=0; i < pSpace->steps; ++i) { + printf("s"); + id = random() % pSpace->refNum; + code = taosAddRef(pSpace->refId, pSpace->p[id]); + usleep(1); + + id = random() % pSpace->refNum; + code = taosAcquireRef(pSpace->refId, pSpace->p[id]); + if (code >= 0) { + usleep(id % 5 + 1); + taosReleaseRef(pSpace->refId, pSpace->p[id]); + } + + id = random() % pSpace->refNum; + taosRemoveRef(pSpace->refId, pSpace->p[id]); + usleep(id %5 + 1); + + id = random() % pSpace->refNum; + code = taosAcquireRef(pSpace->refId, pSpace->p[id]); + if (code >= 0) { + usleep(id % 5 + 1); + taosReleaseRef(pSpace->refId, pSpace->p[id]); + } + + id = random() % pSpace->refNum; + iterateRefs(id); + } + + for (int i=0; i < pSpace->refNum; ++i) { + taosRemoveRef(pSpace->refId, pSpace->p[i]); + } + + //uInfo("refId:%d thread exits", pSpace->refId); + + return NULL; +} + +void myfree(void *p) { + return; +} + +void *openRefSpace(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + + printf("c"); + pSpace->refId = taosOpenRef(50, myfree); + + if (pSpace->refId < 0) { + printf("failed to open ref, reson:%s\n", tstrerror(pSpace->refId)); + return NULL; + } + + pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); + for (int i=0; irefNum; ++i) { + pSpace->p[i] = (void *) malloc(128); + } + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + pthread_t thread1, thread2, thread3; + pthread_create(&(thread1), &thattr, takeRefActions, (void *)(pSpace)); + pthread_create(&(thread2), &thattr, takeRefActions, (void *)(pSpace)); + pthread_create(&(thread3), &thattr, takeRefActions, (void *)(pSpace)); + + pthread_join(thread1, NULL); + pthread_join(thread2, NULL); + pthread_join(thread3, NULL); + + taosCloseRef(pSpace->refId); + + for (int i=0; irefNum; ++i) { + free(pSpace->p[i]); + } + + uInfo("refId:%d main thread exit", pSpace->refId); + free(pSpace->p); + pSpace->p = NULL; + + return NULL; +} + +int main(int argc, char *argv[]) { + int refNum = 100; + int threads = 10; + int steps = 10000; + int loops = 1; + + uDebugFlag = 143; + + for (int i=1; itsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) { + } else if (terrno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeCleanUp(pVnode); - return terrno; + if (pVnode->syncCfg.replica <= 1) { + vnodeCleanUp(pVnode); + return terrno; + } else { + pVnode->version = 0; + } } sprintf(temp, "%s/wal", rootDir); diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h new file mode 100644 index 0000000000..593611589d --- /dev/null +++ b/src/wal/inc/walInt.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_WAL_INT_H +#define TDENGINE_WAL_INT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +extern int32_t wDebugFlag; + +#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} +#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} +#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} +#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} +#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} + +#define walPrefix "wal" +#define walSignature (uint32_t)(0xFAFBFDFE) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/wal/inc/walMgmt.h b/src/wal/inc/walMgmt.h new file mode 100644 index 0000000000..a23c7f8ec3 --- /dev/null +++ b/src/wal/inc/walMgmt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_WAL_MGMT_H +#define TDENGINE_WAL_MGMT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c new file mode 100644 index 0000000000..2dd094d860 --- /dev/null +++ b/src/wal/src/walMgmt.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "twal.h" +#include "walInt.h" +#include "walMgmt.h" \ No newline at end of file diff --git a/src/wal/src/walMain.c b/src/wal/src/walWrite.c similarity index 95% rename from src/wal/src/walMain.c rename to src/wal/src/walWrite.c index 1826002042..95587caa14 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walWrite.c @@ -14,11 +14,10 @@ */ #define _DEFAULT_SOURCE - -#define TAOS_RANDOM_FILE_FAIL_TEST - #include "os.h" -#include "tlog.h" +#include "twal.h" +#include "walInt.h" +#include "walMgmt.h" #include "tchecksum.h" #include "tutil.h" #include "ttimer.h" @@ -26,14 +25,6 @@ #include "twal.h" #include "tqueue.h" -#define walPrefix "wal" - -#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} -#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} -#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} -#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} -#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} typedef struct { uint64_t version; @@ -54,7 +45,6 @@ typedef struct { static void *walTmrCtrl = NULL; static int tsWalNum = 0; static pthread_once_t walModuleInit = PTHREAD_ONCE_INIT; -static uint32_t walSignature = 0xFAFBFDFE; static int walHandleExistingFiles(const char *path); static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp); static int walRemoveWalFiles(const char *path); @@ -250,11 +240,13 @@ int walWrite(void *handle, SWalHead *pHead) { if (taosTWrite(pWal->fd, pHead, contLen) != contLen) { wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); + return terrno; } else { pWal->version = pHead->version; } + ASSERT(contLen == pHead->len + sizeof(SWalHead)); - return terrno; + return 0; } void walFsync(void *handle) { @@ -424,7 +416,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) { if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wWarn("wal:%s, cksum is messed up, skip the rest of file", name); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - // ASSERT(false); + ASSERT(false); break; } diff --git a/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md new file mode 100644 index 0000000000..1c0e4c100b --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md @@ -0,0 +1,268 @@ +# 如何在 windows环境下使用jdbc进行TDengine应用开发 + +本文以windows环境为例，介绍java如何进行TDengine开发应用 + +## 环境准备 + +（1）安装jdk + +官网下载jdk-1.8，下载页面：https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html + +安装，配置环境变量，把jdk加入到环境变量里。 + +命令行内查看java的版本。 + +```shell +>java -version +java version "1.8.0_131" +Java(TM) SE Runtime Environment (build 1.8.0_131-b11) +Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode) +``` + + +（2）安装配置maven + +官网下载maven，下载地址：http://maven.apache.org/download.cgi + +配置环境变量MAVEN_HOME，将MAVEN_HOME/bin添加到PATH + +命令行里查看maven的版本 + +```shell +>mvn --version +Apache Maven 3.5.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2017-04-04T03:39:06+08:00) +Maven home: D:\apache-maven-3.5.0\bin\.. +Java version: 1.8.0_131, vendor: Oracle Corporation +Java home: C:\Program Files\Java\jdk1.8.0_131\jre +Default locale: zh_CN, platform encoding: GBK +OS name: "windows 10", version: "10.0", arch: "amd64", family: "windows" +``` + +为了加快maven下载依赖的速度，可以为maven配置mirror，修改MAVEN_HOME\config\settings.xml文件 + +```xml + + + D:\apache-maven-localRepository + + + + + alimaven + aliyun maven + http://maven.aliyun.com/nexus/content/groups/public/ + central + + + + + + + jdk-1.8 + + true + 1.8 + + + 1.8 + 1.8 + 1.8 + + + + +``` + + + +（3）在linux服务器上安装TDengine-server + +在taosdata官网下载TDengine-server，下载地址：https://www.taosdata.com/cn/all-downloads/ + +在linux服务器上安装TDengine-server + +```shell +# tar -zxvf package/TDengine-server-2.0.1.1-Linux-x64.tar.gz +# cd TDengine-server/ +# ./install.sh +``` + +启动taosd + +```shell +# systemctl start taosd +``` + +在server上用taos连接taosd + +```shell +# taos +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | +================================================================================================================== + 1 | td01:6030 | 2 | 4 | ready | any | 2020-08-19 18:40:25.045 | +Query OK, 1 row(s) in set (0.005765s) +``` + +如果可以正确连接到taosd实例，并打印出databases的信息，说明TDengine的server已经正确启动。这里查看server的hostname + +```shell +# hostname -f +td01 +``` + +注意，如果安装TDengine后，使用默认的taos.cfg配置文件，taosd会使用当前server的hostname创建dnode实例。之后，在client也需要使用这个hostname来连接taosd。 + + + +（4）在windows上安装TDengine-client + +在taosdata官网下载taos客户端，下载地址： +https://www.taosdata.com/cn/all-downloads/ +下载后，双击exe安装。 + +修改client的hosts文件（C:\Windows\System32\drivers\etc\hosts），将server的hostname和ip配置到client的hosts文件中 + +``` +192.168.236.136 td01 +``` + +配置完成后，在命令行内使用taos shell连接server端 + +```shell +C:\TDengine>taos +Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | +=================================================================================================================================================================================================================================================================== + test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | + log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | +Query OK, 2 row(s) in set (0.068000s) +``` + +如果windows上的client能够正常连接，并打印database信息，说明client可以正常连接server了。 + + + +## 应用开发 + +（1）新建maven工程，在pom.xml中引入taos-jdbcdriver依赖。 + +```xml + + + 4.0.0 + + com.taosdata.demo + JdbcDemo + 1.0-SNAPSHOT + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.8 + + + +``` + +（2）使用jdbc查询TDengine数据库 + +下面是示例代码： + +```java +public class JdbcDemo { + + public static void main(String[] args) throws Exception { + Connection conn = getConn(); + Statement stmt = conn.createStatement(); + // create database + stmt.executeUpdate("create database if not exists db"); + // use database + stmt.executeUpdate("use db"); + // create table + stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)"); + // insert data + int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now + 1s, 20, 9.3)"); + System.out.println("insert " + affectedRows + " rows."); + // query data + ResultSet resultSet = stmt.executeQuery("select * from tb"); + Timestamp ts = null; + int temperature = 0; + float humidity = 0; + while(resultSet.next()){ + ts = resultSet.getTimestamp(1); + temperature = resultSet.getInt(2); + humidity = resultSet.getFloat("humidity"); + System.out.printf("%s, %d, %s\n", ts, temperature, humidity); + } + } + + public static Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://td01:0/log?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; + } + +} +``` + +（3）测试jdbc访问tdengine的sever实例 + +console输出： + +``` +insert 2 rows. +2020-08-26 00:06:34.575, 23, 10.3 +2020-08-26 00:06:35.575, 20, 9.3 +``` + + + +## 指南 + +（1）如何设置主机名和hosts + +在server上查看hostname和fqdn +```shell +查看hostname +# hostname +taos-server + +查看fqdn +# hostname -f +taos-server +``` + +windows下hosts文件位于： +C:\\Windows\System32\drivers\etc\hosts +修改hosts文件，添加server的ip和hostname + +```s +192.168.56.101 node5 +``` + +（2）什么是fqdn？ + + +> 什么是FQDN？ +> +> FQDN（Full qualified domain name）全限定域名，fqdn由2部分组成：hostname+domainname。 +> +> 例如，一个邮件服务器的fqdn可能是：mymail.somecollege.edu，其中mymail是hostname（主机名），somcollege.edu是domainname（域名）。本例中，.edu是顶级域名，.somecollege是二级域名。 +> +> 当连接服务器时，必须指定fqdn，然后，dns服务器通过查看dns表，将hostname解析为相应的ip地址。如果只指定hostname（不指定domainname），应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时，本地的dns服务器和可能没有远程服务器的hostname列表。 +> +> 参考：https://kb.iu.edu/d/aiuv diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index b42e1e6d70..14a67b93d3 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -107,7 +107,7 @@ func main() { fmt.Scanln() url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/" - //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) + //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) // open connect to taos server //db, err := sql.Open(taosDriverName, url) //if err != nil { @@ -115,6 +115,7 @@ func main() { // os.Exit(1) //} //defer db.Close() + rand.Seed(time.Now().Unix()) createDatabase(configPara.dbName, configPara.supTblName) fmt.Printf("======== create database success! ========\n\n") diff --git a/tests/examples/nodejs/README-win.md b/tests/examples/nodejs/README-win.md new file mode 100644 index 0000000000..75fec69413 --- /dev/null +++ b/tests/examples/nodejs/README-win.md @@ -0,0 +1,200 @@ +# 如何在windows上使用nodejs进行TDengine应用开发 + +## 环境准备 + +（1）安装nodejs-10.22.0 + +下载链接：https://nodejs.org/dist/v10.22.0/node-v10.22.0-win-x64.zip +解压安装，把node配置到环境变量里 + +cmd启动命令行，查看node的版本 + +```shell +> node.exe --version +v10.22.0 + +> npm --version +6.14.6 +``` + + + +（2）安装python2.7 + +下载链接：https://www.python.org/ftp/python/2.7.18/python-2.7.18.amd64.msi + +查看python版本 + +```shell +>python --version +Python 2.7.18 +``` + + +（3）安装TDengine-client + +下载地址：https://www.taosdata.com/cn/all-downloads/，选择一个合适的windows-client下载（client应该尽量与server端的版本保持一致） + +使用client的taos shell连接server + +```shell +>taos -h node5 + +Welcome to the TDengine shell from Linux, Client Version:2.0.6.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | offline reason | +============================================================================================================================================ + 1 | node5:6030 | 7 | 1 | ready | any | 2020-10-26 09:45:26.308 | | +Query OK, 1 row(s) in set (0.036000s) +``` + +注意： +* 检查能否在client的机器上ping通server的fqdn +* 如果你的dns server并没有提供到server的域名解析，可以将server的hostname配置到client的hosts文件中 + + +## 应用开发 + +（1）建立nodejs项目 + +``` +npm init +``` + +（2）安装windows-build-tools +``` +npm install --global --production windows-build-tools +``` + +（3）安装td2.0-connector驱动 + +``` tdshell +npm install td2.0-connector +``` + +（4）nodejs访问tdengine的示例程序 + +```javascript +const taos = require('td2.0-connector'); + +var host = null; +var port = 6030; +for (var i = 2; i < global.process.argv.length; i++) { + var key = global.process.argv[i].split("=")[0]; + var value = global.process.argv[i].split("=")[1]; + + if ("host" == key) { + host = value; + } + if ("port" == key) { + port = value; + } +} + +if (host == null) { + console.log("Usage: node nodejsChecker.js host= port="); + process.exit(0); +} + +// establish connection +var conn = taos.connect({host: host, user: "root", password: "taosdata", port: port}); +var cursor = conn.cursor(); +// create database +executeSql("create database if not exists testnodejs", 0); +// use db +executeSql("use testnodejs", 0); +// drop table +executeSql("drop table if exists testnodejs.weather", 0); +// create table +executeSql("create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int)", 0); +// insert +executeSql("insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34)", 1); +// select +executeQuery("select * from testnodejs.weather"); +// close connection +conn.close(); + +function executeQuery(sql) { + var start = new Date().getTime(); + var promise = cursor.query(sql, true); + var end = new Date().getTime(); + promise.then(function (result) { + printSql(sql, result != null, (end - start)); + result.pretty(); + }); +} + +function executeSql(sql, affectRows) { + var start = new Date().getTime(); + var promise = cursor.execute(sql); + var end = new Date().getTime(); + printSql(sql, promise == affectRows, (end - start)); +} + +function printSql(sql, succeed, cost) { + console.log("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); +} +``` + +（5）测试nodejs程序 + +```shell +>node nodejsChecker.js +Usage: node nodejsChecker.js host= port= +# 提示指定host + +>node nodejsChecker.js host=node5 +Successfully connected to TDengine +Query OK, 0 row(s) affected (0.00997610s) +[ OK ] time cost: 14 ms, execute statement ====> create database if not exists testnodejs +Query OK, 0 row(s) affected (0.00235920s) +[ OK ] time cost: 4 ms, execute statement ====> use testnodejs +Query OK, 0 row(s) affected (0.06604280s) +[ OK ] time cost: 67 ms, execute statement ====> drop table if exists testnodejs.weather +Query OK, 0 row(s) affected (0.59403290s) +[ OK ] time cost: 595 ms, execute statement ====> create table if not exists testnodejs.weather(ts timestamp, temperature float, humidity int) +Query OK, 1 row(s) affected (0.01058950s) +[ OK ] time cost: 12 ms, execute statement ====> insert into testnodejs.weather (ts, temperature, humidity) values(now, 20.5, 34) +Query OK, 1 row(s) in set (0.00401490s) +[ OK ] time cost: 10 ms, execute statement ====> select * from testnodejs.weather +Connection is closed + + ts | temperature | humidity | +===================================================================== +2020-10-27 18:49:15.547 | 20.5 | 34 | +``` + +## 指南 + +### 如何设置主机名和hosts + +在server上查看hostname和fqdn +```shell +查看hostname +# hostname +taos-server + +查看fqdn +# hostname -f +taos-server +``` + +windows下hosts文件位于： +C:\\Windows\System32\drivers\etc\hosts +修改hosts文件，添加server的ip和hostname + +``` +192.168.56.101 node5 +``` + +> 什么是FQDN？ +> +> FQDN（Full qualified domain name）全限定域名，fqdn由2部分组成：hostname+domainname。 +> +> 例如，一个邮件服务器的fqdn可能是：mymail.somecollege.edu，其中mymail是hostname（主机名），somcollege.edu是domainname（域名）。本例中，.edu是顶级域名，.somecollege是二级域名。 +> +> 当连接服务器时，必须指定fqdn，然后，dns服务器通过查看dns表，将hostname解析为相应的ip地址。如果只指定hostname（不指定domainname），应用程序可能服务解析主机名。因为如果你试图访问不在本地的远程服务器时，本地的dns服务器和可能没有远程服务器的hostname列表。 +> +> 参考：https://kb.iu.edu/d/aiuv diff --git a/tests/examples/nodejs/nodejsChecker.js b/tests/examples/nodejs/nodejsChecker.js index c77944f752..f838d5cc84 100644 --- a/tests/examples/nodejs/nodejsChecker.js +++ b/tests/examples/nodejs/nodejsChecker.js @@ -42,8 +42,8 @@ function executeQuery(sql){ var start = new Date().getTime(); var promise = cursor.query(sql, true); var end = new Date().getTime(); - printSql(sql, promise != null,(end - start)); promise.then(function(result){ + printSql(sql, result != null,(end - start)); result.pretty(); }); } diff --git a/tests/pytest/crash_gen.sh b/tests/pytest/crash_gen.sh index 9cca23ac79..0af09634df 100755 --- a/tests/pytest/crash_gen.sh +++ b/tests/pytest/crash_gen.sh @@ -70,10 +70,12 @@ if [[ $1 == '--valgrind' ]]; then $CRASH_GEN_EXEC $@ > $VALGRIND_OUT 2> $VALGRIND_ERR elif [[ $1 == '--helgrind' ]]; then shift + HELGRIND_OUT=helgrind.out + HELGRIND_ERR=helgrind.err valgrind \ --tool=helgrind \ $PYTHON_EXEC \ - $CRASH_GEN_EXEC $@ + $CRASH_GEN_EXEC $@ > $HELGRIND_OUT 2> $HELGRIND_ERR else $PYTHON_EXEC $CRASH_GEN_EXEC $@ fi diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen.py index 739fb699d6..8d2b0080bc 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen.py @@ -1226,6 +1226,11 @@ class Task(): "To be implemeted by child classes, class name: {}".format( self.__class__.__name__)) + def _isServiceStable(self): + if not gSvcMgr: + return True # we don't run service, so let's assume it's stable + return gSvcMgr.isStable() # otherwise let's examine the service + def _isErrAcceptable(self, errno, msg): if errno in [ 0x05, # TSDB_CODE_RPC_NOT_READY @@ -1263,7 +1268,7 @@ class Task(): return True elif msg.find("duplicated column names") != -1: # also alter table tag issues return True - elif gSvcMgr and (not gSvcMgr.isStable()): # We are managing service, and ... + elif not self._isServiceStable(): # We are managing service, and ... Logging.info("Ignoring error when service starting/stopping: errno = {}, msg = {}".format(errno, msg)) return True @@ -1641,15 +1646,39 @@ class TaskReadData(StateTransitionTask): def canBeginFrom(cls, state: AnyState): return state.canReadData() + # def _canRestartService(self): + # if not gSvcMgr: + # return True # always + # return gSvcMgr.isActive() # only if it's running TODO: race condition here + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): sTable = self._db.getFixedSuperTable() - # 1 in 5 chance, simulate a broken connection. - if random.randrange(5) == 0: # TODO: break connection in all situations - wt.getDbConn().close() - wt.getDbConn().open() - print("_r", end="", flush=True) - + # 1 in 5 chance, simulate a broken connection, only if service stable (not restarting) + if random.randrange(20)==0: # and self._canRestartService(): # TODO: break connection in all situations + # Logging.info("Attempting to reconnect to server") # TODO: change to DEBUG + Progress.emit(Progress.SERVICE_RECONNECT_START) + try: + wt.getDbConn().close() + wt.getDbConn().open() + except ConnectionError as err: # may fail + if not gSvcMgr: + Logging.error("Failed to reconnect in client-only mode") + raise # Not OK if we are running in client-only mode + if gSvcMgr.isRunning(): # may have race conditon, but low prob, due to + Logging.error("Failed to reconnect when managed server is running") + raise # Not OK if we are running normally + + Progress.emit(Progress.SERVICE_RECONNECT_FAILURE) + # Logging.info("Ignoring DB reconnect error") + + # print("_r", end="", flush=True) + Progress.emit(Progress.SERVICE_RECONNECT_SUCCESS) + # The above might have taken a lot of time, service might be running + # by now, causing error below to be incorrectly handled due to timing issue + return # TODO: fix server restart status race condtion + + dbc = wt.getDbConn() dbName = self._db.getName() for rTbName in sTable.getRegTables(dbc, dbName): # regular tables diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 8a2817b389..34a33c6af6 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -163,11 +163,17 @@ class Progress: BEGIN_THREAD_STEP = 1 END_THREAD_STEP = 2 SERVICE_HEART_BEAT= 3 + SERVICE_RECONNECT_START = 4 + SERVICE_RECONNECT_SUCCESS = 5 + SERVICE_RECONNECT_FAILURE = 6 tokens = { STEP_BOUNDARY: '.', BEGIN_THREAD_STEP: '[', END_THREAD_STEP: '] ', - SERVICE_HEART_BEAT: '.Y.' + SERVICE_HEART_BEAT: '.Y.', + SERVICE_RECONNECT_START: '', + SERVICE_RECONNECT_FAILURE: '.xr>', } @classmethod diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index bb2becb55b..196e9d944a 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -280,16 +280,18 @@ class TdeSubProcess: # process still alive, let's interrupt it print("Terminate running process, send SIG_INT and wait...") # sub process should end, then IPC queue should end, causing IO thread to end - self.subProcess.send_signal(signal.SIGINT) + # sig = signal.SIGINT + sig = signal.SIGKILL + self.subProcess.send_signal(sig) # SIGNINT or SIGKILL self.subProcess.wait(20) retCode = self.subProcess.returncode # should always be there # May throw subprocess.TimeoutExpired exception above, therefore # The process is guranteed to have ended by now self.subProcess = None if retCode != 0: # != (- signal.SIGINT): - Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG_INT, retCode={}".format(retCode)) + Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode)) else: - Logging.info("TSP.stop(): sub proc successfully terminated with SIG_INT") + Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig)) return - retCode class ServiceManager: @@ -395,6 +397,13 @@ class ServiceManager: return True return False + def isRunning(self): + for ti in self._tInsts: + if not ti.getStatus().isRunning(): + return False + return True + + # def isRestarting(self): # """ # Determine if the service/cluster is being "restarted", i.e., at least diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index d706065348..b14321a4ef 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -24,6 +24,7 @@ python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py +python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py # tag @@ -150,8 +151,10 @@ python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py python3 ./test.py -f query/queryInsertValue.py python3 ./test.py -f query/queryConnection.py +python3 ./test.py -f query/queryCountCSVData.py python3 ./test.py -f query/natualInterval.py python3 ./test.py -f query/bug1471.py +python3 ./test.py -f query/dataLossTest.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh new file mode 100755 index 0000000000..1a4c12a16c --- /dev/null +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log + +for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` +do +if [ -n "$memError" ]; then + if [ "$memError" -gt 12 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + More than our threshold! ## ${NC}" + fi +fi +done + +grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log +for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` +do + +if [ -n "$defiMemError" ]; then + if [ "$defiMemError" -gt 3 ]; then + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + fi +fi +done \ No newline at end of file diff --git a/tests/pytest/query/dataLossTest.py b/tests/pytest/query/dataLossTest.py new file mode 100644 index 0000000000..b29dc1fa9f --- /dev/null +++ b/tests/pytest/query/dataLossTest.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import inspect + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.numberOfTables = 240 + self.numberOfRecords = 10000 + + def run(self): + tdSql.prepare() + + os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords)) + print("==============step1") + + tdSql.execute("use test") + sql = "select count(*) from meters" + tdSql.query(sql) + rows = tdSql.getData(0, 0) + print ("number of records: %d" % rows) + + newRows = rows + for i in range(10000): + print("kill taosd") + time.sleep(10) + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.startWithoutSleep(1) + while True: + try: + tdSql.query(sql) + newRows = tdSql.getData(0, 0) + print("numer of records after kill taosd %d" % newRows) + time.sleep(10) + break + except Exception as e: + pass + continue + + if newRows < rows: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, newRows, rows) + tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + break + + tdSql.query(sql) + tdSql.checkData(0, 0, rows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryCountCSVData.py b/tests/pytest/query/queryCountCSVData.py new file mode 100644 index 0000000000..6c73425fae --- /dev/null +++ b/tests/pytest/query/queryCountCSVData.py @@ -0,0 +1,71 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + + +class TDTestCase: + """ + create table and insert data from disordered.csv which timestamp is disordered and + ordered.csv which timestamp is ordered. + then execute 'select count(*) from table xx;' + """ + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create database if not exists demo;"); + tdSql.execute("use demo;") + tdSql.execute("CREATE TABLE IF NOT EXISTS test1 (ts TIMESTAMP, ValueID int, " + "VariantValue float, Quality int, Flags int);") + tdSql.execute("CREATE TABLE IF NOT EXISTS test2 (ts TIMESTAMP, ValueID int, " + "VariantValue float, Quality int, Flags int);") + ordered_csv = __file__.split('query')[0] + 'test_data/ordered.csv' + disordered_csv = __file__.split('query')[0] + 'test_data/disordered.csv' + + tdSql.execute(" insert into test1 file '{file}';".format(file=ordered_csv)) + tdSql.execute(" insert into test2 file '{file}';".format(file=disordered_csv)) + print("==============insert into test1 and test2 form test file") + + + print("==============step2") + tdSql.query('select * from test1;') + with open(ordered_csv) as f1: + num1 = len(f1.readlines()) + tdSql.checkRows(num1) + + + tdSql.query('select * from test2;') + with open(disordered_csv) as f2: + num2 = len(f2.readlines()) + tdSql.checkRows(num2) + print("=============execute select count(*) from xxx") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 17027cf498..a5e3ab21b3 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -95,14 +95,16 @@ class TDTestCase: tdSql.error( "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id group by stb_t.id") tdSql.error( - "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;") - tdSql.error( - "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name") + "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.name;") tdSql.execute("alter table stb_t add tag pid int") tdSql.execute("alter table tb_t1 set tag pid=2") tdSql.execute("alter table tb_t2 set tag pid=1") + tdSql.query( + "select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.location = stb_t.name") + tdSql.checkRows(0) + tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.pid") tdSql.checkRows(3) diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index a7fc08c5a3..e09900acc4 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -11,6 +11,7 @@ # -*- coding: utf-8 -*- + import sys import os import taos @@ -32,17 +33,23 @@ class taosdemoQueryPerformace: def query(self): cursor = self.conn.cursor() - cursor.execute("use test") + cursor.execute("use test") totalTime = 0 - for i in range(100): - startTime = time.time() + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() cursor.execute("select count(*) from test.meters") totalTime += time.time() - startTime print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100)) totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.meters") totalTime += time.time() - startTime @@ -50,6 +57,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select count(*) from test.meters where loc='beijing'") totalTime += time.time() - startTime @@ -57,6 +67,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10") totalTime += time.time() - startTime @@ -64,6 +77,9 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)") totalTime += time.time() - startTime @@ -71,11 +87,34 @@ class taosdemoQueryPerformace: totalTime = 0 for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") startTime = time.time() cursor.execute("select last_row(*) from meters") totalTime += time.time() - startTime print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100)) + totalTime = 0 + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() + cursor.execute("select * from meters") + totalTime += time.time() - startTime + print("query time for: select * from meters %f seconds" % (totalTime / 100)) + + totalTime = 0 + for i in range(100): + if(sys.argv[1] == '1'): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + startTime = time.time() + cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'") + totalTime += time.time() - startTime + print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100)) + if __name__ == '__main__': perftest = taosdemoQueryPerformace() perftest.initConnection() diff --git a/tests/pytest/table/create_sensitive.py b/tests/pytest/table/create_sensitive.py new file mode 100644 index 0000000000..1934b662c7 --- /dev/null +++ b/tests/pytest/table/create_sensitive.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +import sys +import string +import random +import subprocess +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table TestSensitiveT(ts timestamp, i int)') + tdSql.execute('create table TestSensitiveT(ts timestamp, i int)') + tdLog.info('create table TestSensitiveSt(ts timestamp,i int) tags(j int)') + tdSql.execute('create table TestSensitiveSt(ts timestamp,i int) tags(j int)') + tdLog.info('create table Abcde using TestSensitiveSt tags(1)') + tdSql.execute('create table AbcdeFgh using TestSensitiveSt tags(1)') + tdLog.info('=============== step2') + tdLog.info('test normal table ') + tdSql.error('create table testsensitivet(ts timestamp, i int)') + tdSql.error('create table testsensitivet(ts timestamp, j int)') + tdSql.error('create table testsensItivet(ts timestamp, j int)') + tdSql.error('create table TESTSENSITIVET(ts timestamp, i int)') + tdLog.info('=============== step3') + tdLog.info('test super table ') + tdSql.error('create table testsensitivest(ts timestamp,i int) tags(j int)') + tdSql.error('create table testsensitivest(ts timestamp,i int) tags(k int)') + tdSql.error('create table TESTSENSITIVEST(ts timestamp,i int) tags(j int)') + tdSql.error('create table Testsensitivest(ts timestamp,i int) tags(j int)') + tdLog.info('=============== step4') + tdLog.info('test subtable ') + tdSql.error('create table abcdefgh using TestSensitiveSt tags(1)') + tdSql.error('create table ABCDEFGH using TestSensitiveSt tags(1)') + tdSql.error('create table Abcdefgh using TestSensitiveSt tags(1)') + tdSql.error('create table abcdeFgh using TestSensitiveSt tags(1)') + tdSql.error('insert into table abcdefgh using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table ABCDEFGH using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table Abcdefgh using TestSensitiveSt tags(1) values(now,1)') + tdSql.error('insert into table abcdeFgH using TestSensitiveSt tags(1) values(now,1)') + tdSql.query('show tables') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(2) + + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/test_data/__init__.py b/tests/pytest/test_data/__init__.py new file mode 100644 index 0000000000..1bd7af5b9d --- /dev/null +++ b/tests/pytest/test_data/__init__.py @@ -0,0 +1,15 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + + +""" +this directory contains test data files +""" \ No newline at end of file diff --git a/tests/pytest/test_data/disordered.csv b/tests/pytest/test_data/disordered.csv new file mode 100644 index 0000000000..0e6fd75e5b --- /dev/null +++ b/tests/pytest/test_data/disordered.csv @@ -0,0 +1,500 @@ +"2020-03-01 20:01:49.493","130","7.595","128","8392704" +"2020-03-01 20:01:50.493","130","7.598","128","8392704" +"2020-03-01 20:01:51.493","130","7.602","128","8392704" +"2020-03-01 20:01:52.493","130","7.604","128","8392704" +"2020-03-01 20:01:53.493","130","7.604","128","8392704" +"2020-03-01 20:01:54.493","130","7.606","128","8392704" +"2020-03-01 20:01:55.493","130","7.607","128","8392704" +"2020-03-01 20:01:56.493","130","7.607","128","8392704" +"2020-03-01 20:01:57.493","130","7.607","128","8392704" +"2020-03-01 20:01:58.493","130","7.607","128","8392704" +"2020-03-01 20:01:59.493","130","7.606","128","8392704" +"2020-03-01 20:02:00.493","130","7.606","128","8392704" +"2020-03-01 20:02:01.493","130","7.606","128","8392704" +"2020-03-01 20:02:02.493","130","7.607","128","8392704" +"2020-03-01 20:02:03.493","130","7.608","128","8392704" +"2020-03-01 20:02:04.493","130","7.609","128","8392704" +"2020-03-01 20:02:05.493","130","7.609","128","8392704" +"2020-03-01 20:02:06.493","130","7.608","128","8392704" +"2020-03-01 20:02:07.493","130","7.606","128","8392704" +"2020-03-01 20:02:08.493","130","7.606","128","8392704" +"2020-03-01 20:02:09.493","130","7.607","128","8392704" +"2020-03-01 20:02:10.493","130","7.609","128","8392704" +"2020-03-01 20:02:11.493","130","7.61","128","8392704" +"2020-03-01 20:02:12.493","130","7.611","128","8392704" +"2020-03-01 20:02:13.493","130","7.61","128","8392704" +"2020-03-01 20:02:14.493","130","7.607","128","8392704" +"2020-03-01 20:02:15.493","130","7.605","128","8392704" +"2020-03-01 20:02:16.493","130","7.604","128","8392704" +"2020-03-01 20:02:17.493","130","7.603","128","8392704" +"2020-03-01 20:02:18.493","130","7.604","128","8392704" +"2020-03-01 20:02:19.493","130","7.604","128","8392704" +"2020-03-01 20:02:20.493","130","7.603","128","8392704" +"2020-03-01 20:02:21.493","130","7.601","128","8392704" +"2020-03-01 20:02:22.493","130","7.598","128","8392704" +"2020-03-01 20:02:23.493","130","7.595","128","8392704" +"2020-03-01 20:02:24.493","130","7.594","128","8392704" +"2020-03-01 20:02:25.493","130","7.594","128","8392704" +"2020-03-01 20:02:26.493","130","7.595","128","8392704" +"2020-03-01 20:02:27.493","130","7.596","128","8392704" +"2020-03-01 20:02:28.493","130","7.596","128","8392704" +"2020-03-01 20:02:29.493","130","7.598","128","8392704" +"2020-03-01 20:02:30.493","130","7.6","128","8392704" +"2020-03-01 20:02:31.493","130","7.6","128","8392704" +"2020-03-01 20:02:32.493","130","7.6","128","8392704" +"2020-03-01 20:02:33.493","130","7.601","128","8392704" +"2020-03-01 20:02:34.493","130","7.603","128","8392704" +"2020-03-01 20:02:35.493","130","7.604","128","8392704" +"2020-03-01 20:02:36.493","130","7.605","128","8392704" +"2020-03-01 20:02:37.493","130","7.606","128","8392704" +"2020-03-01 20:02:38.493","130","7.61","128","8392704" +"2020-03-01 20:02:39.493","130","7.611","128","8392704" +"2020-03-01 20:02:40.493","130","7.61","128","8392704" +"2020-03-01 20:02:41.493","130","7.609","128","8392704" +"2020-03-01 20:02:42.493","130","7.611","128","8392704" +"2020-03-01 20:02:43.493","130","7.61","128","8392704" +"2020-03-01 20:02:44.493","130","7.607","128","8392704" +"2020-03-01 20:02:45.493","130","7.605","128","8392704" +"2020-03-01 20:02:46.493","130","7.606","128","8392704" +"2020-03-01 20:02:47.493","130","7.604","128","8392704" +"2020-03-01 20:02:48.493","130","7.599","128","8392704" +"2020-03-01 20:02:49.493","130","7.595","128","8392704" +"2020-03-01 06:41:17.493","130","6.742","128","8392704" +"2020-03-01 06:41:18.493","130","6.741","128","8392704" +"2020-03-01 06:41:19.493","130","6.737","128","8392704" +"2020-03-01 06:41:20.493","130","6.734","128","8392704" +"2020-03-01 06:41:21.493","130","6.734","128","8392704" +"2020-03-01 06:41:22.493","130","6.733","128","8392704" +"2020-03-01 06:41:23.493","130","6.736","128","8392704" +"2020-03-01 06:41:24.493","130","6.739","128","8392704" +"2020-03-01 06:41:25.493","130","6.738","128","8392704" +"2020-03-01 06:41:26.493","130","6.74","128","8392704" +"2020-03-01 06:41:27.493","130","6.745","128","8392704" +"2020-03-01 06:41:28.493","130","6.749","128","8392704" +"2020-03-01 06:41:29.493","130","6.753","128","8392704" +"2020-03-01 06:41:30.493","130","6.753","128","8392704" +"2020-03-01 06:41:31.493","130","6.757","128","8392704" +"2020-03-01 06:41:32.493","130","6.763","128","8392704" +"2020-03-01 06:41:33.493","130","6.765","128","8392704" +"2020-03-01 06:41:34.493","130","6.764","128","8392704" +"2020-03-01 06:41:35.493","130","6.762","128","8392704" +"2020-03-01 06:41:36.493","130","6.758","128","8392704" +"2020-03-01 06:41:37.493","130","6.756","128","8392704" +"2020-03-01 06:41:38.493","130","6.755","128","8392704" +"2020-03-01 06:41:39.493","130","6.754","128","8392704" +"2020-03-01 06:41:40.493","130","6.755","128","8392704" +"2020-03-01 06:41:41.493","130","6.756","128","8392704" +"2020-03-01 06:41:42.493","130","6.757","128","8392704" +"2020-03-01 06:41:43.493","130","6.756","128","8392704" +"2020-03-01 06:41:44.493","130","6.756","128","8392704" +"2020-03-01 06:41:45.493","130","6.756","128","8392704" +"2020-03-01 06:41:46.493","130","6.759","128","8392704" +"2020-03-01 06:41:47.493","130","6.759","128","8392704" +"2020-03-01 06:41:48.493","130","6.758","128","8392704" +"2020-03-01 06:41:49.493","130","6.758","128","8392704" +"2020-03-01 06:41:50.493","130","6.756","128","8392704" +"2020-03-01 06:41:51.493","130","6.755","128","8392704" +"2020-03-01 06:41:52.493","130","6.755","128","8392704" +"2020-03-01 06:41:53.493","130","6.754","128","8392704" +"2020-03-01 06:41:54.493","130","6.751","128","8392704" +"2020-03-01 06:41:55.493","130","6.752","128","8392704" +"2020-03-01 06:41:56.493","130","6.753","128","8392704" +"2020-03-01 06:41:57.493","130","6.753","128","8392704" +"2020-03-01 06:41:58.493","130","6.753","128","8392704" +"2020-03-01 06:41:59.493","130","6.755","128","8392704" +"2020-03-01 06:42:00.493","130","6.752","128","8392704" +"2020-03-01 06:42:01.493","130","6.75","128","8392704" +"2020-03-01 06:42:02.493","130","6.75","128","8392704" +"2020-03-01 06:42:03.493","130","6.753","128","8392704" +"2020-03-01 06:42:04.493","130","6.755","128","8392704" +"2020-03-01 06:42:05.493","130","6.754","128","8392704" +"2020-03-01 06:42:06.493","130","6.754","128","8392704" +"2020-03-01 06:42:07.493","130","6.752","128","8392704" +"2020-03-01 06:42:08.493","130","6.748","128","8392704" +"2020-03-01 06:42:09.493","130","6.747","128","8392704" +"2020-03-01 06:42:10.493","130","6.747","128","8392704" +"2020-03-01 06:42:11.493","130","6.748","128","8392704" +"2020-03-01 06:42:12.493","130","6.748","128","8392704" +"2020-03-01 06:42:13.493","130","6.75","128","8392704" +"2020-03-01 06:42:14.493","130","6.754","128","8392704" +"2020-03-01 06:42:15.493","130","6.754","128","8392704" +"2020-03-01 06:42:16.493","130","6.756","128","8392704" +"2020-03-01 06:42:17.493","130","6.756","128","8392704" +"2020-03-01 06:42:18.493","130","6.757","128","8392704" +"2020-03-01 06:42:19.493","130","6.757","128","8392704" +"2020-03-01 06:42:20.493","130","6.76","128","8392704" +"2020-03-01 06:42:21.493","130","6.761","128","8392704" +"2020-03-01 06:42:22.493","130","6.76","128","8392704" +"2020-03-01 06:42:23.493","130","6.76","128","8392704" +"2020-03-01 06:42:24.493","130","6.76","128","8392704" +"2020-03-01 06:42:25.493","130","6.76","128","8392704" +"2020-03-01 06:42:26.493","130","6.758","128","8392704" +"2020-03-01 06:42:27.493","130","6.757","128","8392704" +"2020-03-01 06:42:28.493","130","6.752","128","8392704" +"2020-03-01 06:42:29.493","130","6.746","128","8392704" +"2020-03-01 06:42:30.493","130","6.742","128","8392704" +"2020-03-01 06:42:31.493","130","6.741","128","8392704" +"2020-03-01 06:42:32.493","130","6.739","128","8392704" +"2020-03-01 06:42:33.493","130","6.739","128","8392704" +"2020-03-01 06:42:34.493","130","6.737","128","8392704" +"2020-03-01 06:42:35.493","130","6.737","128","8392704" +"2020-03-01 06:42:36.493","130","6.738","128","8392704" +"2020-03-01 06:42:37.493","130","6.739","128","8392704" +"2020-03-01 06:42:38.493","130","6.743","128","8392704" +"2020-03-01 06:42:39.493","130","6.747","128","8392704" +"2020-03-01 06:42:40.493","130","6.748","128","8392704" +"2020-03-01 06:42:41.493","130","6.746","128","8392704" +"2020-03-01 06:42:42.493","130","6.746","128","8392704" +"2020-03-01 06:42:43.493","130","6.745","128","8392704" +"2020-03-01 06:42:44.493","130","6.742","128","8392704" +"2020-03-01 06:42:45.493","130","6.741","128","8392704" +"2020-03-01 06:42:46.493","130","6.74","128","8392704" +"2020-03-01 06:42:47.493","130","6.742","128","8392704" +"2020-03-01 06:42:48.493","130","6.743","128","8392704" +"2020-03-01 06:42:49.493","130","6.742","128","8392704" +"2020-03-01 06:42:50.493","130","6.742","128","8392704" +"2020-03-01 06:42:51.493","130","6.741","128","8392704" +"2020-03-01 06:42:52.493","130","6.741","128","8392704" +"2020-03-01 06:42:53.493","130","6.742","128","8392704" +"2020-03-01 06:42:54.493","130","6.742","128","8392704" +"2020-03-01 06:42:55.493","130","6.745","128","8392704" +"2020-03-01 06:42:56.493","130","6.747","128","8392704" +"2020-03-01 06:42:57.493","130","6.748","128","8392704" +"2020-03-01 06:42:58.493","130","6.75","128","8392704" +"2020-03-01 06:42:59.493","130","6.75","128","8392704" +"2020-03-01 06:43:00.493","130","6.748","128","8392704" +"2020-03-01 06:43:01.493","130","6.748","128","8392704" +"2020-03-01 06:43:02.493","130","6.746","128","8392704" +"2020-03-01 06:43:03.493","130","6.745","128","8392704" +"2020-03-01 06:43:04.493","130","6.745","128","8392704" +"2020-03-01 06:43:05.493","130","6.745","128","8392704" +"2020-03-01 06:43:06.493","130","6.744","128","8392704" +"2020-03-01 06:43:07.493","130","6.749","128","8392704" +"2020-03-01 06:43:08.493","130","6.756","128","8392704" +"2020-03-01 06:43:09.493","130","6.759","128","8392704" +"2020-03-01 06:43:10.493","130","6.759","128","8392704" +"2020-03-01 06:43:11.493","130","6.758","128","8392704" +"2020-03-01 06:43:12.493","130","6.758","128","8392704" +"2020-03-01 06:43:13.493","130","6.759","128","8392704" +"2020-03-01 06:43:14.493","130","6.759","128","8392704" +"2020-03-01 06:43:15.493","130","6.753","128","8392704" +"2020-03-01 06:43:16.493","130","6.751","128","8392704" +"2020-03-01 20:02:50.493","130","7.596","128","8392704" +"2020-03-01 20:02:51.493","130","7.597","128","8392704" +"2020-03-01 20:02:52.493","130","7.599","128","8392704" +"2020-03-01 20:02:53.493","130","7.6","128","8392704" +"2020-03-01 20:02:54.493","130","7.601","128","8392704" +"2020-03-01 20:02:55.493","130","7.603","128","8392704" +"2020-03-01 20:02:56.493","130","7.602","128","8392704" +"2020-03-01 20:02:57.493","130","7.601","128","8392704" +"2020-03-01 20:02:58.493","130","7.601","128","8392704" +"2020-03-01 20:02:59.493","130","7.6","128","8392704" +"2020-03-01 20:03:00.493","130","7.599","128","8392704" +"2020-03-01 20:03:01.493","130","7.599","128","8392704" +"2020-03-01 20:03:02.493","130","7.6","128","8392704" +"2020-03-01 20:03:03.493","130","7.601","128","8392704" +"2020-03-01 20:03:04.493","130","7.601","128","8392704" +"2020-03-01 20:03:05.493","130","7.601","128","8392704" +"2020-03-01 20:03:06.493","130","7.6","128","8392704" +"2020-03-01 20:03:07.493","130","7.602","128","8392704" +"2020-03-01 20:03:08.493","130","7.606","128","8392704" +"2020-03-01 20:03:09.493","130","7.609","128","8392704" +"2020-03-01 20:03:10.493","130","7.612","128","8392704" +"2020-03-01 20:03:11.493","130","7.614","128","8392704" +"2020-03-01 20:03:12.493","130","7.615","128","8392704" +"2020-03-01 20:03:13.493","130","7.614","128","8392704" +"2020-03-01 20:03:14.493","130","7.613","128","8392704" +"2020-03-01 20:03:15.493","130","7.614","128","8392704" +"2020-03-01 20:03:16.493","130","7.612","128","8392704" +"2020-03-01 20:03:17.493","130","7.609","128","8392704" +"2020-03-01 20:03:18.493","130","7.606","128","8392704" +"2020-03-01 20:03:19.493","130","7.604","128","8392704" +"2020-03-01 20:03:20.493","130","7.604","128","8392704" +"2020-03-01 20:03:21.493","130","7.605","128","8392704" +"2020-03-01 20:03:22.493","130","7.605","128","8392704" +"2020-03-01 20:03:23.493","130","7.605","128","8392704" +"2020-03-01 20:03:24.493","130","7.605","128","8392704" +"2020-03-01 20:03:25.493","130","7.604","128","8392704" +"2020-03-01 20:03:26.493","130","7.603","128","8392704" +"2020-03-01 20:03:27.493","130","7.604","128","8392704" +"2020-03-01 20:03:28.493","130","7.605","128","8392704" +"2020-03-01 20:03:29.493","130","7.607","128","8392704" +"2020-03-01 20:03:30.493","130","7.609","128","8392704" +"2020-03-01 20:03:31.493","130","7.609","128","8392704" +"2020-03-01 20:03:32.493","130","7.607","128","8392704" +"2020-03-01 20:03:33.493","130","7.606","128","8392704" +"2020-03-01 20:03:34.493","130","7.607","128","8392704" +"2020-03-01 20:03:35.493","130","7.608","128","8392704" +"2020-03-01 20:03:36.493","130","7.609","128","8392704" +"2020-03-01 20:03:37.493","130","7.609","128","8392704" +"2020-03-01 20:03:38.493","130","7.607","128","8392704" +"2020-03-01 20:03:39.493","130","7.602","128","8392704" +"2020-03-01 20:03:40.493","130","7.599","128","8392704" +"2020-03-01 20:03:41.493","130","7.598","128","8392704" +"2020-03-01 20:03:42.493","130","7.596","128","8392704" +"2020-03-01 20:03:43.493","130","7.595","128","8392704" +"2020-03-01 20:03:44.493","130","7.594","128","8392704" +"2020-03-01 20:03:45.493","130","7.595","128","8392704" +"2020-03-01 20:03:46.493","130","7.597","128","8392704" +"2020-03-01 20:03:47.493","130","7.596","128","8392704" +"2020-03-01 20:03:48.493","130","7.595","128","8392704" +"2020-03-01 20:03:49.493","130","7.596","128","8392704" +"2020-03-01 20:03:50.493","130","7.596","128","8392704" +"2020-03-01 20:03:51.493","130","7.595","128","8392704" +"2020-03-01 20:03:52.493","130","7.596","128","8392704" +"2020-03-01 20:03:53.493","130","7.597","128","8392704" +"2020-03-01 20:03:54.493","130","7.598","128","8392704" +"2020-03-01 20:03:55.493","130","7.596","128","8392704" +"2020-03-01 20:03:56.493","130","7.596","128","8392704" +"2020-03-01 20:03:57.493","130","7.599","128","8392704" +"2020-03-01 20:03:58.493","130","7.602","128","8392704" +"2020-03-01 20:03:59.493","130","7.603","128","8392704" +"2020-03-01 20:04:00.493","130","7.602","128","8392704" +"2020-03-01 20:04:01.493","130","7.6","128","8392704" +"2020-03-01 20:04:02.493","130","7.598","128","8392704" +"2020-03-01 20:04:03.493","130","7.595","128","8392704" +"2020-03-01 20:04:04.493","130","7.593","128","8392704" +"2020-03-01 20:04:05.493","130","7.592","128","8392704" +"2020-03-01 20:04:06.493","130","7.591","128","8392704" +"2020-03-01 20:04:07.493","130","7.591","128","8392704" +"2020-03-01 20:04:08.493","130","7.591","128","8392704" +"2020-03-01 20:04:09.493","130","7.592","128","8392704" +"2020-03-01 20:04:10.493","130","7.59","128","8392704" +"2020-03-01 20:04:11.493","130","7.587","128","8392704" +"2020-03-01 20:04:12.493","130","7.584","128","8392704" +"2020-03-01 20:04:13.493","130","7.583","128","8392704" +"2020-03-01 20:04:14.493","130","7.581","128","8392704" +"2020-03-01 20:04:15.493","130","7.578","128","8392704" +"2020-03-01 20:04:16.493","130","7.576","128","8392704" +"2020-03-01 20:04:17.493","130","7.577","128","8392704" +"2020-03-01 20:04:18.493","130","7.579","128","8392704" +"2020-03-01 20:04:19.493","130","7.583","128","8392704" +"2020-03-01 20:04:20.493","130","7.587","128","8392704" +"2020-03-01 20:04:21.493","130","7.588","128","8392704" +"2020-03-01 20:04:22.493","130","7.589","128","8392704" +"2020-03-01 20:04:23.493","130","7.59","128","8392704" +"2020-03-01 20:04:24.493","130","7.593","128","8392704" +"2020-03-01 20:04:25.493","130","7.597","128","8392704" +"2020-03-01 20:04:26.493","130","7.6","128","8392704" +"2020-03-01 20:04:27.493","130","7.603","128","8392704" +"2020-03-01 20:04:28.493","130","7.606","128","8392704" +"2020-03-01 20:04:29.493","130","7.608","128","8392704" +"2020-03-01 20:04:30.493","130","7.609","128","8392704" +"2020-03-01 20:04:31.493","130","7.607","128","8392704" +"2020-03-01 20:04:32.493","130","7.607","128","8392704" +"2020-03-01 20:04:33.493","130","7.607","128","8392704" +"2020-03-01 20:04:34.493","130","7.602","128","8392704" +"2020-03-01 20:04:35.493","130","7.599","128","8392704" +"2020-03-01 20:04:36.493","130","7.599","128","8392704" +"2020-03-01 20:04:37.493","130","7.599","128","8392704" +"2020-03-01 20:04:38.493","130","7.598","128","8392704" +"2020-03-01 20:04:39.493","130","7.596","128","8392704" +"2020-03-01 20:04:40.493","130","7.595","128","8392704" +"2020-03-01 20:04:41.493","130","7.592","128","8392704" +"2020-03-01 20:04:42.493","130","7.586","128","8392704" +"2020-03-01 20:04:43.493","130","7.582","128","8392704" +"2020-03-01 20:04:44.493","130","7.582","128","8392704" +"2020-03-01 20:04:45.493","130","7.584","128","8392704" +"2020-03-01 20:04:46.493","130","7.583","128","8392704" +"2020-03-01 20:04:47.493","130","7.582","128","8392704" +"2020-03-01 20:04:48.493","130","7.582","128","8392704" +"2020-03-01 20:04:49.493","130","7.585","128","8392704" +"2020-03-01 06:43:17.493","130","6.751","128","8392704" +"2020-03-01 06:43:18.493","130","6.75","128","8392704" +"2020-03-01 06:43:19.493","130","6.748","128","8392704" +"2020-03-01 06:43:20.493","130","6.751","128","8392704" +"2020-03-01 06:43:21.493","130","6.752","128","8392704" +"2020-03-01 06:43:22.493","130","6.751","128","8392704" +"2020-03-01 06:43:23.493","130","6.746","128","8392704" +"2020-03-01 06:43:24.493","130","6.739","128","8392704" +"2020-03-01 06:43:25.493","130","6.737","128","8392704" +"2020-03-01 06:43:26.493","130","6.735","128","8392704" +"2020-03-01 06:43:27.493","130","6.735","128","8392704" +"2020-03-01 06:43:28.493","130","6.734","128","8392704" +"2020-03-01 06:43:29.493","130","6.731","128","8392704" +"2020-03-01 06:43:30.493","130","6.729","128","8392704" +"2020-03-01 06:43:31.493","130","6.73","128","8392704" +"2020-03-01 06:43:32.493","130","6.736","128","8392704" +"2020-03-01 06:43:33.493","130","6.74","128","8392704" +"2020-03-01 06:43:34.493","130","6.741","128","8392704" +"2020-03-01 06:43:35.493","130","6.743","128","8392704" +"2020-03-01 06:43:36.493","130","6.743","128","8392704" +"2020-03-01 06:43:37.493","130","6.745","128","8392704" +"2020-03-01 06:43:38.493","130","6.747","128","8392704" +"2020-03-01 06:43:39.493","130","6.747","128","8392704" +"2020-03-01 06:43:40.493","130","6.746","128","8392704" +"2020-03-01 06:43:41.493","130","6.745","128","8392704" +"2020-03-01 06:43:42.493","130","6.743","128","8392704" +"2020-03-01 06:43:43.493","130","6.741","128","8392704" +"2020-03-01 06:43:44.493","130","6.737","128","8392704" +"2020-03-01 06:43:45.493","130","6.737","128","8392704" +"2020-03-01 06:43:46.493","130","6.74","128","8392704" +"2020-03-01 06:43:47.493","130","6.744","128","8392704" +"2020-03-01 06:43:48.493","130","6.746","128","8392704" +"2020-03-01 06:43:49.493","130","6.745","128","8392704" +"2020-03-01 06:43:50.493","130","6.743","128","8392704" +"2020-03-01 06:43:51.493","130","6.745","128","8392704" +"2020-03-01 06:43:52.493","130","6.747","128","8392704" +"2020-03-01 06:43:53.493","130","6.748","128","8392704" +"2020-03-01 06:43:54.493","130","6.748","128","8392704" +"2020-03-01 06:43:55.493","130","6.747","128","8392704" +"2020-03-01 06:43:56.493","130","6.746","128","8392704" +"2020-03-01 06:43:57.493","130","6.744","128","8392704" +"2020-03-01 06:43:58.493","130","6.742","128","8392704" +"2020-03-01 06:43:59.493","130","6.74","128","8392704" +"2020-03-01 06:44:00.493","130","6.739","128","8392704" +"2020-03-01 06:44:01.493","130","6.739","128","8392704" +"2020-03-01 06:44:02.493","130","6.742","128","8392704" +"2020-03-01 06:44:03.493","130","6.742","128","8392704" +"2020-03-01 06:44:04.493","130","6.756","128","8392704" +"2020-03-01 06:44:05.493","130","6.757","128","8392704" +"2020-03-01 06:44:06.493","130","6.757","128","8392704" +"2020-03-01 06:44:07.493","130","6.757","128","8392704" +"2020-03-01 06:44:08.493","130","6.759","128","8392704" +"2020-03-01 06:44:09.493","130","6.759","128","8392704" +"2020-03-01 06:44:10.493","130","6.75","128","8392704" +"2020-03-01 06:44:11.493","130","6.744","128","8392704" +"2020-03-01 06:44:12.493","130","6.739","128","8392704" +"2020-03-01 06:44:13.493","130","6.739","128","8392704" +"2020-03-01 06:44:14.493","130","6.736","128","8392704" +"2020-03-01 06:44:15.493","130","6.734","128","8392704" +"2020-03-01 06:44:16.493","130","6.735","128","8392704" +"2020-03-01 06:44:17.493","130","6.734","128","8392704" +"2020-03-01 06:44:18.493","130","6.736","128","8392704" +"2020-03-01 06:44:19.493","130","6.741","128","8392704" +"2020-03-01 06:44:20.493","130","6.744","128","8392704" +"2020-03-01 06:44:21.493","130","6.746","128","8392704" +"2020-03-01 06:44:22.493","130","6.746","128","8392704" +"2020-03-01 06:44:23.493","130","6.748","128","8392704" +"2020-03-01 06:44:24.493","130","6.751","128","8392704" +"2020-03-01 06:44:25.493","130","6.752","128","8392704" +"2020-03-01 06:44:26.493","130","6.752","128","8392704" +"2020-03-01 06:44:27.493","130","6.752","128","8392704" +"2020-03-01 06:44:28.493","130","6.753","128","8392704" +"2020-03-01 06:44:29.493","130","6.751","128","8392704" +"2020-03-01 06:44:30.493","130","6.751","128","8392704" +"2020-03-01 06:44:31.493","130","6.749","128","8392704" +"2020-03-01 06:44:32.493","130","6.747","128","8392704" +"2020-03-01 06:44:33.493","130","6.748","128","8392704" +"2020-03-01 06:44:34.493","130","6.749","128","8392704" +"2020-03-01 06:44:35.493","130","6.746","128","8392704" +"2020-03-01 06:44:36.493","130","6.742","128","8392704" +"2020-03-01 06:44:37.493","130","6.742","128","8392704" +"2020-03-01 06:44:38.493","130","6.743","128","8392704" +"2020-03-01 06:44:39.493","130","6.743","128","8392704" +"2020-03-01 06:44:40.493","130","6.743","128","8392704" +"2020-03-01 06:44:41.493","130","6.741","128","8392704" +"2020-03-01 06:44:42.493","130","6.741","128","8392704" +"2020-03-01 06:44:43.493","130","6.741","128","8392704" +"2020-03-01 06:44:44.493","130","6.74","128","8392704" +"2020-03-01 06:44:45.493","130","6.74","128","8392704" +"2020-03-01 06:44:46.493","130","6.739","128","8392704" +"2020-03-01 06:44:47.493","130","6.738","128","8392704" +"2020-03-01 06:44:48.493","130","6.738","128","8392704" +"2020-03-01 06:44:49.493","130","6.741","128","8392704" +"2020-03-01 06:44:50.493","130","6.749","128","8392704" +"2020-03-01 06:44:51.493","130","6.756","128","8392704" +"2020-03-01 06:44:52.493","130","6.763","128","8392704" +"2020-03-01 06:44:53.493","130","6.768","128","8392704" +"2020-03-01 06:44:54.493","130","6.771","128","8392704" +"2020-03-01 06:44:55.493","130","6.774","128","8392704" +"2020-03-01 06:44:56.493","130","6.774","128","8392704" +"2020-03-01 06:44:57.493","130","6.774","128","8392704" +"2020-03-01 06:44:58.493","130","6.765","128","8392704" +"2020-03-01 06:44:59.493","130","6.763","128","8392704" +"2020-03-01 06:45:00.493","130","6.761","128","8392704" +"2020-03-01 06:45:01.493","130","6.758","128","8392704" +"2020-03-01 06:45:02.493","130","6.756","128","8392704" +"2020-03-01 06:45:03.493","130","6.756","128","8392704" +"2020-03-01 06:45:04.493","130","6.756","128","8392704" +"2020-03-01 06:45:05.493","130","6.763","128","8392704" +"2020-03-01 06:45:06.493","130","6.763","128","8392704" +"2020-03-01 06:45:07.493","130","6.764","128","8392704" +"2020-03-01 06:45:08.493","130","6.762","128","8392704" +"2020-03-01 06:45:09.493","130","6.763","128","8392704" +"2020-03-01 06:45:10.493","130","6.764","128","8392704" +"2020-03-01 06:45:11.493","130","6.763","128","8392704" +"2020-03-01 06:45:12.493","130","6.76","128","8392704" +"2020-03-01 06:45:13.493","130","6.759","128","8392704" +"2020-03-01 06:45:14.493","130","6.758","128","8392704" +"2020-03-01 06:45:15.493","130","6.758","128","8392704" +"2020-03-01 06:45:16.493","130","6.755","128","8392704" +"2020-03-01 20:04:50.493","130","7.59","128","8392704" +"2020-03-01 20:04:51.493","130","7.592","128","8392704" +"2020-03-01 20:04:52.493","130","7.592","128","8392704" +"2020-03-01 20:04:53.493","130","7.593","128","8392704" +"2020-03-01 20:04:54.493","130","7.592","128","8392704" +"2020-03-01 20:04:55.493","130","7.592","128","8392704" +"2020-03-01 20:04:56.493","130","7.593","128","8392704" +"2020-03-01 20:04:57.493","130","7.593","128","8392704" +"2020-03-01 20:04:58.493","130","7.593","128","8392704" +"2020-03-01 20:04:59.493","130","7.594","128","8392704" +"2020-03-01 20:05:00.493","130","7.595","128","8392704" +"2020-03-01 20:05:01.493","130","7.596","128","8392704" +"2020-03-01 20:05:02.493","130","7.595","128","8392704" +"2020-03-01 20:05:03.493","130","7.595","128","8392704" +"2020-03-01 20:05:04.493","130","7.594","128","8392704" +"2020-03-01 20:05:05.493","130","7.595","128","8392704" +"2020-03-01 20:05:06.493","130","7.598","128","8392704" +"2020-03-01 20:05:07.493","130","7.597","128","8392704" +"2020-03-01 20:05:08.493","130","7.595","128","8392704" +"2020-03-01 20:05:09.493","130","7.597","128","8392704" +"2020-03-01 20:05:10.493","130","7.598","128","8392704" +"2020-03-01 20:05:11.493","130","7.598","128","8392704" +"2020-03-01 20:05:12.493","130","7.597","128","8392704" +"2020-03-01 20:05:13.493","130","7.595","128","8392704" +"2020-03-01 20:05:14.493","130","7.591","128","8392704" +"2020-03-01 20:05:15.493","130","7.589","128","8392704" +"2020-03-01 20:05:16.493","130","7.588","128","8392704" +"2020-03-01 20:05:17.493","130","7.589","128","8392704" +"2020-03-01 20:05:18.493","130","7.589","128","8392704" +"2020-03-01 20:05:19.493","130","7.589","128","8392704" +"2020-03-01 20:05:20.493","130","7.587","128","8392704" +"2020-03-01 20:05:21.493","130","7.584","128","8392704" +"2020-03-01 20:05:22.493","130","7.583","128","8392704" +"2020-03-01 20:05:23.493","130","7.585","128","8392704" +"2020-03-01 20:05:24.493","130","7.586","128","8392704" +"2020-03-01 20:05:25.493","130","7.586","128","8392704" +"2020-03-01 20:05:26.493","130","7.586","128","8392704" +"2020-03-01 20:05:27.493","130","7.586","128","8392704" +"2020-03-01 20:05:28.493","130","7.587","128","8392704" +"2020-03-01 20:05:29.493","130","7.585","128","8392704" +"2020-03-01 20:05:30.493","130","7.584","128","8392704" +"2020-03-01 20:05:31.493","130","7.586","128","8392704" +"2020-03-01 20:05:32.493","130","7.589","128","8392704" +"2020-03-01 20:05:33.493","130","7.59","128","8392704" +"2020-03-01 20:05:34.493","130","7.591","128","8392704" +"2020-03-01 20:05:35.493","130","7.591","128","8392704" +"2020-03-01 20:05:36.493","130","7.594","128","8392704" +"2020-03-01 20:05:37.493","130","7.599","128","8392704" +"2020-03-01 20:05:38.493","130","7.602","128","8392704" +"2020-03-01 20:05:39.493","130","7.604","128","8392704" +"2020-03-01 20:05:40.493","130","7.605","128","8392704" +"2020-03-01 20:05:41.493","130","7.607","128","8392704" +"2020-03-01 20:05:42.493","130","7.607","128","8392704" +"2020-03-01 20:05:43.493","130","7.604","128","8392704" +"2020-03-01 20:05:44.493","130","7.597","128","8392704" +"2020-03-01 20:05:45.493","130","7.592","128","8392704" +"2020-03-01 20:05:46.493","130","7.59","128","8392704" +"2020-03-01 20:05:47.493","130","7.59","128","8392704" +"2020-03-01 20:05:48.493","130","7.591","128","8392704" +"2020-03-01 20:05:49.493","130","7.591","128","8392704" +"2020-03-01 20:05:50.493","130","7.591","128","8392704" +"2020-03-01 20:05:51.493","130","7.594","128","8392704" +"2020-03-01 20:05:52.493","130","7.599","128","8392704" +"2020-03-01 20:05:53.493","130","7.601","128","8392704" +"2020-03-01 20:05:54.493","130","7.602","128","8392704" +"2020-03-01 20:05:55.493","130","7.602","128","8392704" +"2020-03-01 20:05:56.493","130","7.602","128","8392704" +"2020-03-01 20:05:57.493","130","7.603","128","8392704" +"2020-03-01 20:05:58.493","130","7.604","128","8392704" +"2020-03-01 20:05:59.493","130","7.604","128","8392704" +"2020-03-01 20:06:00.493","130","7.605","128","8392704" +"2020-03-01 20:06:01.493","130","7.606","128","8392704" +"2020-03-01 20:06:02.493","130","7.607","128","8392704" +"2020-03-01 20:06:03.493","130","7.605","128","8392704" +"2020-03-01 20:06:04.493","130","7.604","128","8392704" +"2020-03-01 20:06:05.493","130","7.603","128","8392704" +"2020-03-01 20:06:06.493","130","7.602","128","8392704" +"2020-03-01 20:06:07.493","130","7.603","128","8392704" +"2020-03-01 20:06:08.493","130","7.604","128","8392704" \ No newline at end of file diff --git a/tests/pytest/test_data/ordered.csv b/tests/pytest/test_data/ordered.csv new file mode 100644 index 0000000000..14da572d75 --- /dev/null +++ b/tests/pytest/test_data/ordered.csv @@ -0,0 +1,500 @@ +"2020-03-01 19:46:50.493","130","7.617","128","8392704" +"2020-03-01 19:46:51.493","130","7.615","128","8392704" +"2020-03-01 19:46:52.493","130","7.613","128","8392704" +"2020-03-01 19:46:53.493","130","7.612","128","8392704" +"2020-03-01 19:46:54.493","130","7.611","128","8392704" +"2020-03-01 19:46:55.493","130","7.612","128","8392704" +"2020-03-01 19:46:56.493","130","7.611","128","8392704" +"2020-03-01 19:46:57.493","130","7.61","128","8392704" +"2020-03-01 19:46:58.493","130","7.61","128","8392704" +"2020-03-01 19:46:59.493","130","7.613","128","8392704" +"2020-03-01 19:47:00.493","130","7.617","128","8392704" +"2020-03-01 19:47:01.493","130","7.618","128","8392704" +"2020-03-01 19:47:02.493","130","7.619","128","8392704" +"2020-03-01 19:47:03.493","130","7.62","128","8392704" +"2020-03-01 19:47:04.493","130","7.619","128","8392704" +"2020-03-01 19:47:05.493","130","7.62","128","8392704" +"2020-03-01 19:47:06.493","130","7.62","128","8392704" +"2020-03-01 19:47:07.493","130","7.618","128","8392704" +"2020-03-01 19:47:08.493","130","7.618","128","8392704" +"2020-03-01 19:47:09.493","130","7.616","128","8392704" +"2020-03-01 19:47:10.493","130","7.615","128","8392704" +"2020-03-01 19:47:11.493","130","7.614","128","8392704" +"2020-03-01 19:47:12.493","130","7.614","128","8392704" +"2020-03-01 19:47:13.493","130","7.615","128","8392704" +"2020-03-01 19:47:14.493","130","7.617","128","8392704" +"2020-03-01 19:47:15.493","130","7.617","128","8392704" +"2020-03-01 19:47:16.493","130","7.612","128","8392704" +"2020-03-01 19:47:17.493","130","7.609","128","8392704" +"2020-03-01 19:47:18.493","130","7.609","128","8392704" +"2020-03-01 19:47:19.493","130","7.609","128","8392704" +"2020-03-01 19:47:20.493","130","7.611","128","8392704" +"2020-03-01 19:47:21.493","130","7.613","128","8392704" +"2020-03-01 19:47:22.493","130","7.612","128","8392704" +"2020-03-01 19:47:23.493","130","7.612","128","8392704" +"2020-03-01 19:47:24.493","130","7.612","128","8392704" +"2020-03-01 19:47:25.493","130","7.613","128","8392704" +"2020-03-01 19:47:26.493","130","7.617","128","8392704" +"2020-03-01 19:47:27.493","130","7.62","128","8392704" +"2020-03-01 19:47:28.493","130","7.621","128","8392704" +"2020-03-01 19:47:29.493","130","7.621","128","8392704" +"2020-03-01 19:47:30.493","130","7.623","128","8392704" +"2020-03-01 19:47:31.493","130","7.624","128","8392704" +"2020-03-01 19:47:32.493","130","7.621","128","8392704" +"2020-03-01 19:47:33.493","130","7.619","128","8392704" +"2020-03-01 19:47:34.493","130","7.618","128","8392704" +"2020-03-01 19:47:35.493","130","7.616","128","8392704" +"2020-03-01 19:47:36.493","130","7.618","128","8392704" +"2020-03-01 19:47:37.493","130","7.618","128","8392704" +"2020-03-01 19:47:38.493","130","7.616","128","8392704" +"2020-03-01 19:47:39.493","130","7.615","128","8392704" +"2020-03-01 19:47:40.493","130","7.615","128","8392704" +"2020-03-01 19:47:41.493","130","7.614","128","8392704" +"2020-03-01 19:47:42.493","130","7.613","128","8392704" +"2020-03-01 19:47:43.493","130","7.612","128","8392704" +"2020-03-01 19:47:44.493","130","7.611","128","8392704" +"2020-03-01 19:47:45.493","130","7.612","128","8392704" +"2020-03-01 19:47:46.493","130","7.614","128","8392704" +"2020-03-01 19:47:47.493","130","7.618","128","8392704" +"2020-03-01 19:47:48.493","130","7.62","128","8392704" +"2020-03-01 19:47:49.493","130","7.62","128","8392704" +"2020-03-01 19:47:50.493","130","7.621","128","8392704" +"2020-03-01 19:47:51.493","130","7.62","128","8392704" +"2020-03-01 19:47:52.493","130","7.619","128","8392704" +"2020-03-01 19:47:53.493","130","7.621","128","8392704" +"2020-03-01 19:47:54.493","130","7.622","128","8392704" +"2020-03-01 19:47:55.493","130","7.622","128","8392704" +"2020-03-01 19:47:56.493","130","7.62","128","8392704" +"2020-03-01 19:47:57.493","130","7.617","128","8392704" +"2020-03-01 19:47:58.493","130","7.616","128","8392704" +"2020-03-01 19:47:59.493","130","7.618","128","8392704" +"2020-03-01 19:48:00.493","130","7.62","128","8392704" +"2020-03-01 19:48:01.493","130","7.62","128","8392704" +"2020-03-01 19:48:02.493","130","7.616","128","8392704" +"2020-03-01 19:48:03.493","130","7.612","128","8392704" +"2020-03-01 19:48:04.493","130","7.609","128","8392704" +"2020-03-01 19:48:05.493","130","7.608","128","8392704" +"2020-03-01 19:48:06.493","130","7.605","128","8392704" +"2020-03-01 19:48:07.493","130","7.604","128","8392704" +"2020-03-01 19:48:08.493","130","7.605","128","8392704" +"2020-03-01 19:48:09.493","130","7.604","128","8392704" +"2020-03-01 19:48:10.493","130","7.604","128","8392704" +"2020-03-01 19:48:11.493","130","7.608","128","8392704" +"2020-03-01 19:48:12.493","130","7.611","128","8392704" +"2020-03-01 19:48:13.493","130","7.614","128","8392704" +"2020-03-01 19:48:14.493","130","7.616","128","8392704" +"2020-03-01 19:48:15.493","130","7.618","128","8392704" +"2020-03-01 19:48:16.493","130","7.62","128","8392704" +"2020-03-01 19:48:17.493","130","7.617","128","8392704" +"2020-03-01 19:48:18.493","130","7.61","128","8392704" +"2020-03-01 19:48:19.493","130","7.607","128","8392704" +"2020-03-01 19:48:20.493","130","7.604","128","8392704" +"2020-03-01 19:48:21.493","130","7.601","128","8392704" +"2020-03-01 19:48:22.493","130","7.601","128","8392704" +"2020-03-01 19:48:23.493","130","7.601","128","8392704" +"2020-03-01 19:48:24.493","130","7.598","128","8392704" +"2020-03-01 19:48:25.493","130","7.598","128","8392704" +"2020-03-01 19:48:26.493","130","7.604","128","8392704" +"2020-03-01 19:48:27.493","130","7.608","128","8392704" +"2020-03-01 19:48:28.493","130","7.609","128","8392704" +"2020-03-01 19:48:29.493","130","7.61","128","8392704" +"2020-03-01 19:48:30.493","130","7.611","128","8392704" +"2020-03-01 19:48:31.493","130","7.614","128","8392704" +"2020-03-01 19:48:32.493","130","7.614","128","8392704" +"2020-03-01 19:48:33.493","130","7.611","128","8392704" +"2020-03-01 19:48:34.493","130","7.607","128","8392704" +"2020-03-01 19:48:35.493","130","7.601","128","8392704" +"2020-03-01 19:48:36.493","130","7.596","128","8392704" +"2020-03-01 19:48:37.493","130","7.593","128","8392704" +"2020-03-01 19:48:38.493","130","7.593","128","8392704" +"2020-03-01 19:48:39.493","130","7.593","128","8392704" +"2020-03-01 19:48:40.493","130","7.595","128","8392704" +"2020-03-01 19:48:41.493","130","7.596","128","8392704" +"2020-03-01 19:48:42.493","130","7.599","128","8392704" +"2020-03-01 19:48:43.493","130","7.603","128","8392704" +"2020-03-01 19:48:44.493","130","7.605","128","8392704" +"2020-03-01 19:48:45.493","130","7.607","128","8392704" +"2020-03-01 19:48:46.493","130","7.608","128","8392704" +"2020-03-01 19:48:47.493","130","7.609","128","8392704" +"2020-03-01 19:48:48.493","130","7.61","128","8392704" +"2020-03-01 19:48:49.493","130","7.608","128","8392704" +"2020-03-01 19:48:50.493","130","7.605","128","8392704" +"2020-03-01 19:48:51.493","130","7.605","128","8392704" +"2020-03-01 19:48:52.493","130","7.607","128","8392704" +"2020-03-01 19:48:53.493","130","7.608","128","8392704" +"2020-03-01 19:48:54.493","130","7.608","128","8392704" +"2020-03-01 19:48:55.493","130","7.608","128","8392704" +"2020-03-01 19:48:56.493","130","7.61","128","8392704" +"2020-03-01 19:48:57.493","130","7.613","128","8392704" +"2020-03-01 19:48:58.493","130","7.612","128","8392704" +"2020-03-01 19:48:59.493","130","7.61","128","8392704" +"2020-03-01 19:49:00.493","130","7.609","128","8392704" +"2020-03-01 19:49:01.493","130","7.61","128","8392704" +"2020-03-01 19:49:02.493","130","7.611","128","8392704" +"2020-03-01 19:49:03.493","130","7.61","128","8392704" +"2020-03-01 19:49:04.493","130","7.61","128","8392704" +"2020-03-01 19:49:05.493","130","7.613","128","8392704" +"2020-03-01 19:49:06.493","130","7.615","128","8392704" +"2020-03-01 19:49:07.493","130","7.614","128","8392704" +"2020-03-01 19:49:08.493","130","7.613","128","8392704" +"2020-03-01 19:49:09.493","130","7.613","128","8392704" +"2020-03-01 19:49:10.493","130","7.615","128","8392704" +"2020-03-01 19:49:11.493","130","7.619","128","8392704" +"2020-03-01 19:49:12.493","130","7.62","128","8392704" +"2020-03-01 19:49:13.493","130","7.618","128","8392704" +"2020-03-01 19:49:14.493","130","7.619","128","8392704" +"2020-03-01 19:49:15.493","130","7.618","128","8392704" +"2020-03-01 19:49:16.493","130","7.617","128","8392704" +"2020-03-01 19:49:17.493","130","7.617","128","8392704" +"2020-03-01 19:49:18.493","130","7.618","128","8392704" +"2020-03-01 19:49:19.493","130","7.617","128","8392704" +"2020-03-01 19:49:20.493","130","7.616","128","8392704" +"2020-03-01 19:49:21.493","130","7.615","128","8392704" +"2020-03-01 19:49:22.493","130","7.616","128","8392704" +"2020-03-01 19:49:23.493","130","7.617","128","8392704" +"2020-03-01 19:49:24.493","130","7.615","128","8392704" +"2020-03-01 19:49:25.493","130","7.613","128","8392704" +"2020-03-01 19:49:26.493","130","7.612","128","8392704" +"2020-03-01 19:49:27.493","130","7.613","128","8392704" +"2020-03-01 19:49:28.493","130","7.614","128","8392704" +"2020-03-01 19:49:29.493","130","7.612","128","8392704" +"2020-03-01 19:49:30.493","130","7.611","128","8392704" +"2020-03-01 19:49:31.493","130","7.611","128","8392704" +"2020-03-01 19:49:32.493","130","7.612","128","8392704" +"2020-03-01 19:49:33.493","130","7.613","128","8392704" +"2020-03-01 19:49:34.493","130","7.614","128","8392704" +"2020-03-01 19:49:35.493","130","7.612","128","8392704" +"2020-03-01 19:49:36.493","130","7.607","128","8392704" +"2020-03-01 19:49:37.493","130","7.603","128","8392704" +"2020-03-01 19:49:38.493","130","7.599","128","8392704" +"2020-03-01 19:49:39.493","130","7.599","128","8392704" +"2020-03-01 19:49:40.493","130","7.599","128","8392704" +"2020-03-01 19:49:41.493","130","7.599","128","8392704" +"2020-03-01 19:49:42.493","130","7.601","128","8392704" +"2020-03-01 19:49:43.493","130","7.605","128","8392704" +"2020-03-01 19:49:44.493","130","7.606","128","8392704" +"2020-03-01 19:49:45.493","130","7.606","128","8392704" +"2020-03-01 19:49:46.493","130","7.606","128","8392704" +"2020-03-01 19:49:47.493","130","7.604","128","8392704" +"2020-03-01 19:49:48.493","130","7.604","128","8392704" +"2020-03-01 19:49:49.493","130","7.603","128","8392704" +"2020-03-01 19:49:50.493","130","7.604","128","8392704" +"2020-03-01 19:49:51.493","130","7.608","128","8392704" +"2020-03-01 19:49:52.493","130","7.614","128","8392704" +"2020-03-01 19:49:53.493","130","7.618","128","8392704" +"2020-03-01 19:49:54.493","130","7.621","128","8392704" +"2020-03-01 19:49:55.493","130","7.623","128","8392704" +"2020-03-01 19:49:56.493","130","7.623","128","8392704" +"2020-03-01 19:49:57.493","130","7.624","128","8392704" +"2020-03-01 19:49:58.493","130","7.626","128","8392704" +"2020-03-01 19:49:59.493","130","7.628","128","8392704" +"2020-03-01 19:50:00.493","130","7.627","128","8392704" +"2020-03-01 19:50:01.493","130","7.625","128","8392704" +"2020-03-01 19:50:02.493","130","7.627","128","8392704" +"2020-03-01 19:50:03.493","130","7.63","128","8392704" +"2020-03-01 19:50:04.493","130","7.633","128","8392704" +"2020-03-01 19:50:05.493","130","7.635","128","8392704" +"2020-03-01 19:50:06.493","130","7.634","128","8392704" +"2020-03-01 19:50:07.493","130","7.632","128","8392704" +"2020-03-01 19:50:08.493","130","7.628","128","8392704" +"2020-03-01 19:50:09.493","130","7.625","128","8392704" +"2020-03-01 19:50:10.493","130","7.625","128","8392704" +"2020-03-01 19:50:11.493","130","7.623","128","8392704" +"2020-03-01 19:50:12.493","130","7.623","128","8392704" +"2020-03-01 19:50:13.493","130","7.623","128","8392704" +"2020-03-01 19:50:14.493","130","7.622","128","8392704" +"2020-03-01 19:50:15.493","130","7.621","128","8392704" +"2020-03-01 19:50:16.493","130","7.618","128","8392704" +"2020-03-01 19:50:17.493","130","7.618","128","8392704" +"2020-03-01 19:50:18.493","130","7.617","128","8392704" +"2020-03-01 19:50:19.493","130","7.616","128","8392704" +"2020-03-01 19:50:20.493","130","7.615","128","8392704" +"2020-03-01 19:50:21.493","130","7.615","128","8392704" +"2020-03-01 19:50:22.493","130","7.616","128","8392704" +"2020-03-01 19:50:23.493","130","7.619","128","8392704" +"2020-03-01 19:50:24.493","130","7.622","128","8392704" +"2020-03-01 19:50:25.493","130","7.624","128","8392704" +"2020-03-01 19:50:26.493","130","7.627","128","8392704" +"2020-03-01 19:50:27.493","130","7.627","128","8392704" +"2020-03-01 19:50:28.493","130","7.625","128","8392704" +"2020-03-01 19:50:29.493","130","7.625","128","8392704" +"2020-03-01 19:50:30.493","130","7.625","128","8392704" +"2020-03-01 19:50:31.493","130","7.624","128","8392704" +"2020-03-01 19:50:32.493","130","7.624","128","8392704" +"2020-03-01 19:50:33.493","130","7.624","128","8392704" +"2020-03-01 19:50:34.493","130","7.626","128","8392704" +"2020-03-01 19:50:35.493","130","7.627","128","8392704" +"2020-03-01 19:50:36.493","130","7.627","128","8392704" +"2020-03-01 19:50:37.493","130","7.626","128","8392704" +"2020-03-01 19:50:38.493","130","7.623","128","8392704" +"2020-03-01 19:50:39.493","130","7.619","128","8392704" +"2020-03-01 19:50:40.493","130","7.616","128","8392704" +"2020-03-01 19:50:41.493","130","7.616","128","8392704" +"2020-03-01 19:50:42.493","130","7.615","128","8392704" +"2020-03-01 19:50:43.493","130","7.613","128","8392704" +"2020-03-01 19:50:44.493","130","7.614","128","8392704" +"2020-03-01 19:50:45.493","130","7.614","128","8392704" +"2020-03-01 19:50:46.493","130","7.612","128","8392704" +"2020-03-01 19:50:47.493","130","7.611","128","8392704" +"2020-03-01 19:50:48.493","130","7.611","128","8392704" +"2020-03-01 19:50:49.493","130","7.611","128","8392704" +"2020-03-01 19:50:50.493","130","7.612","128","8392704" +"2020-03-01 19:50:51.493","130","7.613","128","8392704" +"2020-03-01 19:50:52.493","130","7.613","128","8392704" +"2020-03-01 19:50:53.493","130","7.615","128","8392704" +"2020-03-01 19:50:54.493","130","7.617","128","8392704" +"2020-03-01 19:50:55.493","130","7.617","128","8392704" +"2020-03-01 19:50:56.493","130","7.619","128","8392704" +"2020-03-01 19:50:57.493","130","7.622","128","8392704" +"2020-03-01 19:50:58.493","130","7.624","128","8392704" +"2020-03-01 19:50:59.493","130","7.625","128","8392704" +"2020-03-01 19:51:00.493","130","7.624","128","8392704" +"2020-03-01 19:51:01.493","130","7.624","128","8392704" +"2020-03-01 19:51:02.493","130","7.622","128","8392704" +"2020-03-01 19:51:03.493","130","7.62","128","8392704" +"2020-03-01 19:51:04.493","130","7.617","128","8392704" +"2020-03-01 19:51:05.493","130","7.617","128","8392704" +"2020-03-01 19:51:06.493","130","7.618","128","8392704" +"2020-03-01 19:51:07.493","130","7.618","128","8392704" +"2020-03-01 19:51:08.493","130","7.618","128","8392704" +"2020-03-01 19:51:09.493","130","7.62","128","8392704" +"2020-03-01 19:51:10.493","130","7.622","128","8392704" +"2020-03-01 19:51:11.493","130","7.623","128","8392704" +"2020-03-01 19:51:12.493","130","7.624","128","8392704" +"2020-03-01 19:51:13.493","130","7.625","128","8392704" +"2020-03-01 19:51:14.493","130","7.626","128","8392704" +"2020-03-01 19:51:15.493","130","7.626","128","8392704" +"2020-03-01 19:51:16.493","130","7.626","128","8392704" +"2020-03-01 19:51:17.493","130","7.627","128","8392704" +"2020-03-01 19:51:18.493","130","7.627","128","8392704" +"2020-03-01 19:51:19.493","130","7.629","128","8392704" +"2020-03-01 19:51:20.493","130","7.629","128","8392704" +"2020-03-01 19:51:21.493","130","7.626","128","8392704" +"2020-03-01 19:51:22.493","130","7.625","128","8392704" +"2020-03-01 19:51:23.493","130","7.625","128","8392704" +"2020-03-01 19:51:24.493","130","7.626","128","8392704" +"2020-03-01 19:51:25.493","130","7.626","128","8392704" +"2020-03-01 19:51:26.493","130","7.624","128","8392704" +"2020-03-01 19:51:27.493","130","7.623","128","8392704" +"2020-03-01 19:51:28.493","130","7.624","128","8392704" +"2020-03-01 19:51:29.493","130","7.624","128","8392704" +"2020-03-01 19:51:30.493","130","7.624","128","8392704" +"2020-03-01 19:51:31.493","130","7.624","128","8392704" +"2020-03-01 19:51:32.493","130","7.626","128","8392704" +"2020-03-01 19:51:33.493","130","7.626","128","8392704" +"2020-03-01 19:51:34.493","130","7.626","128","8392704" +"2020-03-01 19:51:35.493","130","7.625","128","8392704" +"2020-03-01 19:51:36.493","130","7.624","128","8392704" +"2020-03-01 19:51:37.493","130","7.623","128","8392704" +"2020-03-01 19:51:38.493","130","7.622","128","8392704" +"2020-03-01 19:51:39.493","130","7.62","128","8392704" +"2020-03-01 19:51:40.493","130","7.62","128","8392704" +"2020-03-01 19:51:41.493","130","7.62","128","8392704" +"2020-03-01 19:51:42.493","130","7.621","128","8392704" +"2020-03-01 19:51:43.493","130","7.62","128","8392704" +"2020-03-01 19:51:44.493","130","7.619","128","8392704" +"2020-03-01 19:51:45.493","130","7.62","128","8392704" +"2020-03-01 19:51:46.493","130","7.62","128","8392704" +"2020-03-01 19:51:47.493","130","7.618","128","8392704" +"2020-03-01 19:51:48.493","130","7.619","128","8392704" +"2020-03-01 19:51:49.493","130","7.62","128","8392704" +"2020-03-01 19:51:50.493","130","7.622","128","8392704" +"2020-03-01 19:51:51.493","130","7.622","128","8392704" +"2020-03-01 19:51:52.493","130","7.62","128","8392704" +"2020-03-01 19:51:53.493","130","7.62","128","8392704" +"2020-03-01 19:51:54.493","130","7.622","128","8392704" +"2020-03-01 19:51:55.493","130","7.624","128","8392704" +"2020-03-01 19:51:56.493","130","7.622","128","8392704" +"2020-03-01 19:51:57.493","130","7.616","128","8392704" +"2020-03-01 19:51:58.493","130","7.611","128","8392704" +"2020-03-01 19:51:59.493","130","7.61","128","8392704" +"2020-03-01 19:52:00.493","130","7.608","128","8392704" +"2020-03-01 19:52:01.493","130","7.606","128","8392704" +"2020-03-01 19:52:02.493","130","7.607","128","8392704" +"2020-03-01 19:52:03.493","130","7.608","128","8392704" +"2020-03-01 19:52:04.493","130","7.61","128","8392704" +"2020-03-01 19:52:05.493","130","7.612","128","8392704" +"2020-03-01 19:52:06.493","130","7.615","128","8392704" +"2020-03-01 19:52:07.493","130","7.62","128","8392704" +"2020-03-01 19:52:08.493","130","7.623","128","8392704" +"2020-03-01 19:52:09.493","130","7.624","128","8392704" +"2020-03-01 19:52:10.493","130","7.623","128","8392704" +"2020-03-01 19:52:11.493","130","7.623","128","8392704" +"2020-03-01 19:52:12.493","130","7.624","128","8392704" +"2020-03-01 19:52:13.493","130","7.622","128","8392704" +"2020-03-01 19:52:14.493","130","7.62","128","8392704" +"2020-03-01 19:52:15.493","130","7.621","128","8392704" +"2020-03-01 19:52:16.493","130","7.62","128","8392704" +"2020-03-01 19:52:17.493","130","7.622","128","8392704" +"2020-03-01 19:52:18.493","130","7.625","128","8392704" +"2020-03-01 19:52:19.493","130","7.627","128","8392704" +"2020-03-01 19:52:20.493","130","7.625","128","8392704" +"2020-03-01 19:52:21.493","130","7.621","128","8392704" +"2020-03-01 19:52:22.493","130","7.617","128","8392704" +"2020-03-01 19:52:23.493","130","7.617","128","8392704" +"2020-03-01 19:52:24.493","130","7.617","128","8392704" +"2020-03-01 19:52:25.493","130","7.616","128","8392704" +"2020-03-01 19:52:26.493","130","7.615","128","8392704" +"2020-03-01 19:52:27.493","130","7.616","128","8392704" +"2020-03-01 19:52:28.493","130","7.619","128","8392704" +"2020-03-01 19:52:29.493","130","7.621","128","8392704" +"2020-03-01 19:52:30.493","130","7.621","128","8392704" +"2020-03-01 19:52:31.493","130","7.621","128","8392704" +"2020-03-01 19:52:32.493","130","7.621","128","8392704" +"2020-03-01 19:52:33.493","130","7.621","128","8392704" +"2020-03-01 19:52:34.493","130","7.623","128","8392704" +"2020-03-01 19:52:35.493","130","7.621","128","8392704" +"2020-03-01 19:52:36.493","130","7.617","128","8392704" +"2020-03-01 19:52:37.493","130","7.615","128","8392704" +"2020-03-01 19:52:38.493","130","7.612","128","8392704" +"2020-03-01 19:52:39.493","130","7.609","128","8392704" +"2020-03-01 19:52:40.493","130","7.606","128","8392704" +"2020-03-01 19:52:41.493","130","7.606","128","8392704" +"2020-03-01 19:52:42.493","130","7.609","128","8392704" +"2020-03-01 19:52:43.493","130","7.612","128","8392704" +"2020-03-01 19:52:44.493","130","7.616","128","8392704" +"2020-03-01 19:52:45.493","130","7.619","128","8392704" +"2020-03-01 19:52:46.493","130","7.62","128","8392704" +"2020-03-01 19:52:47.493","130","7.622","128","8392704" +"2020-03-01 19:52:48.493","130","7.622","128","8392704" +"2020-03-01 19:52:49.493","130","7.621","128","8392704" +"2020-03-01 19:52:50.493","130","7.618","128","8392704" +"2020-03-01 19:52:51.493","130","7.616","128","8392704" +"2020-03-01 19:52:52.493","130","7.613","128","8392704" +"2020-03-01 19:52:53.493","130","7.612","128","8392704" +"2020-03-01 19:52:54.493","130","7.612","128","8392704" +"2020-03-01 19:52:55.493","130","7.611","128","8392704" +"2020-03-01 19:52:56.493","130","7.609","128","8392704" +"2020-03-01 19:52:57.493","130","7.608","128","8392704" +"2020-03-01 19:52:58.493","130","7.609","128","8392704" +"2020-03-01 19:52:59.493","130","7.611","128","8392704" +"2020-03-01 19:53:00.493","130","7.612","128","8392704" +"2020-03-01 19:53:01.493","130","7.614","128","8392704" +"2020-03-01 19:53:02.493","130","7.618","128","8392704" +"2020-03-01 19:53:03.493","130","7.62","128","8392704" +"2020-03-01 19:53:04.493","130","7.62","128","8392704" +"2020-03-01 19:53:05.493","130","7.62","128","8392704" +"2020-03-01 19:53:06.493","130","7.619","128","8392704" +"2020-03-01 19:53:07.493","130","7.617","128","8392704" +"2020-03-01 19:53:08.493","130","7.615","128","8392704" +"2020-03-01 19:53:09.493","130","7.612","128","8392704" +"2020-03-01 19:53:10.493","130","7.61","128","8392704" +"2020-03-01 19:53:11.493","130","7.609","128","8392704" +"2020-03-01 19:53:12.493","130","7.608","128","8392704" +"2020-03-01 19:53:13.493","130","7.61","128","8392704" +"2020-03-01 19:53:14.493","130","7.611","128","8392704" +"2020-03-01 19:53:15.493","130","7.609","128","8392704" +"2020-03-01 19:53:16.493","130","7.608","128","8392704" +"2020-03-01 19:53:17.493","130","7.608","128","8392704" +"2020-03-01 19:53:18.493","130","7.607","128","8392704" +"2020-03-01 19:53:19.493","130","7.607","128","8392704" +"2020-03-01 19:53:20.493","130","7.605","128","8392704" +"2020-03-01 19:53:21.493","130","7.603","128","8392704" +"2020-03-01 19:53:22.493","130","7.606","128","8392704" +"2020-03-01 19:53:23.493","130","7.611","128","8392704" +"2020-03-01 19:53:24.493","130","7.615","128","8392704" +"2020-03-01 19:53:25.493","130","7.618","128","8392704" +"2020-03-01 19:53:26.493","130","7.62","128","8392704" +"2020-03-01 19:53:27.493","130","7.622","128","8392704" +"2020-03-01 19:53:28.493","130","7.624","128","8392704" +"2020-03-01 19:53:29.493","130","7.626","128","8392704" +"2020-03-01 19:53:30.493","130","7.624","128","8392704" +"2020-03-01 19:53:31.493","130","7.617","128","8392704" +"2020-03-01 19:53:32.493","130","7.613","128","8392704" +"2020-03-01 19:53:33.493","130","7.613","128","8392704" +"2020-03-01 19:53:34.493","130","7.613","128","8392704" +"2020-03-01 19:53:35.493","130","7.61","128","8392704" +"2020-03-01 19:53:36.493","130","7.609","128","8392704" +"2020-03-01 19:53:37.493","130","7.611","128","8392704" +"2020-03-01 19:53:38.493","130","7.61","128","8392704" +"2020-03-01 19:53:39.493","130","7.609","128","8392704" +"2020-03-01 19:53:40.493","130","7.608","128","8392704" +"2020-03-01 19:53:41.493","130","7.605","128","8392704" +"2020-03-01 19:53:42.493","130","7.601","128","8392704" +"2020-03-01 19:53:43.493","130","7.6","128","8392704" +"2020-03-01 19:53:44.493","130","7.602","128","8392704" +"2020-03-01 19:53:45.493","130","7.604","128","8392704" +"2020-03-01 19:53:46.493","130","7.605","128","8392704" +"2020-03-01 19:53:47.493","130","7.606","128","8392704" +"2020-03-01 19:53:48.493","130","7.605","128","8392704" +"2020-03-01 19:53:49.493","130","7.605","128","8392704" +"2020-03-01 19:53:50.493","130","7.606","128","8392704" +"2020-03-01 19:53:51.493","130","7.606","128","8392704" +"2020-03-01 19:53:52.493","130","7.604","128","8392704" +"2020-03-01 19:53:53.493","130","7.606","128","8392704" +"2020-03-01 19:53:54.493","130","7.61","128","8392704" +"2020-03-01 19:53:55.493","130","7.612","128","8392704" +"2020-03-01 19:53:56.493","130","7.613","128","8392704" +"2020-03-01 19:53:57.493","130","7.613","128","8392704" +"2020-03-01 19:53:58.493","130","7.613","128","8392704" +"2020-03-01 19:53:59.493","130","7.615","128","8392704" +"2020-03-01 19:54:00.493","130","7.616","128","8392704" +"2020-03-01 19:54:01.493","130","7.616","128","8392704" +"2020-03-01 19:54:02.493","130","7.618","128","8392704" +"2020-03-01 19:54:03.493","130","7.621","128","8392704" +"2020-03-01 19:54:04.493","130","7.623","128","8392704" +"2020-03-01 19:54:05.493","130","7.623","128","8392704" +"2020-03-01 19:54:06.493","130","7.624","128","8392704" +"2020-03-01 19:54:07.493","130","7.624","128","8392704" +"2020-03-01 19:54:08.493","130","7.622","128","8392704" +"2020-03-01 19:54:09.493","130","7.62","128","8392704" +"2020-03-01 19:54:10.493","130","7.621","128","8392704" +"2020-03-01 19:54:11.493","130","7.62","128","8392704" +"2020-03-01 19:54:12.493","130","7.617","128","8392704" +"2020-03-01 19:54:13.493","130","7.617","128","8392704" +"2020-03-01 19:54:14.493","130","7.618","128","8392704" +"2020-03-01 19:54:15.493","130","7.617","128","8392704" +"2020-03-01 19:54:16.493","130","7.615","128","8392704" +"2020-03-01 19:54:17.493","130","7.613","128","8392704" +"2020-03-01 19:54:18.493","130","7.612","128","8392704" +"2020-03-01 19:54:19.493","130","7.609","128","8392704" +"2020-03-01 19:54:20.493","130","7.605","128","8392704" +"2020-03-01 19:54:21.493","130","7.604","128","8392704" +"2020-03-01 19:54:22.493","130","7.605","128","8392704" +"2020-03-01 19:54:23.493","130","7.608","128","8392704" +"2020-03-01 19:54:24.493","130","7.611","128","8392704" +"2020-03-01 19:54:25.493","130","7.613","128","8392704" +"2020-03-01 19:54:26.493","130","7.613","128","8392704" +"2020-03-01 19:54:27.493","130","7.611","128","8392704" +"2020-03-01 19:54:28.493","130","7.611","128","8392704" +"2020-03-01 19:54:29.493","130","7.611","128","8392704" +"2020-03-01 19:54:30.493","130","7.612","128","8392704" +"2020-03-01 19:54:31.493","130","7.614","128","8392704" +"2020-03-01 19:54:32.493","130","7.61","128","8392704" +"2020-03-01 19:54:33.493","130","7.603","128","8392704" +"2020-03-01 19:54:34.493","130","7.598","128","8392704" +"2020-03-01 19:54:35.493","130","7.594","128","8392704" +"2020-03-01 19:54:36.493","130","7.591","128","8392704" +"2020-03-01 19:54:37.493","130","7.591","128","8392704" +"2020-03-01 19:54:38.493","130","7.59","128","8392704" +"2020-03-01 19:54:39.493","130","7.588","128","8392704" +"2020-03-01 19:54:40.493","130","7.593","128","8392704" +"2020-03-01 19:54:41.493","130","7.599","128","8392704" +"2020-03-01 19:54:42.493","130","7.602","128","8392704" +"2020-03-01 19:54:43.493","130","7.604","128","8392704" +"2020-03-01 19:54:44.493","130","7.606","128","8392704" +"2020-03-01 19:54:45.493","130","7.609","128","8392704" +"2020-03-01 19:54:46.493","130","7.612","128","8392704" +"2020-03-01 19:54:47.493","130","7.614","128","8392704" +"2020-03-01 19:54:48.493","130","7.616","128","8392704" +"2020-03-01 19:54:49.493","130","7.617","128","8392704" +"2020-03-01 19:54:50.493","130","7.619","128","8392704" +"2020-03-01 19:54:51.493","130","7.623","128","8392704" +"2020-03-01 19:54:52.493","130","7.626","128","8392704" +"2020-03-01 19:54:53.493","130","7.626","128","8392704" +"2020-03-01 19:54:54.493","130","7.624","128","8392704" +"2020-03-01 19:54:55.493","130","7.623","128","8392704" +"2020-03-01 19:54:56.493","130","7.618","128","8392704" +"2020-03-01 19:54:57.493","130","7.613","128","8392704" +"2020-03-01 19:54:58.493","130","7.61","128","8392704" +"2020-03-01 19:54:59.493","130","7.605","128","8392704" +"2020-03-01 19:55:00.493","130","7.604","128","8392704" +"2020-03-01 19:55:01.493","130","7.603","128","8392704" +"2020-03-01 19:55:02.493","130","7.602","128","8392704" +"2020-03-01 19:55:03.493","130","7.602","128","8392704" +"2020-03-01 19:55:04.493","130","7.602","128","8392704" +"2020-03-01 19:55:05.493","130","7.603","128","8392704" +"2020-03-01 19:55:06.493","130","7.608","128","8392704" +"2020-03-01 19:55:07.493","130","7.609","128","8392704" +"2020-03-01 19:55:08.493","130","7.608","128","8392704" +"2020-03-01 19:55:09.493","130","7.609","128","8392704" \ No newline at end of file diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 1ac492bb3a..757399b4a2 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -15,6 +15,7 @@ import sys import os import os.path import subprocess +from time import sleep from util.log import * @@ -210,6 +211,7 @@ class TDDnode: (self.index, self.cfgPath)) def getBuildPath(self): + buildPath = "" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -256,6 +258,35 @@ class TDDnode: tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) time.sleep(5) + + def startWithoutSleep(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + if self.valgrind == 0: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + else: + valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + + cmd = "nohup %s %s -c %s 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) + + print(cmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) def stop(self): if self.valgrind == 0: @@ -425,6 +456,10 @@ class TDDnodes: def start(self, index): self.check(index) self.dnodes[index - 1].start() + + def startWithoutSleep(self, index): + self.check(index) + self.dnodes[index - 1].startWithoutSleep() def stop(self, index): self.check(index) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9abec354c6..b2ed6212fd 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -25,7 +25,7 @@ class TDSql: self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=True): + def init(self, cursor, log=False): self.cursor = cursor if (log): diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim new file mode 100644 index 0000000000..98ac4ec012 --- /dev/null +++ b/tests/script/general/db/nosuchfile.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step3 +sql create database d1 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +print ========== step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sleep 3000 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +print ========== step6 +system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != null then + return -1 +endi +if $data11 != null then + return -1 +endi +if $data21 != null then + return -1 +endi +if $data31 != null then + return -1 +endi +if $data41 != null then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 5968a9cd5e..102d5ba93f 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -132,4 +132,187 @@ sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbna sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1 +#1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 | +#1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 | +sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19; +if $rows != 20 then + return -1 +endi + +if $data00 != @70-01-01 08:01:40.800@ then + return -1 +endi + +if $data01 != 10 then + return -1 +endi + +if $data02 != 45.000000000 then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +if $data05 != 0 then + return -1 +endi + +if $data06 != 0 then + return -1 +endi + +if $data10 != @70-01-01 08:01:40.790@ then + return -1 +endi + +if $data11 != 10 then + return -1 +endi + +if $data12 != 945.000000000 then + return -1 +endi + +if $data13 != 90 then + return -1 +endi + +if $data14 != 1 then + return -1 +endi + +if $data15 != 1 then + return -1 +endi + +if $data16 != 0 then + return -1 +endi + +sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; +if $rows != 100 then + return -1 +endi + +if $data00 != @70-01-01 08:01:40.990@ then + print expect 0, actual: $data00 + return -1 +endi + +if $data01 != 30 then + return -1 +endi + +if $data02 != 94.500000000 then + print expect 94.500000000, actual $data02 + return -1 +endi + +if $data03 != 94.500000000 then + return -1 +endi + +if $data04 != 90 then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data06 != 2 then + return -1 +endi + +if $data10 != @70-01-01 08:01:40.980@ then + print expect 70-01-01 08:01:40.980, actual: $data10 + return -1 +endi + +if $data11 != 30 then + return -1 +endi + +if $data12 != 84.500000000 then + print expect 84.500000000, actual $data12 + return -1 +endi + +if $data13 != 84.500000000 then + return -1 +endi + +if $data14 != 80 then + return -1 +endi + +if $data15 != 1 then + return -1 +endi + +if $data16 != 2 then + return -1 +endi + +# this function will cause shell crash +sql select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; +if $rows != 100 then + return -1 +endi + +if $data00 != @70-01-01 08:01:40.990@ then + return -1 +endi + +if $data01 != 10 then + return -1 +endi + +if $data02 != 90 then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +if $data11 != 10 then + return -1 +endi + +if $data12 != 80 then + return -1 +endi + +if $data13 != 0 then + return -1 +endi + +sql select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc; +if $rows != 1 then + return -1 +endi + +if $data00 != @70-01-01 08:00:00.000@ then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 0 then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index 127ade66c5..7c83ca0c2f 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -324,8 +324,22 @@ sql create table tm0 using m1 tags(1); sql create table tm1 using m1 tags(2); sql insert into tm0 values(10000, 1) (20000, 2)(30000, 3) (40000, NULL) (50000, 2) tm1 values(10001, 2)(20000,4)(90000,9); -sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc; +#=============================tbase-1205 +sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); + +print ===================>TD-1834 +sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc +if $rows != 0 then + return -1 +endi + +sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc +if $rows != 0 then + return -1 +endi + +sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc; if $row != 5 then return -1 endi @@ -386,7 +400,25 @@ sql_error select k+1,sum(k) from tm0; sql_error select k, sum(k) from tm0; sql_error select k, sum(k)+1 from tm0; +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 5000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed + #=============================tbase-1205 sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); +print ===================>TD-1834 +sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc +if $rows != 0 then + return -1 +endi + +sql select * from tm0 where ts>11000 and ts< 20000 order by ts desc +if $rows != 0 then + return -1 +endi + + diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e68d1566a..1b2fe37c71 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -64,6 +64,7 @@ cd ../../../debug; make ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim ./test.sh -f general/db/vnodes.sim +./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim @@ -277,6 +278,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim ./test.sh -f unique/dnode/reason.sim diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim new file mode 100644 index 0000000000..61a991148b --- /dev/null +++ b/tests/script/unique/dnode/data1.sim @@ -0,0 +1,137 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step2 +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show2: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show2 +endi +if $data2_2 != 0 then + goto show2 +endi +if $data2_3 != 0 then + goto show2 +endi +if $data2_4 != 0 then + goto show2 +endi + +print ========== step3 +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +$x = 0 +show3: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show3 +endi +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 1 then + goto show3 +endi + +print ========== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ========== step5 +system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data + +print ========== step6 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sleep 10000 + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file