diff --git a/.gitignore b/.gitignore index d0af51d31b..83df761654 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,8 @@ Target/ *.failed *.sql sim/ +psim/ +pysim/ +*.out *DS_Store diff --git a/.travis.yml b/.travis.yml index a86f2463e9..9bc576dcf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ # matrix: - os: linux + dist: bionic language: c git: @@ -49,7 +50,7 @@ matrix: ./test-all.sh $TRAVIS_EVENT_TYPE || travis_terminate $? cd ${TRAVIS_BUILD_DIR}/tests/pytest - ./valgrind-test.sh -g 2>&1 | tee mem-error-out.txt + ./valgrind-test.sh 2>&1 > mem-error-out.txt sleep 1 # Color setting @@ -59,12 +60,12 @@ matrix: GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - grep 'ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt + grep 'start to execute\|ERROR SUMMARY' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-mem-error-out.txt - for memError in `cat uniq-mem-error-out.txt | awk '{print $4}'` + for memError in `grep 'ERROR SUMMARY' uniq-mem-error-out.txt | awk '{print $4}'` do if [ -n "$memError" ]; then - if [ "$memError" -gt 5 ]; then + if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ More than our threshold! ## ${NC}" travis_terminate $memError @@ -72,11 +73,11 @@ matrix: fi done - grep 'definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt - for defiMemError in `cat uniq-definitely-lost-out.txt | awk '{print $7}'` + grep 'start to execute\|definitely lost:' mem-error-out.txt|grep -v 'grep'|uniq|tee uniq-definitely-lost-out.txt + for defiMemError in `grep 'definitely lost:' uniq-definitely-lost-out.txt | awk '{print $7}'` do if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 3 ]; then + if [ "$defiMemError" -gt 13 ]; then echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $defiMemError @@ -203,6 +204,29 @@ matrix: ;; esac + - os: linux + dist: trusty + language: c + git: + - depth: 1 + + addons: + apt: + packages: + - build-essential + - cmake + env: + - DESC="trusty/gcc-4.8 build" + + before_script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir debug + - cd debug + + script: + - cmake .. > /dev/null + - make > /dev/null + - os: linux language: c compiler: clang diff --git a/README.md b/README.md index 0babfe7776..d75680cb22 100644 --- a/README.md +++ b/README.md @@ -115,251 +115,6 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo - [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector) - [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector) -# How to run the test cases and how to add a new test case? - -### Prepare development environment - -1. sudo apt install - build-essential cmake net-tools python-pip python-setuptools python3-pip - python3-setuptools valgrind psmisc curl - -2. git clone ; cd TDengine - -3. mkdir debug; cd debug; cmake ..; make ; sudo make install - -4. pip install src/connector/python/linux/python2 ; pip3 install - src/connector/python/linux/python3 - -### How to run TSIM test suite - -1. cd \/tests/script - -2. sudo ./test.sh - -### How to run Python test suite - -1. cd \/tests/pytest - -2. ./smoketest.sh \# for smoke test - -3. ./smoketest.sh -g \# for memory leak detection test with valgrind - -4. ./fulltest.sh \# for full test - -> Note1: TDengine daemon's configuration and data files are stored in -> \/sim directory. As a historical design, it's same place with -> TSIM script. So after the TSIM script ran with sudo privilege, the directory -> has been used by TSIM then the python script cannot write it by a normal -> user. You need to remove the directory completely first before running the -> Python test case. We should consider using two different locations to store -> for TSIM and Python script. - -> Note2: if you need to debug crash problem with a core dump, you need -> manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited" -> before the script line. Then you can look for the core file in -> \/tests/pytest after the program crash. - -### How to add a new test case - -**1. add a new TSIM test cases:** - -TSIM test cases are now included in the new development branch and can be -added to the TDengine/tests/script/test.sh script based on the manual test -methods necessary to add test cases as described above. - -**2. add a new Python test cases:** - -**2.1 Please refer to \/tests/pytest/insert/basic.py to add a new -test case.** The new test case must implement 3 functions, where self.init() -and self.stop() simply copy the contents of insert/basic.py and the test -logic is implemented in self.run(). You can refer to the code in the util -directory for more information. - -**2.2 Edit smoketest.sh to add the path and filename of the new test case** - -Note: The Python test framework may continue to be improved in the future, -hopefully, to provide more functionality and ease of writing test cases. The -method of writing the test case above does not exclude that it will also be -affected. - -**2.3 What test.py does in detail:** - -test.py is the entry program for test case execution and monitoring. - -test.py has the following functions. - -\-f --file, Specifies the test case file name to be executed --p --path, Specifies deployment path - -\-m --master, Specifies the master server IP for cluster deployment --c--cluster, test cluster function --s--stop, terminates all running nodes - -\-g--valgrind, load valgrind for memory leak detection test - -\-h--help, display help - -**2.4 What util/log.py does in detail:** - -log.py is quite simple, the main thing is that you can print the output in -different colors as needed. The success() should be called for successful -test case execution and the success() will print green text. The exit() will -print red text and exit the program, exit() should be called for test -failure. - -**util/log.py** - -... - -    def info(self, info): - -        printf("%s %s" % (datetime.datetime.now(), info)) - -  - -    def sleep(self, sec): - -        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) - -        time.sleep(sec) - -  - -    def debug(self, err): - -        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err)) - -  - -    def success(self, info): - -        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info)) - -  - -    def notice(self, err): - -        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err)) - -  - -    def exit(self, err): - -        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err)) - -        sys.exit(1) - -  - -    def printNoPrefix(self, info): - -        printf("\\033[1;36m%s\\033[0m" % (info) - -... - -**2.5 What util/sql.py does in detail:** - -SQL.py is mainly used to execute SQL statements to manipulate the database, -and the code is extracted and commented as follows: - -**util/sql.py** - -\# prepare() is mainly used to set up the environment for testing table and -data, and to set up the database db for testing. do not call prepare() if you -need to test the database operation command. - -def prepare(self): - -tdLog.info("prepare database:db") - -self.cursor.execute('reset query cache') - -self.cursor.execute('drop database if exists db') - -self.cursor.execute('create database db') - -self.cursor.execute('use db') - -... - -\# query() is mainly used to execute select statements for normal syntax input - -def query(self, sql): - -... - -\# error() is mainly used to execute the select statement with the wrong syntax -input, the error will be caught as a reasonable behavior, if not caught it will -prove that the test failed - -def error() - -... - -\# checkRows() is used to check the number of returned lines after calling -query(select ...) after calling the query(select ...) to check the number of -rows of returned results. - -def checkRows(self, expectRows): - -... - -\# checkData() is used to check the returned result data after calling -query(select ...) after the query(select ...) is called, failure to meet -expectation is - -def checkData(self, row, col, data): - -... - -\# getData() returns the result data after calling query(select ...) to return -the resulting data after calling query(select ...) - -def getData(self, row, col): - -... - -\# execute() used to execute sql and return the number of affected rows - -def execute(self, sql): - -... - -\# executeTimes() Multiple executions of the same sql statement - -def executeTimes(self, sql, times): - -... - -\# CheckAffectedRows() Check if the number of affected rows is as expected - -def checkAffectedRows(self, expectAffectedRows): - -... - -> Note: Both Python2 and Python3 are currently supported by the Python test -> case. Since Python2 is no longer officially supported by January 1, 2020, it -> is recommended that subsequent test case development be guaranteed to run -> correctly on Python3. For Python2, please consider being compatible if -> appropriate without additional -> burden.   - -### CI Covenant submission adoption principle. - -- Every commit / PR compilation must pass. Currently, the warning is treated - as an error, so the warning must also be resolved. - -- Test cases that already exist must pass. - -- Because CI is very important to support build and automatically test - procedure, it is necessary to manually test the test case before adding it - and do as many iterations as possible to ensure that the test case provides - stable and reliable test results when added. - -> Note: In the future, according to the requirements and test development -> progress will add stress testing, performance testing, code style, -> and other features based on functional testing. - ### Third Party Connectors The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them. @@ -367,6 +122,10 @@ The TDengine community has also kindly built some of their own connectors! Follo - [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust) - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos) +# How to run the test cases and how to add a new test case? + TDengine's test framework and all test cases are fully open source. + Please refer to [this document](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md) for how to run test and develop new test case. + # TDengine Roadmap - Support event-driven stream computing - Support user defined functions diff --git a/src/client/inc/tscSecondaryMerge.h b/src/client/inc/tscSecondaryMerge.h index f6c7516754..ad743eeea3 100644 --- a/src/client/inc/tscSecondaryMerge.h +++ b/src/client/inc/tscSecondaryMerge.h @@ -116,7 +116,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF * create local reducer to launch the second-stage reduce process at client site */ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SColumnModel *finalModel, SSqlCmd *pSqlCmd, SSqlRes *pRes); + SColumnModel *finalModel, SSqlObj* pSql); void tscDestroyLocalReducer(SSqlObj *pSql); diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 71c24501d1..715d76e072 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -31,19 +31,19 @@ extern "C" { #include "tscSecondaryMerge.h" #include "tsclient.h" -#define UTIL_TABLE_IS_SUPERTABLE(metaInfo) \ +#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE)) -#define UTIL_TABLE_IS_NOMRAL_TABLE(metaInfo)\ - (!(UTIL_TABLE_IS_SUPERTABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) +#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ + (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) typedef struct SParsedColElem { int16_t colIndex; - int16_t offset; + uint16_t offset; } SParsedColElem; typedef struct SParsedDataColInfo { @@ -183,7 +183,7 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); -void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); +SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); @@ -264,6 +264,11 @@ bool hasMoreVnodesToTry(SSqlObj *pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()); +int tscSetMgmtIpListFromCfg(const char *first, const char *second); + +void* malloc_throw(size_t size); +void* calloc_throw(size_t nmemb, size_t size); +char* strdup_throw(const char* str); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6ea1ee6440..e6a37a2745 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -49,7 +49,7 @@ typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; int16_t numOfColumns; - int16_t rowSize; + int32_t rowSize; } STableComInfo; typedef struct STableMeta { @@ -438,6 +438,9 @@ extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows); +int32_t tscCompareTidTags(const void* p1, const void* p2); +void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables); + #ifdef __cplusplus } #endif diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 9f207936df..82b0f852ab 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -145,7 +145,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { return; } - // local reducer has handle this situation during super table non-projection query. + // local merge has handle this situation during super table non-projection query. if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE) { pRes->numOfTotalInCurrentClause += pRes->numOfRows; } @@ -222,9 +222,30 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi tscResetForNextRetrieve(pRes); // handle the sub queries of join query - if (pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscFetchDatablockFromSubquery(pSql); - } else { + } else if (pRes->completed && pCmd->command == TSDB_SQL_FETCH) { + if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. + tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); + return; + } else { + /* + * all available virtual node has been checked already, now we need to check + * for the next subclause queries + */ + if (pCmd->clauseIndex < pCmd->numOfClause - 1) { + tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); + return; + } + + /* + * 1. has reach the limitation + * 2. no remain virtual nodes to be retrieved anymore + */ + (*pSql->fetchFp)(param, pSql, 0); + } + return; + } else { // current query is not completed, continue retrieve from node if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; } @@ -425,6 +446,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo->pTableMeta == NULL){ + code = tscGetTableMeta(pSql, pTableMetaInfo); + assert(code == TSDB_CODE_SUCCESS); + } + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0) && pTableMetaInfo->vgroupIndex >= 0 && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; @@ -433,12 +459,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0); - tscTrace("%p get metricMeta during super table query successfully", pSql); - - code = tscGetTableMeta(pSql, pTableMetaInfo); - pRes->code = code; - - if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; + tscTrace("%p get metricMeta during super table query successfully", pSql); code = tscGetSTableVgroupInfo(pSql, 0); pRes->code = code; @@ -471,7 +492,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex); pRes->code = code; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 6fb8df2444..c6e8499426 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2904,7 +2904,11 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { param[1][2] /= param[1][1]; - sprintf(pCtx->aOutputBuf, "(%lf, %lf)", param[0][2], param[1][2]); + int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; + size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", + param[0][2], param[1][2]); + + varDataSetLen(pCtx->aOutputBuf, n); doFinalizer(pCtx); } @@ -3616,299 +3620,6 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void getStatics_i8(int64_t *primaryKey, int32_t type, int8_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = INT64_MAX; - *max = INT64_MIN; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - // int64_t lastKey = 0; - // int8_t lastVal = TSDB_DATA_TINYINT_NULL; - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((char *)&data[i], type)) { - (*numOfNull) += 1; - continue; - } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; - } - - // if (type != TSDB_DATA_TYPE_BOOL) { // ignore the bool data type pre-calculation - // if (isNull((char *)&lastVal, type)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - // } - } -} - -static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = INT64_MAX; - *max = INT64_MIN; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - // int64_t lastKey = 0; - // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { - (*numOfNull) += 1; - continue; - } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; - } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - } -} - -static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = INT64_MAX; - *max = INT64_MIN; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - // int64_t lastKey = 0; - // int32_t lastVal = TSDB_DATA_INT_NULL; - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { - (*numOfNull) += 1; - continue; - } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; - } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - } -} - -static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - *min = INT64_MAX; - *max = INT64_MIN; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { - (*numOfNull) += 1; - continue; - } - - *sum += data[i]; - if (*min > data[i]) { - *min = data[i]; - *minIndex = i; - } - - if (*max < data[i]) { - *max = data[i]; - *maxIndex = i; - } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - } -} - -static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, double *min, double *max, double *sum, - int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - float fmin = DBL_MAX; - float fmax = -DBL_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { - (*numOfNull) += 1; - continue; - } - - float fv = 0; - fv = GET_FLOAT_VAL(&(data[i])); - dsum += fv; - if (fmin > fv) { - fmin = fv; - *minIndex = i; - } - - if (fmax < fv) { - fmax = fv; - *maxIndex = i; - } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - } - - double csum = 0; - csum = GET_DOUBLE_VAL(sum); - csum += dsum; -#ifdef _TD_ARM_32_ - SET_DOUBLE_VAL_ALIGN(sum, &csum); - SET_DOUBLE_VAL_ALIGN(max, &fmax); - SET_DOUBLE_VAL_ALIGN(min, &fmin); -#else - *sum = csum; - *max = fmax; - *min = fmin; -#endif -} - -static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, double *min, double *max, double *sum, - int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - double dmin = DBL_MAX; - double dmax = -DBL_MAX; - double dsum = 0; - *minIndex = 0; - *maxIndex = 0; - - assert(numOfRow <= INT16_MAX); - - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { - (*numOfNull) += 1; - continue; - } - - double dv = 0; - dv = GET_DOUBLE_VAL(&(data[i])); - dsum += dv; - if (dmin > dv) { - dmin = dv; - *minIndex = i; - } - - if (dmax < dv) { - dmax = dv; - *maxIndex = i; - } - - // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } else { - // *wsum = lastVal * (primaryKey[i] - lastKey); - // lastKey = primaryKey[i]; - // lastVal = data[i]; - // } - } - - double csum = 0; - csum = GET_DOUBLE_VAL(sum); - csum += dsum; - - -#ifdef _TD_ARM_32_ - SET_DOUBLE_VAL_ALIGN(sum, &csum); - SET_DOUBLE_VAL_ALIGN(max, &dmax); - SET_DOUBLE_VAL_ALIGN(min, &dmin); -#else - *sum = csum; - *max = dmax; - *min = dmin; -#endif -} - -void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, int32_t type, int64_t *min, int64_t *max, - int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int32_t *numOfNull) { - int64_t *primaryKey = (int64_t *)priData; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(data + i * size, type)) { - (*numOfNull) += 1; - continue; - } - } - } else { - if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { - getStatics_i8(primaryKey, type, (int8_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); - } else if (type == TSDB_DATA_TYPE_SMALLINT) { - getStatics_i16(primaryKey, (int16_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); - } else if (type == TSDB_DATA_TYPE_INT) { - getStatics_i32(primaryKey, (int32_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); - } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { - getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - getStatics_d(primaryKey, (double *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); - } else if (type == TSDB_DATA_TYPE_FLOAT) { - getStatics_f(primaryKey, (float *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); - } - } -} /** * param[1]: start time diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 5ae226a577..2da786d1d8 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -122,7 +122,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { int32_t numOfRows = tscGetNumOfColumns(pMeta); int32_t totalNumOfRows = numOfRows + tscGetNumOfTags(pMeta); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { numOfRows = numOfRows + tscGetNumOfTags(pMeta); } @@ -161,7 +161,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { } } - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return 0; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 65555dba13..7e67ff82e9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -796,7 +796,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } - if (!UTIL_TABLE_IS_SUPERTABLE(pSTableMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMeterMetaInfo)) { return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z); } @@ -1097,7 +1097,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; // TODO: should _clean or _error_clean to async flow ???? } - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscInvalidSQLErrMsg(pCmd->payload, "insert data into super table is not supported", NULL); goto _error_clean; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 260649050b..2caa4ad436 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -194,7 +194,7 @@ static int normalStmtPrepare(STscStmt* stmt) { static char* normalStmtBuildSql(STscStmt* stmt) { SNormalStmt* normal = &stmt->normal; - SStringBuilder sb = {0}; + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); if (taosStringBuilderSetJmp(&sb) != 0) { taosStringBuilderDestroy(&sb); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index a81ad19e4f..551b0a0457 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1297,10 +1297,6 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) { - -} - static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); @@ -1352,7 +1348,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum STableComInfo tinfo = tscGetTableInfo(pTableMeta); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { numOfTotalColumns = tinfo.numOfColumns + tinfo.numOfTags; } else { numOfTotalColumns = tinfo.numOfColumns; @@ -1412,7 +1408,7 @@ int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pI STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -1866,7 +1862,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr case TK_TBID: { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg7); } @@ -2283,7 +2279,7 @@ bool validateIpAddress(const char* ip, size_t size) { int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return TSDB_CODE_INVALID_SQL; } @@ -2322,7 +2318,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { /* transfer the field-info back to original input format */ void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return; } @@ -2546,7 +2542,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } if (groupTag) { - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg9); } @@ -3258,7 +3254,7 @@ static bool validateJoinExprNode(SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColum } // table to table/ super table to super table are allowed - if (UTIL_TABLE_IS_SUPERTABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPERTABLE(pRightMeterMeta)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pLeftMeterMeta) != UTIL_TABLE_IS_SUPER_TABLE(pRightMeterMeta)) { invalidSqlErrMsg(pQueryInfo->msg, msg5); return false; } @@ -3341,7 +3337,7 @@ static int32_t handleExprInQueryCond(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, S } else if (index.columnIndex >= tscGetNumOfColumns(pTableMeta) || index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { // query on tags // check for tag query condition - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -3562,7 +3558,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1 = {0}; + SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; @@ -3701,7 +3697,7 @@ static int32_t validateJoinExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { } STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // for stable join, tag columns + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // for stable join, tag columns // must be present for join if (pCondExpr->pJoinExpr == NULL) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -3739,7 +3735,7 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { SColumnIndex index = {0}; getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); @@ -3796,6 +3792,8 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExprDestroy(p1); tExprTreeDestroy(&p, NULL); + + taosArrayDestroy(colList); } pCondExpr->pTagCond = NULL; @@ -3815,7 +3813,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql pQueryInfo->window.ekey = INT64_MAX; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space - SStringBuilder sb = {0}; + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); SCondExpr condExpr = {0}; if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { @@ -4104,7 +4102,7 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { } /* for super table query, set default ascending order for group output */ - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; } } @@ -4130,7 +4128,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema * * for super table query, the order option must be less than 3. */ - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { if (pSortorder->nExpr > 1) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } @@ -4151,7 +4149,7 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // super table query + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query if (getColumnIndexByName(&columnName, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -4304,10 +4302,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo))) { + } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) && (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { return invalidSqlErrMsg(pQueryInfo->msg, msg4); } else if ((pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) && UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo)) { @@ -4693,7 +4691,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* } // todo refactor - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { @@ -5629,7 +5627,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } - bool isSTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); + bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } @@ -5773,7 +5771,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); bool isSTable = false; - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { isSTable = true; code = tscGetSTableVgroupInfo(pSql, index); if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 8c75ed6c35..a3a0441ff8 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -55,7 +55,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { } } -static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pReducer, tOrderDescriptor *pDesc) { +static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDescriptor *pDesc) { /* * the fields and offset attributes in pCmd and pModel may be different due to * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. @@ -96,13 +96,13 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; pCtx->param[2].i64Key = pQueryInfo->order.order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64Key = pQueryInfo->order.orderColId; } SResultInfo *pResInfo = &pReducer->pResInfo[i]; pResInfo->bufLen = pExpr->interBytes; - pResInfo->interResultBuf = calloc(1, (size_t)pResInfo->bufLen); + pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen); pCtx->resultInfo = &pReducer->pResInfo[i]; pCtx->resultInfo->superTableQ = true; @@ -132,16 +132,15 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu } } -/* - * todo release allocated memory process with async process - */ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, - SColumnModel *finalmodel, SSqlCmd *pCmd, SSqlRes *pRes) { - // offset of cmd in SSqlObj structure - char *pSqlObjAddr = (char *)pCmd - offsetof(SSqlObj, cmd); - + SColumnModel *finalmodel, SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + if (pMemBuffer == NULL) { - tscError("%p pMemBuffer", pMemBuffer); + tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); + + tscError("%p pMemBuffer is NULL", pMemBuffer); pRes->code = TSDB_CODE_APP_ERROR; return; } @@ -149,7 +148,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pDesc->pColumnModel == NULL) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); - tscError("%p no local buffer or intermediate result format model", pSqlObjAddr); + tscError("%p no local buffer or intermediate result format model", pSql); pRes->code = TSDB_CODE_APP_ERROR; return; } @@ -158,7 +157,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd for (int32_t i = 0; i < numOfBuffer; ++i) { int32_t len = pMemBuffer[i]->fileMeta.flushoutData.nLength; if (len == 0) { - tscTrace("%p no data retrieved from orderOfVnode:%d", pSqlObjAddr, i + 1); + tscTrace("%p no data retrieved from orderOfVnode:%d", pSql, i + 1); continue; } @@ -167,13 +166,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (numOfFlush == 0 || numOfBuffer == 0) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); - tscTrace("%p retrieved no data", pSqlObjAddr); + tscTrace("%p retrieved no data", pSql); return; } if (pDesc->pColumnModel->capacity >= pMemBuffer[0]->pageSize) { - tscError("%p Invalid value of buffer capacity %d and page size %d ", pSqlObjAddr, pDesc->pColumnModel->capacity, + tscError("%p Invalid value of buffer capacity %d and page size %d ", pSql, pDesc->pColumnModel->capacity, pMemBuffer[0]->pageSize); tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); @@ -181,10 +180,11 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd return; } - size_t nReducerSize = sizeof(SLocalReducer) + sizeof(void *) * numOfFlush; - SLocalReducer *pReducer = (SLocalReducer *)calloc(1, nReducerSize); + size_t size = sizeof(SLocalReducer) + POINTER_BYTES * numOfFlush; + + SLocalReducer *pReducer = (SLocalReducer *) calloc(1, size); if (pReducer == NULL) { - tscError("%p failed to create merge structure", pSqlObjAddr); + tscError("%p failed to create local merge structure, out of memory", pSql); tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, numOfBuffer); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -199,48 +199,52 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->numOfVnode = numOfBuffer; pReducer->pDesc = pDesc; - tscTrace("%p the number of merged leaves is: %d", pSqlObjAddr, pReducer->numOfBuffer); + tscTrace("%p the number of merged leaves is: %d", pSql, pReducer->numOfBuffer); int32_t idx = 0; for (int32_t i = 0; i < numOfBuffer; ++i) { int32_t numOfFlushoutInFile = pMemBuffer[i]->fileMeta.flushoutData.nLength; for (int32_t j = 0; j < numOfFlushoutInFile; ++j) { - SLocalDataSource *pDS = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); - if (pDS == NULL) { - tscError("%p failed to create merge structure", pSqlObjAddr); + SLocalDataSource *ds = (SLocalDataSource *)malloc(sizeof(SLocalDataSource) + pMemBuffer[0]->pageSize); + if (ds == NULL) { + tscError("%p failed to create merge structure", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return; } - pReducer->pLocalDataSrc[idx] = pDS; + + pReducer->pLocalDataSrc[idx] = ds; - pDS->pMemBuffer = pMemBuffer[i]; - pDS->flushoutIdx = j; - pDS->filePage.numOfElems = 0; - pDS->pageId = 0; - pDS->rowIdx = 0; + ds->pMemBuffer = pMemBuffer[i]; + ds->flushoutIdx = j; + ds->filePage.numOfElems = 0; + ds->pageId = 0; + ds->rowIdx = 0; - tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSqlObjAddr, i + 1, idx + 1); - tExtMemBufferLoadData(pMemBuffer[i], &(pDS->filePage), j, 0); + tscTrace("%p load data from disk into memory, orderOfVnode:%d, total:%d", pSql, i + 1, idx + 1); + tExtMemBufferLoadData(pMemBuffer[i], &(ds->filePage), j, 0); #ifdef _DEBUG_VIEW - printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", pDS->filePage.numOfElems); + printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", ds->filePage.numOfElems); SSrcColumnInfo colInfo[256] = {0}; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); tscGetSrcColumnInfo(colInfo, pQueryInfo); - tColModelDisplayEx(pDesc->pColumnModel, pDS->filePage.data, pDS->filePage.numOfElems, + tColModelDisplayEx(pDesc->pColumnModel, ds->filePage.data, ds->filePage.numOfElems, pMemBuffer[0]->numOfElemsPerPage, colInfo); #endif - if (pDS->filePage.numOfElems == 0) { // no data in this flush - tscTrace("%p flush data is empty, ignore %d flush record", pSqlObjAddr, idx); - tfree(pDS); + + if (ds->filePage.numOfElems == 0) { // no data in this flush, the index does not increase + tscTrace("%p flush data is empty, ignore %d flush record", pSql, idx); + tfree(ds); continue; } + idx += 1; } } - assert(idx >= pReducer->numOfBuffer); + + // no data actually, no need to merge result. if (idx == 0) { return; } @@ -262,9 +266,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // the input data format follows the old format, but output in a new format. // so, all the input must be parsed as old format - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx)); + pReducer->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; tscRestoreSQLFuncForSTableQuery(pQueryInfo); @@ -281,7 +283,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // used to keep the latest input row pReducer->pTempBuffer = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage)); - pReducer->discardData = (tFilePage *)calloc(1, pReducer->rowSize + sizeof(tFilePage)); pReducer->discard = false; @@ -309,11 +310,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd return; } + size = tscSqlExprNumOfExprs(pQueryInfo); + pReducer->pTempBuffer->numOfElems = 0; pReducer->pResInfo = calloc(size, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); - tscInitSqlContext(pCmd, pRes, pReducer, pDesc); + tscInitSqlContext(pCmd, pReducer, pDesc); // we change the capacity of schema to denote that there is only one row in temp buffer pReducer->pDesc->pColumnModel->capacity = 1; @@ -428,8 +431,7 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa tColModelAppend(pModel, pPage, data, numOfRows - remain, numOfWriteElems, numOfRows); if (pPage->numOfElems == pModel->capacity) { - int32_t ret = tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType); - if (ret != 0) { + if (tscFlushTmpBuffer(pMemoryBuf, pDesc, pPage, orderType) != TSDB_CODE_SUCCESS) { return -1; } } else { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 98cbe9dbde..67971dae68 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -27,6 +27,7 @@ #include "ttimer.h" #include "tutil.h" #include "tscLog.h" +#include "qsqltype.h" #define TSC_MGMT_VNODE 999 @@ -39,7 +40,7 @@ int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); void tscProcessActivityTimer(void *handle, void *tmrId); int tscKeepConn[TSDB_SQL_MAX] = {0}; -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid); +TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt); void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); void tscSaveSubscriptionProgress(void* sub); @@ -67,7 +68,7 @@ void tscPrintMgmtIp() { } } -void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) { +void tscSetMgmtIpList(SRpcIpSet *pIpList) { tscMgmtIpSet.numOfIps = pIpList->numOfIps; tscMgmtIpSet.inUse = pIpList->inUse; for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { @@ -75,16 +76,6 @@ void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) { } } -void tscSetMgmtIpListFromEdge() { - if (tscMgmtIpSet.numOfIps != 1) { - tscMgmtIpSet.numOfIps = 1; - tscMgmtIpSet.inUse = 0; - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); - tscTrace("edge mgmt IP list:"); - tscPrintMgmtIp(); - } -} - void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { tscMgmtIpSet = *pIpSet; tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); @@ -93,18 +84,6 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - /* - * The iplist returned by the cluster edition is the current management nodes - * and the iplist returned by the edge edition is empty - */ - if (pIpList->numOfIps != 0) { - tscSetMgmtIpListFromCluster(pIpList); - } else { - tscSetMgmtIpListFromEdge(); - } -} - /* * For each management node, try twice at least in case of poor network situation. * If the client start to connect to a non-management node from the client, and the first retry may fail due to @@ -132,7 +111,8 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; - tscSetMgmtIpList(pIpList); + if (pIpList->numOfIps > 0) + tscSetMgmtIpList(pIpList); if (pRsp->killConnection) { tscKillConnection(pObj); @@ -207,7 +187,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); } - tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]); + // tscTrace("%p msg:%s is sent to server", pSql, taosMsg[pSql->cmd.msgType]); SRpcMsg rpcMsg = { .msgType = pSql->cmd.msgType, @@ -235,7 +215,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; STscObj *pObj = pSql->pTscObj; - tscTrace("%p msg:%p is received from server", pSql, rpcMsg->pCont); + // tscTrace("%p msg:%s is received from server", pSql, taosMsg[rpcMsg->msgType]); if (pSql->freed || pObj->signature != pObj) { tscTrace("%p sql is already released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed, @@ -340,10 +320,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); pRes->numOfRows += pMsg->affectedRows; - tscTrace("%p cmd:%d code:%s, inserted rows:%d, rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), - pMsg->affectedRows, pRes->rspLen); + tscTrace("%p SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql, sqlCmd[pCmd->command], + tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); } else { - tscTrace("%p cmd:%d code:%s rsp len:%d", pSql, pCmd->command, tstrerror(pRes->code), pRes->rspLen); + tscTrace("%p SQL cmd:%s, code:%s rspLen:%d", pSql, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); } } @@ -426,7 +406,7 @@ int tscProcessSql(SSqlObj *pSql) { assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0); } - tscTrace("%p SQL cmd:%d will be processed, name:%s, type:%d", pSql, pCmd->command, name, type); + tscTrace("%p SQL cmd:%s will be processed, name:%s, type:%d", pSql, sqlCmd[pCmd->command], name, type); if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL if (pTableMetaInfo == NULL) { pSql->res.code = TSDB_CODE_OTHERS; @@ -500,7 +480,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // todo valid the vgroupId at the client side STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t vgIndex = pTableMetaInfo->vgroupIndex; SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; @@ -550,8 +530,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } /* - * for meter query, simply return the size <= 1k - * for metric query, estimate size according to meter tags + * for table query, simply return the size <= 1k */ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; @@ -562,25 +541,18 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); int32_t exprSize = sizeof(SSqlFuncMsg) * numOfExprs; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // meter query without tags values - if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize; - } - - int32_t size = 4096; - return size; + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); + TSKEY dfltKey = htobe64(pQueryMsg->window.skey); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { SCMVgroupInfo* pVgroupInfo = NULL; - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); @@ -589,25 +561,25 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - + tscSetDnodeIpList(pSql, pVgroupInfo); pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->sid); pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); - + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey)); + pQueryMsg->numOfTables = htonl(1); // set the number of tables - + pMsg += sizeof(STableIdInfo); } else { int32_t index = pTableMetaInfo->vgroupIndex; int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(index >= 0 && index < numOfVgroups); - + tscTrace("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups); - + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info @@ -624,7 +596,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pItem->tid); pTableIdInfo->uid = htobe64(pItem->uid); -// pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid)); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); pMsg += sizeof(STableIdInfo); } } @@ -1453,8 +1425,9 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { } pRes->row = 0; + pRes->completed = (pRes->numOfRows == 0); - uint8_t code = pRes->code; + int32_t code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { @@ -2231,7 +2204,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= tListLen(pObj->db)); strncpy(pObj->db, temp, tListLen(pObj->db)); - tscSetMgmtIpList(&pConnect->ipList); + if (pConnect->ipList.numOfIps > 0) + tscSetMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; @@ -2292,7 +2266,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true); if (pTableMetaInfo->pTableMeta) { - bool isSuperTable = UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); // taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pMetricMeta), true); @@ -2354,6 +2328,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { for (int i = 0; i < numOfTables; i++) { int64_t uid = htobe64(*(int64_t*)p); p += sizeof(int64_t); + p += sizeof(int32_t); // skip tid TSKEY key = htobe64(*(TSKEY*)p); p += sizeof(TSKEY); tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 14f0fa07ca..d8ec104a50 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -65,32 +65,18 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con terrno = TSDB_CODE_INVALID_PASS; return NULL; } - + + if (ip) { + if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; + if (port) tscMgmtIpSet.port[0] = port; + } + void *pDnodeConn = NULL; if (tscInitRpc(user, pass, &pDnodeConn) != 0) { terrno = TSDB_CODE_NETWORK_UNAVAIL; return NULL; } - - tscMgmtIpSet.numOfIps = 0; - - if (ip && ip[0]) { - tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.numOfIps = 1; - strcpy(tscMgmtIpSet.fqdn[0], ip); - tscMgmtIpSet.port[0] = port? port: tsDnodeShellPort; - } else { - if (tsFirst[0] != 0) { - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; - } - - if (tsSecond[0] != 0) { - taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; - } - } - + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -463,11 +449,11 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } - // current data are exhausted, fetch more data - if (pRes->row >= pRes->numOfRows && pRes->completed != true && + // current data set are exhausted, fetch more data from node + if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql)) && (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE || - pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE || + pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_SHOW || pCmd->command == TSDB_SQL_SELECT || diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index d7c22b2248..a00f856b2a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -79,7 +79,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - if (code == 0 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (code == 0 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { code = tscGetSTableVgroupInfo(pSql, 0); pSql->res.code = code; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 856b678391..9092fdd0b3 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -14,8 +14,6 @@ */ #include "os.h" - -#include "shash.h" #include "taos.h" #include "trpc.h" #include "tsclient.h" @@ -44,8 +42,7 @@ typedef struct SSub { int interval; TAOS_SUBSCRIBE_CALLBACK fp; void * param; - int numOfTables; - SSubscriptionProgress * progress; + SArray* progress; } SSub; @@ -57,92 +54,113 @@ static int tscCompareSubscriptionProgress(const void* a, const void* b) { return 0; } -TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid) { - if (sub == NULL) - return 0; - - SSub* pSub = (SSub*)sub; - for (int s = 0, e = pSub->numOfTables; s < e;) { - int m = (s + e) / 2; - SSubscriptionProgress* p = pSub->progress + m; - if (p->uid > uid) - e = m; - else if (p->uid < uid) - s = m + 1; - else - return p->key; +TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { + if (sub == NULL) { + return dflt; } + SSub* pSub = (SSub*)sub; - return 0; + SSubscriptionProgress target = {.uid = uid, .key = 0}; + SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target); + if (p == NULL) { + return dflt; + } + return p->key; } void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { if( sub == NULL) return; - SSub* pSub = (SSub*)sub; - for (int s = 0, e = pSub->numOfTables; s < e;) { - int m = (s + e) / 2; - SSubscriptionProgress* p = pSub->progress + m; - if (p->uid > uid) - e = m; - else if (p->uid < uid) - s = m + 1; - else { - if (ts >= p->key) p->key = ts; - break; - } + + SSubscriptionProgress target = {.uid = uid, .key = ts}; + SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target); + if (p != NULL) { + p->key = ts; } } +static void asyncCallback(void *param, TAOS_RES *tres, int code) { + assert(param != NULL); + SSqlObj *pSql = ((SSqlObj *)param); + + pSql->res.code = code; + sem_post(&pSql->rspSem); +} + + static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { - SSub* pSub = calloc(1, sizeof(SSub)); - if (pSub == NULL) { - terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscError("failed to allocate memory for subscription"); - return NULL; - } + SSub* pSub = NULL; - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; - tscError("failed to allocate SSqlObj for subscription"); - goto _pSql_failed; - } + TRY( 8 ) { + SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj)); + CLEANUP_PUSH_FREE(true, pSql); + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; - pSql->signature = pSql; - pSql->pTscObj = pObj; + if (tsem_init(&pSql->rspSem, 0, 0) == -1) { + THROW(TAOS_SYSTEM_ERROR(errno)); + } + CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem); - char* sqlstr = (char*)malloc(strlen(sql) + 1); - if (sqlstr == NULL) { - tscError("failed to allocate sql string for subscription"); - goto failed; - } - strcpy(sqlstr, sql); - strtolower(sqlstr, sqlstr); - pSql->sqlstr = sqlstr; + pSql->signature = pSql; + pSql->param = pSql; + pSql->pTscObj = pObj; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; + pSql->fp = asyncCallback; - tsem_init(&pSql->rspSem, 0, 0); + int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (code != TSDB_CODE_SUCCESS) { + THROW(code); + } + CLEANUP_PUSH_FREE(true, pCmd->payload); - SSqlRes *pRes = &pSql->res; - pRes->numOfRows = 1; - pRes->numOfTotal = 0; + pRes->qhandle = 0; + pRes->numOfRows = 1; + + pSql->sqlstr = strdup_throw(sql); + CLEANUP_PUSH_FREE(true, pSql->sqlstr); + strtolower(pSql->sqlstr, pSql->sqlstr); + + code = tsParseSql(pSql, false); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + // wait for the callback function to post the semaphore + sem_wait(&pSql->rspSem); + code = pSql->res.code; + } + if (code != TSDB_CODE_SUCCESS) { + tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code)); + THROW( code ); + } + + if (pSql->cmd.command != TSDB_SQL_SELECT) { + tscError("only 'select' statement is allowed in subscription: %s", pSub->topic); + THROW( -1 ); // TODO + } + + pSub = calloc_throw(1, sizeof(SSub)); + CLEANUP_PUSH_FREE(true, pSub); + pSql->pSubscription = pSub; + pSub->pSql = pSql; + pSub->signature = pSub; + strncpy(pSub->topic, topic, sizeof(pSub->topic)); + pSub->topic[sizeof(pSub->topic) - 1] = 0; + pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); + if (pSub->progress == NULL) { + THROW(TSDB_CODE_CLI_OUT_OF_MEMORY); + } + + CLEANUP_EXECUTE(); + + } CATCH( code ) { + tscError("failed to create subscription object: %s", tstrerror(code)); + CLEANUP_EXECUTE(); + pSub = NULL; + + } END_TRY - pSql->pSubscription = pSub; - pSub->pSql = pSql; - pSub->signature = pSub; - strncpy(pSub->topic, topic, sizeof(pSub->topic)); - pSub->topic[sizeof(pSub->topic) - 1] = 0; return pSub; - -failed: - tfree(sqlstr); - -_pSql_failed: - tfree(pSql); - tfree(pSub); - return NULL; } @@ -159,61 +177,69 @@ static void tscProcessSubscriptionTimer(void *handle, void *tmrId) { } -int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { - int code = (uint8_t)tsParseSql(pSub->pSql, false); +static SArray* getTableList( SSqlObj* pSql ) { + const char* p = strstr( pSql->sqlstr, " from " ); + char* sql = alloca(strlen(p) + 32); + sprintf(sql, "select tbid(tbname)%s", p); + int code = taos_query( pSql->pTscObj, sql ); if (code != TSDB_CODE_SUCCESS) { - tscError("failed to parse sql statement: %s", pSub->topic); - return 0; + tscError("failed to retrieve table id: %s", tstrerror(code)); + return NULL; } - SSqlCmd* pCmd = &pSub->pSql->cmd; - if (pCmd->command != TSDB_SQL_SELECT) { - tscError("only 'select' statement is allowed in subscription: %s", pSub->topic); - return 0; + TAOS_RES* res = taos_use_result( pSql->pTscObj ); + TAOS_ROW row; + SArray* result = taosArrayInit( 128, sizeof(STidTags) ); + while ((row = taos_fetch_row(res))) { + STidTags tags; + memcpy(&tags, row[0], sizeof(tags)); + taosArrayPush(result, &tags); } - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); - int numOfTables = 0; - if (!UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { -// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; -// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); -// numOfTables += pVnodeSidList->numOfSids; -// } - } + return result; +} - SSubscriptionProgress* progress = (SSubscriptionProgress*)calloc(numOfTables, sizeof(SSubscriptionProgress)); - if (progress == NULL) { - tscError("failed to allocate memory for progress: %s", pSub->topic); - return 0; - } - if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { - numOfTables = 1; - int64_t uid = pTableMetaInfo->pTableMeta->uid; - progress[0].uid = uid; - progress[0].key = tscGetSubscriptionProgress(pSub, uid); - } else { -// SSuperTableMeta* pMetricMeta = pTableMetaInfo->pMetricMeta; -// numOfTables = 0; -// for (int32_t i = 0; i < pMetricMeta->numOfVnodes; i++) { -// SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, i); -// for (int32_t j = 0; j < pVnodeSidList->numOfSids; j++) { -// STableIdInfo *pTableMetaInfo = tscGetMeterSidInfo(pVnodeSidList, j); -// int64_t uid = pTableMetaInfo->uid; -// progress[numOfTables].uid = uid; -// progress[numOfTables++].key = tscGetSubscriptionProgress(pSub, uid); -// } -// } - qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress); - } +static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { + SSqlObj* pSql = pSub->pSql; - free(pSub->progress); - pSub->numOfTables = numOfTables; - pSub->progress = progress; + SSqlCmd* pCmd = &pSql->cmd; pSub->lastSyncTime = taosGetTimestampMs(); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0}; + SSubscriptionProgress* p = taosArraySearch(pSub->progress, tscCompareSubscriptionProgress, &target); + if (p == NULL) { + taosArrayClear(pSub->progress); + taosArrayPush(pSub->progress, &target); + } + return 1; + } + + SArray* tables = getTableList(pSql); + size_t numOfTables = taosArrayGetSize(tables); + + SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress)); + for( size_t i = 0; i < numOfTables; i++ ) { + STidTags* tt = taosArrayGet( tables, i ); + SSubscriptionProgress p = { .uid = tt->uid }; + p.key = tscGetSubscriptionProgress(pSub, tt->uid, INT64_MIN); + taosArrayPush(progress, &p); + } + taosArraySort(progress, tscCompareSubscriptionProgress); + + taosArrayDestroy(pSub->progress); + pSub->progress = progress; + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + taosArraySort( tables, tscCompareTidTags ); + tscBuildVgroupTableInfo( pTableMetaInfo, tables ); + } + taosArrayDestroy(tables); + return 1; } @@ -248,32 +274,22 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { return 0; } - if (fgets(buf, sizeof(buf), fp) == NULL || atoi(buf) < 0) { - tscTrace("invalid subscription progress file: %s", pSub->topic); - fclose(fp); - return 0; - } - - int numOfTables = atoi(buf); - SSubscriptionProgress* progress = calloc(numOfTables, sizeof(SSubscriptionProgress)); - for (int i = 0; i < numOfTables; i++) { + SArray* progress = pSub->progress; + taosArrayClear(progress); + while( 1 ) { if (fgets(buf, sizeof(buf), fp) == NULL) { fclose(fp); - free(progress); return 0; } - int64_t uid, key; - sscanf(buf, "%" SCNd64 ":%" SCNd64, &uid, &key); - progress[i].uid = uid; - progress[i].key = key; + SSubscriptionProgress p; + sscanf(buf, "%" SCNd64 ":%" SCNd64, &p.uid, &p.key); + taosArrayPush(progress, &p); } fclose(fp); - qsort(progress, numOfTables, sizeof(SSubscriptionProgress), tscCompareSubscriptionProgress); - pSub->numOfTables = numOfTables; - pSub->progress = progress; - tscTrace("subscription progress loaded, %d tables: %s", numOfTables, pSub->topic); + taosArraySort(progress, tscCompareSubscriptionProgress); + tscTrace("subscription progress loaded, %d tables: %s", taosArrayGetSize(progress), pSub->topic); return 1; } @@ -294,11 +310,10 @@ void tscSaveSubscriptionProgress(void* sub) { } fputs(pSub->pSql->sqlstr, fp); - fprintf(fp, "\n%d\n", pSub->numOfTables); - for (int i = 0; i < pSub->numOfTables; i++) { - int64_t uid = pSub->progress[i].uid; - TSKEY key = pSub->progress[i].key; - fprintf(fp, "%" PRId64 ":%" PRId64 "\n", uid, key); + fprintf(fp, "\n"); + for(size_t i = 0; i < taosArrayGetSize(pSub->progress); i++) { + SSubscriptionProgress* p = taosArrayGet(pSub->progress, i); + fprintf(fp, "%" PRId64 ":%" PRId64 "\n", p->uid, p->key); } fclose(fp); @@ -363,35 +378,34 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { tscRemoveFromSqlList(pSql); if (taosGetTimestampMs() - pSub->lastSyncTime > 10 * 60 * 1000) { - tscTrace("begin meter synchronization"); - char* sqlstr = pSql->sqlstr; - pSql->sqlstr = NULL; - taos_free_result_imp(pSql, 0); - pSql->sqlstr = sqlstr; - taosCacheEmpty(tscCacheHandle); + tscTrace("begin table synchronization"); if (!tscUpdateSubscription(pSub->taos, pSub)) return NULL; - tscTrace("meter synchronization completed"); - } else { - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - - uint32_t type = pQueryInfo->type; - taos_free_result_imp(pSql, 1); - pRes->numOfRows = 1; - pRes->numOfTotal = 0; - pRes->qhandle = 0; - pSql->cmd.command = TSDB_SQL_SELECT; - pQueryInfo->type = type; - - tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0; + tscTrace("table synchronization completed"); } + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + uint32_t type = pQueryInfo->type; + tscFreeSqlResult(pSql); + pRes->numOfRows = 1; + pRes->qhandle = 0; + pSql->cmd.command = TSDB_SQL_SELECT; + pQueryInfo->type = type; + + tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0; + + pSql->fp = asyncCallback; + pSql->param = pSql; tscDoQuery(pSql); - if (pRes->code != TSDB_CODE_NOT_ACTIVE_TABLE) { - break; + sem_wait(&pSql->rspSem); + + if (pRes->code != TSDB_CODE_SUCCESS) { + continue; } // meter was removed, make sync time zero, so that next retry will // do synchronization first pSub->lastSyncTime = 0; + break; } if (pRes->code != TSDB_CODE_SUCCESS) { @@ -421,7 +435,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } tscFreeSqlObj(pSub->pSql); - free(pSub->progress); + taosArrayDestroy(pSub->progress); memset(pSub, 0, sizeof(*pSub)); free(pSub); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 7a1c08d056..75363de3a7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -330,7 +330,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pNewQueryInfo->limit = pSupporter->limit; // fetch the join tag column - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, 0); assert(pQueryInfo->tagCond.joinInfo.hasJoin); @@ -463,77 +463,51 @@ static void tSIntersectionAndLaunchSecQuery(SJoinSupporter* pSupporter, SSqlObj* } } -int32_t tagsOrderCompar(const void* p1, const void* p2) { - STidTags* t1 = (STidTags*) p1; - STidTags* t2 = (STidTags*) p2; +int32_t tscCompareTidTags(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*) p1; + const STidTags* t2 = (const STidTags*) p2; if (t1->vgId != t2->vgId) { - return (t1->vgId > t2->vgId)? 1:-1; - } else { - if (t1->tid != t2->tid) { - return (t1->tid > t2->tid)? 1:-1; - } else { - return 0; - } + return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { + return (t1->tid > t2->tid) ? 1 : -1; + } + return 0; } -static void doBuildVgroupTableInfo(SArray* res, STableMetaInfo* pTableMetaInfo) { - SArray* pGroup = taosArrayInit(4, sizeof(SVgroupTableInfo)); - - SArray* vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo)); - int32_t size = taosArrayGetSize(res); - - STidTags* prev = taosArrayGet(res, 0); - int32_t prevVgId = prev->vgId; - - STableIdInfo item = {.uid = prev->uid, .tid = prev->tid, .key = INT64_MIN}; - taosArrayPush(vgTableIdItem, &item); - - for(int32_t k = 1; k < size; ++k) { - STidTags* t1 = taosArrayGet(res, k); - if (prevVgId != t1->vgId) { - - SVgroupTableInfo info = {0}; - +void tscBuildVgroupTableInfo(STableMetaInfo* pTableMetaInfo, SArray* tables) { + SArray* result = taosArrayInit( 4, sizeof(SVgroupTableInfo) ); + SArray* vgTables = NULL; + STidTags* prev = NULL; + + size_t numOfTables = taosArrayGetSize( tables ); + for( size_t i = 0; i < numOfTables; i++ ) { + STidTags* tt = taosArrayGet( tables, i ); + + if( prev == NULL || tt->vgId != prev->vgId ) { SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; - for(int32_t m = 0; m < pvg->numOfVgroups; ++m) { - if (prevVgId == pvg->vgroups[m].vgId) { + + SVgroupTableInfo info = {{ 0 }}; + for( int32_t m = 0; m < pvg->numOfVgroups; ++m ) { + if( tt->vgId == pvg->vgroups[m].vgId ) { info.vgInfo = pvg->vgroups[m]; break; } } - - assert(info.vgInfo.numOfIps != 0); - info.itemList = vgTableIdItem; - taosArrayPush(pGroup, &info); - - vgTableIdItem = taosArrayInit(4, sizeof(STableIdInfo)); - STableIdInfo item1 = {.uid = t1->uid, .tid = t1->tid, .key = INT64_MIN}; - taosArrayPush(vgTableIdItem, &item1); - prevVgId = t1->vgId; - } else { - taosArrayPush(vgTableIdItem, &item); + assert( info.vgInfo.numOfIps != 0 ); + + vgTables = taosArrayInit( 4, sizeof(STableIdInfo) ); + info.itemList = vgTables; + taosArrayPush( result, &info ); } + + STableIdInfo item = { .uid = tt->uid, .tid = tt->tid, .key = INT64_MIN }; + taosArrayPush( vgTables, &item ); + prev = tt; } - - if (taosArrayGetSize(vgTableIdItem) > 0) { - SVgroupTableInfo info = {0}; - SVgroupsInfo* pvg = pTableMetaInfo->vgroupList; - - for(int32_t m = 0; m < pvg->numOfVgroups; ++m) { - if (prevVgId == pvg->vgroups[m].vgId) { - info.vgInfo = pvg->vgroups[m]; - break; - } - } - - assert(info.vgInfo.numOfIps != 0); - info.itemList = vgTableIdItem; - taosArrayPush(pGroup, &info); - } - - pTableMetaInfo->pVgroupTables = pGroup; + + pTableMetaInfo->pVgroupTables = result; } static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { @@ -627,8 +601,8 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p2 = pParentSql->pSubs[1]->param; - qsort(p1->pIdTagList, p1->num, p1->tagSize, tagsOrderCompar); - qsort(p2->pIdTagList, p2->num, p2->tagSize, tagsOrderCompar); + qsort(p1->pIdTagList, p1->num, p1->tagSize, tscCompareTidTags); + qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -668,11 +642,11 @@ static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0); STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0); - doBuildVgroupTableInfo(s1, pTableMetaInfo1); + tscBuildVgroupTableInfo(pTableMetaInfo1, s1); SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0); STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0); - doBuildVgroupTableInfo(s2, pTableMetaInfo2); + tscBuildVgroupTableInfo(pTableMetaInfo2, s2); pSupporter->pState->numOfCompleted = 0; pSupporter->pState->code = 0; @@ -1096,7 +1070,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter tscInitQueryInfo(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { // return the tableId & tag + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // return the tableId & tag SSchema s = {0}; SColumnIndex index = {0}; @@ -1203,7 +1177,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { } } - pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_METRIC_JOIN_RETRIEVE; + pSql->cmd.command = (pSql->numOfSubs <= 0)? TSDB_SQL_RETRIEVE_EMPTY_RESULT:TSDB_SQL_TABLE_JOIN_RETRIEVE; return TSDB_CODE_SUCCESS; } @@ -1533,8 +1507,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pPObj->cmd, 0); tscClearInterpInfo(pPQueryInfo); - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, - &pPObj->cmd, &pPObj->res); + tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfTotal, pDesc, trsupport->pFinalColModel, pPObj); tscTrace("%p build loser tree completed", pPObj); pPObj->res.precision = pSql->res.precision; @@ -1950,7 +1923,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if(pCmd->command == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + if(pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { if (pRes->completed) { tfree(pRes->tsrow); } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 07926306db..5d56fef1e9 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -23,6 +23,7 @@ #include "tutil.h" #include "tsched.h" #include "tscLog.h" +#include "tscUtil.h" #include "tsclient.h" #include "tglobal.h" #include "tconfig.h" @@ -63,6 +64,7 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) { rpcInit.user = (char*)user; rpcInit.idleTime = 2000; rpcInit.ckey = "key"; + rpcInit.spi = 1; rpcInit.secret = secretEncrypt; *pDnodeConn = rpcOpen(&rpcInit); @@ -113,14 +115,10 @@ void taos_init_imp() { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.numOfIps = 1; - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); - - if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) { - tscMgmtIpSet.numOfIps = 2; - taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]); - } + if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) { + tscError("failed to init mgmt IP list"); + return; + } tscInitMsgsFp(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 00afb977fd..4667606aa8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -157,7 +157,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pTableMetaInfo->vgroupList != NULL); } @@ -172,7 +172,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { if (((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) != TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) { - return UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo); + return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); } return false; @@ -187,7 +187,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { * 4. show queries, instead of a select query */ size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || + if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { return false; } @@ -386,14 +386,16 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { int32_t cmd = pCmd->command; if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_METRIC_JOIN_RETRIEVE) { + cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscRemoveFromSqlList(pSql); } // pSql->sqlstr will be used by tscBuildQueryStreamDesc - pthread_mutex_lock(&pObj->mutex); - tfree(pSql->sqlstr); - pthread_mutex_unlock(&pObj->mutex); + if (pObj->signature == pObj) { + pthread_mutex_lock(&pObj->mutex); + tfree(pSql->sqlstr); + pthread_mutex_unlock(&pObj->mutex); + } tscFreeSqlResult(pSql); @@ -1144,23 +1146,21 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { } SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { - SColumnFilterInfo* pFilter = NULL; - if (numOfFilters > 0) { - pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); - } else { + if (numOfFilters == 0) { assert(src == NULL); return NULL; } + SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1; - - char* pTmp = calloc(1, len); - pFilter[j].pz = (int64_t) pTmp; + pFilter[j].pz = (int64_t) calloc(1, len); - memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len); } } @@ -1211,18 +1211,18 @@ void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { } } -void tscColumnListDestroy(SArray* pColumnBaseInfo) { - if (pColumnBaseInfo == NULL) { +void tscColumnListDestroy(SArray* pColumnList) { + if (pColumnList == NULL) { return; } - size_t num = taosArrayGetSize(pColumnBaseInfo); + size_t num = taosArrayGetSize(pColumnList); for (int32_t i = 0; i < num; ++i) { - SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); + SColumn* pCol = taosArrayGetP(pColumnList, i); tscColumnDestroy(pCol); } - taosArrayDestroy(pColumnBaseInfo); + taosArrayDestroy(pColumnList); } /* @@ -1350,7 +1350,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { return false; } - if (colId == -1 && UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (colId == -1 && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return true; } @@ -1461,10 +1461,11 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) { } /* - * the following three kinds of SqlObj should not be freed + * the following four kinds of SqlObj should not be freed * 1. SqlObj for stream computing * 2. main SqlObj * 3. heartbeat SqlObj + * 4. SqlObj for subscription * * If res code is error and SqlObj does not belong to above types, it should be * automatically freed for async query, ignoring that connection should be kept. @@ -1477,7 +1478,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) { } STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql) { + if (pSql->pStream != NULL || pTscObj->pHb == pSql || pTscObj->pSql == pSql || pSql->pSubscription != NULL) { return false; } @@ -1574,7 +1575,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1646,9 +1647,11 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); - int32_t index = pQueryInfo->numOfTables; - while (index >= 0) { - doRemoveTableMetaInfo(pQueryInfo, --index, removeFromCache); + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + + tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + free(pTableMetaInfo); } tfree(pQueryInfo->pTableMetaInfo); @@ -1668,8 +1671,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST assert(pTableMetaInfo != NULL); if (name != NULL) { - assert(strlen(name) <= TSDB_TABLE_ID_LEN); - strcpy(pTableMetaInfo->name, name); + strncpy(pTableMetaInfo->name, name, TSDB_TABLE_ID_LEN); } pTableMetaInfo->pTableMeta = pTableMeta; @@ -1680,10 +1682,9 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } - if (pTagCols == NULL) { - pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); - } else { - pTableMetaInfo->tagColList = taosArrayClone(pTagCols); + pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTagCols != NULL) { + tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } pQueryInfo->numOfTables += 1; @@ -1702,10 +1703,8 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); - if (pTableMetaInfo->tagColList != NULL) { - taosArrayDestroy(pTableMetaInfo->tagColList); - pTableMetaInfo->tagColList = NULL; - } + tscColumnListDestroy(pTableMetaInfo->tagColList); + pTableMetaInfo->tagColList = NULL; } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -1790,7 +1789,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscFreeSqlObj(pNew); return NULL; } - + tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type @@ -1843,13 +1842,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->fp = fp; pNew->param = param; + pNew->maxRetry = TSDB_MAX_REPLICA_NUM; char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - + // todo handle error + assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); @@ -1861,8 +1862,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); } - assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); - if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { + if (pFinalInfo->pTableMeta == NULL) { + tscError("%p new subquery failed for get pMeterMeta is NULL from cache", pSql); + tscFreeSqlObj(pNew); + return NULL; + } + + assert(pNewQueryInfo->numOfTables == 1); + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pFinalInfo->vgroupList != NULL); } @@ -1988,22 +1996,28 @@ char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; } /** * If current vnode query does not return results anymore (pRes->numOfRows == 0), try the next vnode if exists, - * in case of multi-vnode super table projection query and the result does not reach the limitation. + * while multi-vnode super table projection query and the result does not reach the limitation. */ bool hasMoreVnodesToTry(SSqlObj* pSql) { -// SSqlCmd* pCmd = &pSql->cmd; -// SSqlRes* pRes = &pSql->res; - -// SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || (pTableMetaInfo->pMetricMeta == NULL)) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + if (pCmd->command != TSDB_SQL_FETCH) { return false; -// } + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); -// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; -// return pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && -// (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < totalVnode - 1); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pRes->completed); + + // for normal table, do not try any more if result are exhausted + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) || (pTableMetaInfo->vgroupList == NULL)) { + return false; + } + + int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); } void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { @@ -2019,12 +2033,11 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { assert(pRes->numOfRows == 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && !tscHasReachLimitation(pQueryInfo, pRes)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int32_t totalVnode = 0; -// int32_t totalVnode = pTableMetaInfo->pMetricMeta->numOfVnodes; - - while (++pTableMetaInfo->vgroupIndex < totalVnode) { + + int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + while (++pTableMetaInfo->vgroupIndex < totalVgroups) { tscTrace("%p current vnode:%d exhausted, try next:%d. total vnode:%d. current numOfRes:%d", pSql, - pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVnode, pRes->numOfTotalInCurrentClause); + pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfTotalInCurrentClause); /* * update the limit and offset value for the query on the next vnode, @@ -2040,10 +2053,10 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { } pQueryInfo->limit.offset = pRes->offset; - assert((pRes->offset >= 0 && pRes->numOfRows == 0) || (pRes->offset == 0 && pRes->numOfRows >= 0)); - tscTrace("%p new query to next vnode, vnode index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, pSql, - pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); + + tscTrace("%p new query to next vgroup, index:%d, limit:%" PRId64 ", offset:%" PRId64 ", glimit:%" PRId64, + pSql, pTableMetaInfo->vgroupIndex, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->clauseLimit); /* * For project query with super table join, the numOfSub is equalled to the number of all subqueries. @@ -2056,43 +2069,13 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) { tscResetForNextRetrieve(pRes); - // in case of async query, set the callback function - void* fp1 = pSql->fp; + // set the callback function pSql->fp = fp; - - if (fp1 != NULL) { - assert(fp != NULL); - } - - int32_t ret = tscProcessSql(pSql); // todo check for failure - - // in case of async query, return now - if (fp != NULL) { + int32_t ret = tscProcessSql(pSql); + if (ret == TSDB_CODE_SUCCESS) { return; + } else {// todo check for failure } - - if (ret != TSDB_CODE_SUCCESS) { - pSql->res.code = ret; - return; - } - - // retrieve data - assert(pCmd->command == TSDB_SQL_SELECT); - pCmd->command = TSDB_SQL_FETCH; - - if ((ret = tscProcessSql(pSql)) != TSDB_CODE_SUCCESS) { - pSql->res.code = ret; - return; - } - - // if the result from current virtual node are empty, try next if exists. otherwise, return the results. - if (pRes->numOfRows > 0) { - break; - } - } - - if (pRes->numOfRows == 0) { - tscTrace("%p all vnodes exhausted, prj query completed. total res:%d", pSql, totalVnode, pRes->numOfTotal); } } @@ -2157,3 +2140,56 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column } } +void* malloc_throw(size_t size) { + void* p = malloc(size); + if (p == NULL) { + THROW(TSDB_CODE_CLI_OUT_OF_MEMORY); + } + return p; +} + +void* calloc_throw(size_t nmemb, size_t size) { + void* p = calloc(nmemb, size); + if (p == NULL) { + THROW(TSDB_CODE_CLI_OUT_OF_MEMORY); + } + return p; +} + +char* strdup_throw(const char* str) { + char* p = strdup(str); + if (p == NULL) { + THROW(TSDB_CODE_CLI_OUT_OF_MEMORY); + } + return p; +} + +int tscSetMgmtIpListFromCfg(const char *first, const char *second) { + tscMgmtIpSet.numOfIps = 0; + tscMgmtIpSet.inUse = 0; + + if (first && first[0] != 0) { + if (strlen(first) >= TSDB_FQDN_LEN) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; + } + + if (second && second[0] != 0) { + if (strlen(second) >= TSDB_FQDN_LEN) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; + } + + if ( tscMgmtIpSet.numOfIps == 0) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } + + return 0; +} diff --git a/src/common/inc/qsqltype.h b/src/common/inc/qsqltype.h new file mode 100644 index 0000000000..d1e5bcaa52 --- /dev/null +++ b/src/common/inc/qsqltype.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_QSQLCMD_H +#define TDENGINE_QSQLCMD_H + +#ifdef __cplusplus +extern "C" { +#endif + +// sql type + +#ifdef TSDB_SQL_C +#define TSDB_DEFINE_SQL_TYPE( name, msg ) msg, +char *sqlCmd[] = { + "null", +#else +#define TSDB_DEFINE_SQL_TYPE( name, msg ) name, +enum { + TSDB_SQL_NULL = 0, +#endif + + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) + + // the SQL below is for mgmt node + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_USER, "drop-user" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_USER, "alter-user" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_ACCT, "alter-acct" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DNODE, "drop-dnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" ) + + // SQL below is for read operation + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_READ, "read" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CONNECT, "connect" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_USE_DB, "use-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_META, "meta" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" ) + + // SQL below for client local + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DESCRIBE_TABLE, "describe-table" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_LOCALMERGE, "retrieve-localmerge" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_TABLE_JOIN_RETRIEVE, "join-retrieve" ) + + /* + * build empty result instead of accessing dnode to fetch result + * reset the client cache + */ + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_EMPTY_RESULT, "retrieve-empty-result" ) + + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RESET_CACHE, "reset-cache" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_STATUS, "serv-status" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_DB, "current-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_VERSION, "serv-version" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CLI_VERSION, "cli-version" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_USER, "current-user ") + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_LOCAL, "cfg-local" ) + + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" ) +}; + +// create table operation type +enum TSQL_TYPE { + TSQL_CREATE_TABLE = 0x1, + TSQL_CREATE_STABLE = 0x2, + TSQL_CREATE_TABLE_FROM_STABLE = 0x3, + TSQL_CREATE_STREAM = 0x4, +}; + +extern char *sqlCmd[]; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_QSQLCMD_H diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index b077f40945..da8f3cd1e1 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -175,7 +175,7 @@ void taosInitGlobalCfg(); bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port); +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index a629128324..810d7f492c 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -23,4 +23,5 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); + #endif // TDENGINE_NAME_H diff --git a/src/util/inc/hashstr.h b/src/common/src/sqlcmdstr.c similarity index 62% rename from src/util/inc/hashstr.h rename to src/common/src/sqlcmdstr.c index 66f6e88857..8584ba7976 100644 --- a/src/util/inc/hashstr.h +++ b/src/common/src/sqlcmdstr.c @@ -13,14 +13,6 @@ * along with this program. If not, see . */ -#ifndef _sdb_str_hash_header_ -#define _sdb_str_hash_header_ +#define TSDB_SQL_C -void *sdbOpenStrHash(int maxSessions, int dataSize); -void sdbCloseStrHash(void *handle); -void *sdbAddStrHash(void *handle, void *key, void *pData); -void sdbDeleteStrHash(void *handle, void *key); -void *sdbGetStrHashData(void *handle, void *key); -void *sdbFetchStrHashData(void *handle, void *ptr, void **ppMeta); - -#endif +#include "qsqltype.h" diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 90637265b3..18d8c9ebe2 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -61,7 +61,7 @@ int32_t tscEmbedded = 0; */ int64_t tsMsPerDay[] = {86400000L, 86400000000L}; -char tsFirst[TSDB_FQDN_LEN] = {0}; +char tsFirst[TSDB_FQDN_LEN] = {0}; char tsSecond[TSDB_FQDN_LEN] = {0}; char tsArbitrator[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port @@ -1252,7 +1252,7 @@ bool taosCheckGlobalCfg() { return true; } -int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) { +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = 0; strcpy(fqdn, ep); diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 626fde3293..a972881a41 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -32,18 +32,280 @@ const int32_t TYPE_BYTES[11] = { sizeof(VarDataOffsetT) // TSDB_DATA_TYPE_NCHAR }; +static void getStatics_i8(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int8_t *data = (int8_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int8_t lastVal = TSDB_DATA_TINYINT_NULL; + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((char *)&data[i], TSDB_DATA_TYPE_TINYINT)) { + (*numOfNull) += 1; + continue; + } + + *sum += data[i]; + if (*min > data[i]) { + *min = data[i]; + *minIndex = i; + } + + if (*max < data[i]) { + *max = data[i]; + *maxIndex = i; + } + } +} + +static void getStatics_i16(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int16_t *data = (int16_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { + (*numOfNull) += 1; + continue; + } + + *sum += data[i]; + if (*min > data[i]) { + *min = data[i]; + *minIndex = i; + } + + if (*max < data[i]) { + *max = data[i]; + *maxIndex = i; + } + + // if (isNull(&lastVal, TSDB_DATA_TYPE_SMALLINT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + } +} + +static void getStatics_i32(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int32_t *data = (int32_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + // int64_t lastKey = 0; + // int32_t lastVal = TSDB_DATA_INT_NULL; + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { + (*numOfNull) += 1; + continue; + } + + *sum += data[i]; + if (*min > data[i]) { + *min = data[i]; + *minIndex = i; + } + + if (*max < data[i]) { + *max = data[i]; + *maxIndex = i; + } + + // if (isNull(&lastVal, TSDB_DATA_TYPE_INT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + } +} + +static void getStatics_i64(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int64_t *data = (int64_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { + (*numOfNull) += 1; + continue; + } + + *sum += data[i]; + if (*min > data[i]) { + *min = data[i]; + *minIndex = i; + } + + if (*max < data[i]) { + *max = data[i]; + *maxIndex = i; + } + + // if (isNull(&lastVal, TSDB_DATA_TYPE_BIGINT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + } +} + +static void getStatics_f(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + float *data = (float *)pData; + float fmin = DBL_MAX; + float fmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { + (*numOfNull) += 1; + continue; + } + + float fv = 0; + fv = GET_FLOAT_VAL(&(data[i])); + dsum += fv; + if (fmin > fv) { + fmin = fv; + *minIndex = i; + } + + if (fmax < fv) { + fmax = fv; + *maxIndex = i; + } + + // if (isNull(&lastVal, TSDB_DATA_TYPE_FLOAT)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + } + + double csum = 0; + csum = GET_DOUBLE_VAL(sum); + csum += dsum; +#ifdef _TD_ARM_32_ + SET_DOUBLE_VAL_ALIGN(sum, &csum); + SET_DOUBLE_VAL_ALIGN(max, &fmax); + SET_DOUBLE_VAL_ALIGN(min, &fmin); +#else + *sum = csum; + *max = fmax; + *min = fmin; +#endif +} + +static void getStatics_d(const TSKEY *primaryKey, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + double *data = (double *)pData; + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + ASSERT(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { + (*numOfNull) += 1; + continue; + } + + double dv = 0; + dv = GET_DOUBLE_VAL(&(data[i])); + dsum += dv; + if (dmin > dv) { + dmin = dv; + *minIndex = i; + } + + if (dmax < dv) { + dmax = dv; + *maxIndex = i; + } + + // if (isNull(&lastVal, TSDB_DATA_TYPE_DOUBLE)) { + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } else { + // *wsum = lastVal * (primaryKey[i] - lastKey); + // lastKey = primaryKey[i]; + // lastVal = data[i]; + // } + } + + double csum = 0; + csum = GET_DOUBLE_VAL(sum); + csum += dsum; + + +#ifdef _TD_ARM_32_ + SET_DOUBLE_VAL_ALIGN(sum, &csum); + SET_DOUBLE_VAL_ALIGN(max, &dmax); + SET_DOUBLE_VAL_ALIGN(min, &dmin); +#else + *sum = csum; + *max = dmax; + *min = dmin; +#endif +} + tDataTypeDescriptor tDataTypeDesc[11] = { - {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL}, - {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool}, - {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint}, - {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint}, - {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt}, - {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint}, - {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat}, - {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble}, - {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString}, - {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp}, - {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString}, + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, NULL}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", tsCompressBigint, tsDecompressBigint, getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", tsCompressString, tsDecompressString, NULL}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp, NULL}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", tsCompressString, tsDecompressString, NULL}, }; char tTokenTypeSwitcher[13] = { diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 2b409268d9..d076beb8c0 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -241,17 +241,12 @@ function CTaosInterface (config = null, pass = false) { 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], // Subscription - //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, long time, int mseconds) - ////TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds); - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int64, ref.types.int] ], - //TAOS_ROW taos_consume(TAOS_SUB *tsub); - 'taos_consume': [ ref.refType(ref.types.void_ptr2), [ref.types.void_ptr] ], + //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) + 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + // TAOS_RES *taos_consume(TAOS_SUB *tsub) + 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], //void taos_unsubscribe(TAOS_SUB *tsub); 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], - //int taos_subfields_count(TAOS_SUB *tsub); - 'taos_subfields_count': [ ref.types.int, [ref.types.void_ptr ] ], - //TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub); - 'taos_fetch_subfields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), @@ -362,7 +357,7 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { blocks.fill(null); num_of_rows = Math.abs(num_of_rows); let offset = 0; - pblock = pblock.deref() + pblock = pblock.deref(); for (let i = 0; i < fields.length; i++) { if (!convertFunctions[fields[i]['type']] ) { @@ -472,64 +467,40 @@ CTaosInterface.prototype.getClientInfo = function getClientInfo() { } // Subscription -CTaosInterface.prototype.subscribe = function subscribe(host=null, user="root", password="taosdata", db=null, table=null, time=null, mseconds=null) { - let dbOrig = db; - let tableOrig = table; - try { - host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); +CTaosInterface.prototype.subscribe = function subscribe(connection, restart, topic, sql, interval) { + let topicOrig = topic; + let sqlOrig = sql; + try { + sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } catch(err) { - throw "Attribute Error: host is expected as a str"; + throw "Attribute Error: sql is expected as a str"; } try { - user = ref.allocCString(user) + topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } catch(err) { - throw "Attribute Error: user is expected as a str"; + throw TypeError("topic is expected as a str"); } - try { - password = ref.allocCString(password); - } - catch(err) { - throw "Attribute Error: password is expected as a str"; - } - try { - db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); - } - catch(err) { - throw "Attribute Error: db is expected as a str"; - } - try { - table = table != null ? ref.allocCString(table) : ref.alloc(ref.types.char_ptr, ref.NULL); - } - catch(err) { - throw TypeError("table is expected as a str"); - } - try { - mseconds = ref.alloc(ref.types.int, mseconds); - } - catch(err) { - throw TypeError("mseconds is expected as an int"); - } - //TAOS_SUB *taos_subscribe(char *host, char *user, char *pass, char *db, char *table, int64_t time, int mseconds); - let subscription = this.libtaos.taos_subscribe(host, user, password, db, table, time, mseconds); + + restart = ref.alloc(ref.types.int, restart); + + let subscription = this.libtaos.taos_subscribe(connection, restart, topic, sql, null, null, interval); if (ref.isNull(subscription)) { throw new errors.TDError('Failed to subscribe to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig); } else { - console.log('Successfully subscribed to TDengine | Database: ' + dbOrig + ', Table: ' + tableOrig); + console.log('Successfully subscribed to TDengine - Topic: ' + topicOrig); } return subscription; } -CTaosInterface.prototype.subFieldsCount = function subFieldsCount(subscription) { - return this.libtaos.taos_subfields_count(subscription); -} -CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription) { - let pfields = this.libtaos.taos_fetch_subfields(subscription); - let pfieldscount = this.subFieldsCount(subscription); + +CTaosInterface.prototype.consume = function consume(subscription) { + let result = this.libtaos.taos_consume(subscription); let fields = []; + let pfields = this.fetchFields(result); if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type fields.push( { @@ -539,27 +510,23 @@ CTaosInterface.prototype.fetchSubFields = function fetchSubFields(subscription) }) } } - return fields; -} -CTaosInterface.prototype.consume = function consume(subscription) { - let row = this.libtaos.taos_consume(subscription); - let fields = this.fetchSubFields(subscription); - //let isMicro = (cti.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); - let isMicro = false; //no supported function for determining precision? - let blocks = new Array(fields.length); - blocks.fill(null); - let numOfRows2 = 1; //Math.abs(numOfRows2); - let offset = 0; - if (numOfRows2 > 0){ - for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); + + let data = []; + while(true) { + let { blocks, num_of_rows } = this.fetchBlock(result, fields); + if (num_of_rows == 0) { + break; + } + for (let i = 0; i < num_of_rows; i++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let j = 0; j < fields.length; j++) { + rowBlock[j] = blocks[j][i]; } - blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); - offset += fields[i]['bytes'] * numOfRows2; + data[data.length-1] = (rowBlock); } } - return {blocks:blocks, fields:fields}; + return { data: data, fields: fields, result: result }; } CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { //void taos_unsubscribe(TAOS_SUB *tsub); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index 092c19dfd4..acfe96dfbc 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -405,18 +405,16 @@ TDengineCursor.prototype.getClientInfo = function getClientInfo() { /** * Subscribe to a table from a database in TDengine. * @param {Object} config - A configuration object containing the configuration options for the subscription - * @param {string} config.host - The host to subscribe to - * @param {string} config.user - The user to subscribe as - * @param {string} config.password - The password for the said user - * @param {string} config.db - The db containing the table to subscribe to - * @param {string} config.table - The name of the table to subscribe to - * @param {number} config.time - The start time to start a subscription session - * @param {number} config.mseconds - The pulling period of the subscription session + * @param {string} config.restart - whether or not to continue a subscription if it already exits, otherwise start from beginning + * @param {string} config.topic - The unique identifier of a subscription + * @param {string} config.sql - A sql statement for data query + * @param {string} config.interval - The pulling interval * @return {Buffer} A buffer pointing to the subscription session handle * @since 1.3.0 */ TDengineCursor.prototype.subscribe = function subscribe(config) { - return this._chandle.subscribe(config.host, config.user, config.password, config.db, config.table, config.time, config.mseconds); + let restart = config.restart ? 1 : 0; + return this._chandle.subscribe(this._connection._conn, restart, config.topic, config.sql, config.interval); }; /** * An infinite loop that consumes the latest data and calls a callback function that is provided. @@ -426,18 +424,8 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let res = this._chandle.consume(subscription); - let data = []; - let num_of_rows = res.blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(res.fields.length); - for (let k = 0; k < res.fields.length; k++) { - rowBlock[k] = res.blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(data, res.fields, subscription); + let { data, fields, result} = this._chandle.consume(subscription); + callback(data, fields, result); } } /** diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json index ab7789f7c6..1137e35106 100644 --- a/src/connector/nodejs/package-lock.json +++ b/src/connector/nodejs/package-lock.json @@ -1,6 +1,6 @@ { "name": "td-connector", - "version": "1.5.0", + "version": "1.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 7f42d40a91..8d7a971aa3 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td-connector", - "version": "1.5.0", + "version": "1.6.1", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "scripts": { diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 67f0a783b9..5d96e798d8 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -33,12 +33,12 @@ for (let i = 0; i < 10000; i++) { parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // Int parseInt( R(-Math.pow(2,31) + 1 , Math.pow(2,31) - 1) ), // BigInt parseFloat( R(-3.4E38, 3.4E38) ), // Float - parseFloat( R(-1.7E308, 1.7E308) ), // Double + parseFloat( R(-1.7E30, 1.7E30) ), // Double "\"Long Binary\"", // Binary parseInt( R(-32767, 32767) ), // Small Int parseInt( R(-127, 127) ), // Tiny Int randomBool(), - "\"Nchars 一些中文字幕\""]; // Bool + "\"Nchars\""]; // Bool c1.execute('insert into td_connector_test.all_types values(' + insertData.join(',') + ' );', {quiet:true}); if (i % 1000 == 0) { console.log("Insert # " , i); diff --git a/src/connector/nodejs/test/testSubscribe.js b/src/connector/nodejs/test/testSubscribe.js new file mode 100644 index 0000000000..30fb3f4256 --- /dev/null +++ b/src/connector/nodejs/test/testSubscribe.js @@ -0,0 +1,16 @@ +const taos = require('../tdengine'); +var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:10}); +var c1 = conn.cursor(); +let stime = new Date(); +let interval = 1000; +c1.execute('use td_connector_test'); +let sub = c1.subscribe({ + restart: true, + sql: "select AVG(_int) from td_connector_test.all_Types;", + topic: 'all_Types', + interval: 1000 +}); + +c1.consumeData(sub, (data, fields) => { + console.log(data); +}); \ No newline at end of file diff --git a/src/util/inc/hashint.h b/src/dnode/inc/dnodeMain.h similarity index 63% rename from src/util/inc/hashint.h rename to src/dnode/inc/dnodeMain.h index 052689f337..df7698ffc3 100644 --- a/src/util/inc/hashint.h +++ b/src/dnode/inc/dnodeMain.h @@ -13,14 +13,18 @@ * along with this program. If not, see . */ -#ifndef _sdb_int_hash_header_ -#define _sdb_int_hash_header_ +#ifndef TDENGINE_DNODE_MAIN_H +#define TDENGINE_DNODE_MAIN_H -void *sdbOpenIntHash(int maxSessions, int dataSize); -void sdbCloseIntHash(void *handle); -void *sdbAddIntHash(void *handle, void *key, void *pData); -void sdbDeleteIntHash(void *handle, void *key); -void *sdbGetIntHashData(void *handle, void *key); -void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta); +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dnodeInitSystem(); +void dnodeCleanUpSystem(); + +#ifdef __cplusplus +} +#endif #endif diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index dc5f7d192f..2f693c61fb 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -16,8 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taos.h" -#include "tglobal.h" -#include "trpc.h" #include "tutil.h" #include "tconfig.h" #include "tglobal.h" @@ -29,112 +27,14 @@ #include "dnodeVRead.h" #include "dnodeShell.h" #include "dnodeVWrite.h" -#include "tgrant.h" -static int32_t dnodeInitSystem(); static int32_t dnodeInitStorage(); -extern void grantParseParameter(); static void dnodeCleanupStorage(); -static void dnodeCleanUpSystem(); static void dnodeSetRunStatus(SDnodeRunStatus status); -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static void dnodeCheckDataDirOpenned(char *dir); static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED; -int32_t main(int32_t argc, char *argv[]) { - // Set global configuration file - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - strcpy(configDir, argv[++i]); - } else { - printf("'-c' requires a parameter, default:%s\n", configDir); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-V") == 0) { -#ifdef _SYNC - char *versionStr = "enterprise"; -#else - char *versionStr = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("buildinfo: %s\n", buildinfo); - exit(EXIT_SUCCESS); - } else if (strcmp(argv[i], "-k") == 0) { - grantParseParameter(); - exit(EXIT_SUCCESS); - } -#ifdef TAOS_MEM_CHECK - else if (strcmp(argv[i], "--alloc-random-fail") == 0) { - if ((i < argc - 1) && (argv[i + 1][0] != '-')) { - taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true); - } else { - taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true); - } - } else if (strcmp(argv[i], "--detect-mem-leak") == 0) { - if ((i < argc - 1) && (argv[i + 1][0] != '-')) { - taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true); - } else { - taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); - } - } -#endif - } - - /* Set termination handler. */ - struct sigaction act = {0}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); - - // Open /var/log/syslog file to record information. - openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); - syslog(LOG_INFO, "Starting TDengine service..."); - - // Initialize the system - if (dnodeInitSystem() < 0) { - syslog(LOG_ERR, "Error initialize TDengine system"); - closelog(); - - dnodeCleanUpSystem(); - exit(EXIT_FAILURE); - } - - syslog(LOG_INFO, "Started TDengine service successfully."); - - while (1) { - sleep(1000); - } -} - -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 135"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dPrint("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); - dnodeCleanUpSystem(); - // close the syslog - syslog(LOG_INFO, "Shut down TDengine service successfully"); - dPrint("TDengine is shut down!"); - closelog(); - exit(EXIT_SUCCESS); -} - -static int32_t dnodeInitSystem() { +int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE); tscEmbedded = 1; taosResolveCRC(); @@ -180,7 +80,7 @@ static int32_t dnodeInitSystem() { return 0; } -static void dnodeCleanUpSystem() { +void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) { dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED); dnodeCleanupShell(); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 36a7c98807..81d426b496 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "cJSON.h" -#include "ihash.h" #include "taoserror.h" #include "taosmsg.h" #include "ttime.h" diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 2e081de2a8..86193fcebc 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -33,7 +33,7 @@ typedef struct { void (*stopFp)(); } SModule; -static SModule tsModule[TSDB_MOD_MAX] = {0}; +static SModule tsModule[TSDB_MOD_MAX] = {{0}}; static uint32_t tsModuleStatus = 0; static void dnodeSetModuleStatus(int32_t module) { diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 51913d80c4..c91da4953d 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -52,7 +52,8 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mgmtProcessReqMsgFromDnode; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = mgmtProcessReqMsgFromDnode; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = mgmtProcessReqMsgFromDnode; - + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = mgmtProcessReqMsgFromDnode; + SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = tsDnodeDnodePort; @@ -163,3 +164,9 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) { rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg); } + +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { + SRpcIpSet ipSet = {0}; + dnodeGetMnodeDnodeIpSet(&ipSet); + rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp); +} diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index dc0efd405f..fbed164839 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -21,6 +21,7 @@ #include "trpc.h" #include "tglobal.h" #include "http.h" +#include "mnode.h" #include "dnode.h" #include "dnodeInt.h" #include "dnodeVRead.h" @@ -138,7 +139,34 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { } static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - return TSDB_CODE_SUCCESS; + int code = mgmtRetriveAuth(user, spi, encrypt, secret, ckey); + if (code != TSDB_CODE_NOT_READY) return code; + + SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); + strcpy(pMsg->user, user); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pMsg; + rpcMsg.contLen = sizeof(SDMAuthMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; + + dTrace("user:%s, send auth msg to mnode", user); + SRpcMsg rpcRsp = {0}; + dnodeSendMsgToDnodeRecv(&rpcMsg, &rpcRsp); + + if (rpcRsp.code != 0) { + dError("user:%s, auth msg received from mnode, error:%s", user, tstrerror(rpcRsp.code)); + } else { + SDMAuthRsp *pRsp = rpcRsp.pCont; + dTrace("user:%s, auth msg received from mnode", user); + memcpy(secret, pRsp->secret, TSDB_KEY_LEN); + memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + *spi = pRsp->spi; + *encrypt = pRsp->encrypt; + } + + rpcFreeCont(rpcRsp.pCont); + return rpcRsp.code; } SDnodeStatisInfo dnodeGetStatisInfo() { diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c new file mode 100644 index 0000000000..683328db29 --- /dev/null +++ b/src/dnode/src/dnodeSystem.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tgrant.h" +#include "tutil.h" +#include "tglobal.h" +#include "dnodeInt.h" +#include "dnodeMain.h" + +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); + +int32_t main(int32_t argc, char *argv[]) { + // Set global configuration file + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + strcpy(configDir, argv[++i]); + } else { + printf("'-c' requires a parameter, default:%s\n", configDir); + exit(EXIT_FAILURE); + } + } else if (strcmp(argv[i], "-V") == 0) { +#ifdef _SYNC + char *versionStr = "enterprise"; +#else + char *versionStr = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("buildinfo: %s\n", buildinfo); + exit(EXIT_SUCCESS); + } else if (strcmp(argv[i], "-k") == 0) { + grantParseParameter(); + exit(EXIT_SUCCESS); + } +#ifdef TAOS_MEM_CHECK + else if (strcmp(argv[i], "--alloc-random-fail") == 0) { + if ((i < argc - 1) && (argv[i + 1][0] != '-')) { + taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true); + } else { + taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true); + } + } else if (strcmp(argv[i], "--detect-mem-leak") == 0) { + if ((i < argc - 1) && (argv[i + 1][0] != '-')) { + taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true); + } else { + taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true); + } + } +#endif + } + + /* Set termination handler. */ + struct sigaction act = {{0}}; + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = signal_handler; + sigaction(SIGTERM, &act, NULL); + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); + sigaction(SIGUSR1, &act, NULL); + sigaction(SIGUSR2, &act, NULL); + + // Open /var/log/syslog file to record information. + openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); + syslog(LOG_INFO, "Starting TDengine service..."); + + // Initialize the system + if (dnodeInitSystem() < 0) { + syslog(LOG_ERR, "Error initialize TDengine system"); + closelog(); + + dnodeCleanUpSystem(); + exit(EXIT_FAILURE); + } + + syslog(LOG_INFO, "Started TDengine service successfully."); + + while (1) { + sleep(1000); + } +} + +static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { + if (signum == SIGUSR1) { + taosCfgDynamicOptions("debugFlag 135"); + return; + } + if (signum == SIGUSR2) { + taosCfgDynamicOptions("resetlog"); + return; + } + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); + // clean the system. + dPrint("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); + dnodeCleanUpSystem(); + // close the syslog + syslog(LOG_INFO, "Shut down TDengine service successfully"); + dPrint("TDengine is shut down!"); + closelog(); + exit(EXIT_SUCCESS); +} diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index aaf71838bf..aa8cd99785 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -92,8 +92,6 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) { char *pCont = (char *) pMsg->pCont; void *pVnode; - dTrace("dnode %s msg incoming, thandle:%p", taosMsg[pMsg->msgType], pMsg->handle); - while (leftLen > 0) { SMsgHead *pHead = (SMsgHead *) pCont; pHead->vgId = htonl(pHead->vgId); @@ -214,6 +212,7 @@ static void *dnodeProcessReadQueue(void *param) { continue; } + dTrace("%p, msg:%s will be processed", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]); int32_t code = vnodeProcessRead(pVnode, pReadMsg->rpcMsg.msgType, pReadMsg->pCont, pReadMsg->contLen, &pReadMsg->rspRet); dnodeSendRpcReadRsp(pVnode, pReadMsg, code); taosFreeQitem(pReadMsg); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 5de4c16c50..879082f223 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -200,6 +200,7 @@ static void *dnodeProcessWriteQueue(void *param) { pHead->msgType = pWrite->rpcMsg.msgType; pHead->version = 0; pHead->len = pWrite->contLen; + dTrace("%p, msg:%s will be processed", pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); } else { pHead = (SWalHead *)item; } diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 5145a46831..9884cf2870 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -52,6 +52,7 @@ int32_t dnodeGetDnodeId(); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); #ifdef __cplusplus } diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 37fec24c20..48b1ac97bd 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -26,8 +26,10 @@ void mgmtCleanUpSystem(); void mgmtStopSystem(); void sdbUpdateSync(); -void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg); -void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg); +int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg); +void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg); + #ifdef __cplusplus } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 33a0e4f2c6..ba015d7bbf 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,6 +32,9 @@ extern "C" { #define TSKEY int64_t #endif +#define TSWINDOW_INITIALIZER {INT64_MIN, INT64_MAX}; +#define TSKEY_INITIAL_VAL INT64_MIN + // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; @@ -144,6 +147,8 @@ typedef struct tDataTypeDescriptor { char algorithm, char *const buffer, int bufferSize); int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize); + void (*getStatisFunc)(const TSKEY *primaryKey, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypeDesc[11]; @@ -188,20 +193,20 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_ACCT_LEN TSDB_UNI_LEN #define TSDB_PASSWORD_LEN TSDB_UNI_LEN -#define TSDB_MAX_COLUMNS 256 +#define TSDB_MAX_COLUMNS 1024 #define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 192 #define TSDB_DB_NAME_LEN 32 #define TSDB_COL_NAME_LEN 64 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 16 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 6mb -#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 16 -#define TSDB_MAX_TAGS_LEN 512 -#define TSDB_MAX_TAGS 32 +#define TSDB_MAX_BYTES_PER_ROW TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_TAGS_LEN 65536 +#define TSDB_MAX_TAGS 128 #define TSDB_AUTH_LEN 16 #define TSDB_KEY_LEN 16 @@ -213,7 +218,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 -#define TSDB_FQDN_LEN 72 +#define TSDB_FQDN_LEN 256 #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_METER_VNODE_BITS 20 @@ -229,9 +234,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) -#define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size +#define TSDB_DEFAULT_PAYLOAD_SIZE 2048 // default payload size #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth -#define TSDB_SQLCMD_SIZE 1024 +#define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MAX_VNODES 256 #define TSDB_MIN_VNODES 50 #define TSDB_INVALID_VNODE_NUM 0 @@ -341,8 +346,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_DBS 100 #define TSDB_MAX_VGROUPS 1000 #define TSDB_MAX_SUPER_TABLES 100 -#define TSDB_MAX_NORMAL_TABLES 1000 -#define TSDB_MAX_CHILD_TABLES 100000 #define TSDB_PORT_DNODESHELL 0 #define TSDB_PORT_DNODEDNODE 5 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b5ab4412a9..1390d66113 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -74,6 +74,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_OPTION, 0, 26, "invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_CONFIGURED, 0, 27, "not configured") TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE, 0, 28, "node offline") TAOS_DEFINE_ERROR(TSDB_CODE_NETWORK_UNAVAIL, 0, 29, "network unavailable") +TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_REQUIRED, 0, 30, "auth required") // db TAOS_DEFINE_ERROR(TSDB_CODE_DB_NOT_SELECTED, 0, 100, "db not selected") @@ -169,6 +170,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 459, "invalid handle" TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 460, "query cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 461, "invalid ie") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 462, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FQDN, 0, 463, "invalid FQDN") // others TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 500, "invalid file format") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 8dac73606d..728e733f84 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -100,6 +100,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_TABLE, "config-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_CONFIG_VNODE, "config-vnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_STATUS, "status" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_GRANT, "grant" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DM_AUTH, "auth" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY12, "dummy12" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) @@ -186,13 +187,13 @@ typedef struct SMsgHead { // Submit message for one table typedef struct SSubmitBlk { - int64_t uid; // table unique id - int32_t tid; // table id - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - int16_t numOfRows; // total number of rows in current submit block - char data[]; + uint64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; } SSubmitBlk; // Submit message for this TSDB @@ -326,9 +327,9 @@ typedef struct { } SMDDropTableMsg; typedef struct { - int32_t contLen; - int32_t vgId; - int64_t uid; + int32_t contLen; + int32_t vgId; + uint64_t uid; char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDDropSTableMsg; @@ -403,9 +404,9 @@ typedef struct SColumnInfo { } SColumnInfo; typedef struct STableIdInfo { - int64_t uid; - int32_t tid; - TSKEY key; // last accessed ts, for subscription + uint64_t uid; + int32_t tid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -626,7 +627,6 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; char tableId[TSDB_TABLE_ID_LEN + 1]; // table id - char stableId[TSDB_TABLE_ID_LEN + 1]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -737,6 +737,14 @@ typedef struct { char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDAlterStreamMsg; +typedef struct { + char user[TSDB_USER_LEN + 1]; + char spi; + char encrypt; + char secret[TSDB_KEY_LEN + 1]; + char ckey[TSDB_KEY_LEN + 1]; +} SDMAuthMsg, SDMAuthRsp; + #pragma pack(pop) #ifdef __cplusplus diff --git a/src/inc/trpc.h b/src/inc/trpc.h index eff210433f..16223b813a 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -48,6 +48,7 @@ typedef struct { int contLen; int32_t code; void *handle; + void *ahandle; //app handle set by client, for debug purpose } SRpcMsg; typedef struct { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 77a0e23af9..0b19155626 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -34,12 +34,14 @@ extern "C" { #define TSDB_INVALID_SUPER_TABLE_ID -1 +#define TSDB_STATUS_COMMIT_START 1 +#define TSDB_STATUS_COMMIT_OVER 2 + // --------- TSDB APPLICATION HANDLE DEFINITION typedef struct { - // WAL handle void *appH; void *cqH; - int (*walCallBack)(void *); + int (*notifyStatus)(void *, int status); int (*eventCallBack)(void *); } STsdbAppH; @@ -70,7 +72,7 @@ typedef void TsdbRepoT; // use void to hide implementation details from outside int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); int32_t tsdbDropRepo(TsdbRepoT *repo); TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH); -int32_t tsdbCloseRepo(TsdbRepoT *repo); +int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit); int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg); // --------- TSDB TABLE DEFINITION @@ -109,6 +111,8 @@ int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid); +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size); + // the TSDB repository info typedef struct STsdbRepoInfo { STsdbCfg tsdbCfg; @@ -136,7 +140,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TsdbRepoT *pRepo, SSubmitMsg *pMsg); +int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ; // -- FOR QUERY TIME SERIES DATA diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 0d6004bba5..137b97e287 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -57,7 +57,7 @@ typedef struct { // if name is empty(name[0] is zero), get the file from index or after, used by master // if name is provided(name[0] is not zero), get the named file at the specified index, used by unsynced node // it returns the file magic number and size, if file not there, magic shall be 0. -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size); +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file @@ -73,7 +73,7 @@ typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); typedef void (*FNotifyRole)(void *ahandle, int8_t role); // when data file is synced successfully, notity app -typedef void (*FNotifyFileSynced)(void *ahandle); +typedef void (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); typedef struct { int32_t vgId; // vgroup ID diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 381c6f1dbf..549b0ef977 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -41,14 +41,13 @@ // dynamic config timestamp width according to maximum time precision extern int32_t TIMESTAMP_OUTPUT_LENGTH; -typedef struct History History; -struct History { +typedef struct SShellHistory { char* hist[MAX_HISTORY_SIZE]; int hstart; int hend; -}; +} SShellHistory; -struct arguments { +typedef struct SShellArguments { char* host; char* password; char* user; @@ -62,11 +61,11 @@ struct arguments { char* commands; int abort; int port; -}; +} SShellArguments; /**************** Function declarations ****************/ -extern void shellParseArgument(int argc, char* argv[], struct arguments* arguments); -extern TAOS* shellInit(struct arguments* args); +extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); +extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); extern void taos_error(TAOS* con); extern int regex_match(const char* s, const char* reg, int cflags); @@ -76,7 +75,7 @@ void shellRunCommandOnServer(TAOS* con, char command[]); void read_history(); void write_history(); void source_file(TAOS* con, char* fptr); -void source_dir(TAOS* con, struct arguments* args); +void source_dir(TAOS* con, SShellArguments* args); void get_history_path(char* history); void cleanup_handler(void* arg); void exitShell(); @@ -89,12 +88,12 @@ int isCommentLine(char *line); extern char PROMPT_HEADER[]; extern char CONTINUE_PROMPT[]; extern int prompt_size; -extern History history; +extern SShellHistory history; extern struct termios oldtio; extern void set_terminal_mode(); extern int get_old_terminal_mode(struct termios* tio); -extern void reset_terminal_mode(); -extern struct arguments args; -extern TAOS_RES* result; +extern void reset_terminal_mode(); +extern SShellArguments args; +extern TAOS_RES* result; #endif diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 69475371dc..dbb05f6a35 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -33,12 +33,12 @@ char PROMPT_HEADER[] = "taos> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 6; TAOS_RES *result = NULL; -History history; +SShellHistory history; /* * FUNCTION: Initialize the shell. */ -TAOS *shellInit(struct arguments *args) { +TAOS *shellInit(SShellArguments *args) { printf("\n"); printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); fflush(stdout); diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index b29b96379b..2cbd07db4b 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -221,7 +221,7 @@ void* shellImportThreadFp(void *arg) return NULL; } -static void shellRunImportThreads(struct arguments* args) +static void shellRunImportThreads(SShellArguments* args) { pthread_attr_t thattr; ShellThreadObj *threadObj = (ShellThreadObj *)calloc(args->threadNum, sizeof(ShellThreadObj)); @@ -254,7 +254,7 @@ static void shellRunImportThreads(struct arguments* args) free(threadObj); } -void source_dir(TAOS* con, struct arguments* args) { +void source_dir(TAOS* con, SShellArguments* args) { shellGetDirectoryFileList(args->dir); int64_t start = taosGetTimestampMs(); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index f5a1145cf8..da2bd94814 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -50,7 +50,7 @@ static struct argp_option options[] = { static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we know is a pointer to our arguments structure. */ - struct arguments *arguments = state->input; + SShellArguments *arguments = state->input; wordexp_t full_path; switch (key) { @@ -129,7 +129,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; -void shellParseArgument(int argc, char *argv[], struct arguments *arguments) { +void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { static char verType[32] = {0}; sprintf(verType, "version: %s\n", version); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index b39fba285f..f8010b84cd 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -62,7 +62,7 @@ int checkVersion() { } // Global configurations -struct arguments args = { +SShellArguments args = { .host = NULL, .password = NULL, .user = NULL, diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 33b76cea2d..3cacafa505 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -67,7 +67,7 @@ static struct argp_option options[] = { {0}}; /* Used by main to communicate with parse_opt. */ -struct arguments { +typedef struct DemoArguments { char *host; uint16_t port; char *user; @@ -87,13 +87,13 @@ struct arguments { int num_of_DPT; int abort; char **arg_list; -}; +} SDemoArguments; /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we know is a pointer to our arguments structure. */ - struct arguments *arguments = state->input; + SDemoArguments *arguments = state->input; wordexp_t full_path; char **sptr; switch (key) { @@ -269,7 +269,7 @@ double getCurrentTime(); void callBack(void *param, TAOS_RES *res, int code); int main(int argc, char *argv[]) { - struct arguments arguments = {NULL, // host + SDemoArguments arguments = {NULL, // host 0, // port "root", // user "taosdata", // password diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index ed98a9b92c..adba091136 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -168,7 +168,7 @@ static struct argp_option options[] = { {0}}; /* Used by main to communicate with parse_opt. */ -struct arguments { +typedef struct SDumpArguments { // connection option char *host; char *user; @@ -193,13 +193,13 @@ struct arguments { char **arg_list; int arg_list_len; bool isDumpIn; -}; +} SDumpArguments; /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we know is a pointer to our arguments structure. */ - struct arguments *arguments = state->input; + SDumpArguments *arguments = state->input; wordexp_t full_path; switch (key) { @@ -296,31 +296,31 @@ char *command = NULL; char *lcommand = NULL; char *buffer = NULL; -int taosDumpOut(struct arguments *arguments); +int taosDumpOut(SDumpArguments *arguments); -int taosDumpIn(struct arguments *arguments); +int taosDumpIn(SDumpArguments *arguments); void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp); +int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp); -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct arguments *arguments, FILE *fp); +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp); -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, struct arguments *arguments, +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments, FILE *fp); -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp); +int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp); -int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp); +int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp); -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments); +int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments); -int taosCheckParam(struct arguments *arguments); +int taosCheckParam(SDumpArguments *arguments); void taosFreeDbInfos(); int main(int argc, char *argv[]) { - struct arguments arguments = { + SDumpArguments arguments = { // connection option NULL, "root", "taosdata", 0, // output file @@ -424,7 +424,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) { return -1; } -int taosDumpOut(struct arguments *arguments) { +int taosDumpOut(SDumpArguments *arguments) { TAOS_ROW row; char *temp = NULL; FILE *fp = NULL; @@ -602,7 +602,7 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { fprintf(fp, "%s\n\n", buffer); } -int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp) { +int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { TAOS_ROW row; int fd = -1; STableRecord tableRecord; @@ -660,7 +660,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp) { return 0; } -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct arguments *arguments, FILE *fp) { +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, SDumpArguments *arguments, FILE *fp) { char *pstr = NULL; pstr = buffer; int counter = 0; @@ -703,7 +703,7 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, struct argume fprintf(fp, "%s\n\n", buffer); } -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, struct arguments *arguments, +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, SDumpArguments *arguments, FILE *fp) { char *pstr = NULL; pstr = buffer; @@ -786,7 +786,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) { return count; } -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp) { +int32_t taosDumpTable(char *table, char *metric, SDumpArguments *arguments, FILE *fp) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -828,7 +828,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI return taosDumpTableData(fp, table, arguments); } -int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp) { +int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) { TAOS_ROW row = NULL; int fd = -1; STableRecord tableRecord; @@ -877,7 +877,7 @@ int32_t taosDumpMetric(char *metric, struct arguments *arguments, FILE *fp) { return 0; } -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { +int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { /* char temp[MAX_COMMAND_SIZE] = "\0"; */ int count = 0; char *pstr = NULL; @@ -987,7 +987,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments) { return 0; } -int taosCheckParam(struct arguments *arguments) { +int taosCheckParam(SDumpArguments *arguments) { if (arguments->all_databases && arguments->databases) { fprintf(stderr, "conflict option --all-databases and --databases\n"); return -1; @@ -1072,7 +1072,7 @@ void taosReplaceCtrlChar(char *str) { *pstr = '\0'; } -int taosDumpIn(struct arguments *arguments) { +int taosDumpIn(SDumpArguments *arguments) { assert(arguments->isDumpIn); int tsize = 0; diff --git a/src/mnode/inc/mgmtAcct.h b/src/mnode/inc/mgmtAcct.h index 0c0bb8d05c..136c28f2ae 100644 --- a/src/mnode/inc/mgmtAcct.h +++ b/src/mnode/inc/mgmtAcct.h @@ -25,7 +25,7 @@ extern "C" { int32_t mgmtInitAccts(); void mgmtCleanUpAccts(); void * mgmtGetAcct(char *acctName); -void * mgmtGetNextAcct(void *pNode, SAcctObj **pAcct); +void * mgmtGetNextAcct(void *pIter, SAcctObj **pAcct); void mgmtIncAcctRef(SAcctObj *pAcct); void mgmtDecAcctRef(SAcctObj *pAcct); void mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb); diff --git a/src/mnode/inc/mgmtDb.h b/src/mnode/inc/mgmtDb.h index b00a2bdf3d..4ad46eee7b 100644 --- a/src/mnode/inc/mgmtDb.h +++ b/src/mnode/inc/mgmtDb.h @@ -32,7 +32,7 @@ int32_t mgmtInitDbs(); void mgmtCleanUpDbs(); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByTableId(char *db); -void * mgmtGetNextDb(void *pNode, SDbObj **pDb); +void * mgmtGetNextDb(void *pIter, SDbObj **pDb); void mgmtIncDbRef(SDbObj *pDb); void mgmtDecDbRef(SDbObj *pDb); bool mgmtCheckIsMonitorDB(char *db, char *monitordb); diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 3ac2efb83b..7c1fd6c5ab 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -68,7 +68,7 @@ typedef struct SMnodeObj { // todo use dynamic length string typedef struct { - char tableId[TSDB_TABLE_ID_LEN + 1]; + char *tableId; int8_t type; } STableObj; @@ -77,6 +77,7 @@ typedef struct SSuperTableObj { uint64_t uid; int64_t createdTime; int32_t sversion; + int32_t tversion; int32_t numOfColumns; int32_t numOfTags; int8_t reserved[15]; @@ -223,7 +224,7 @@ typedef struct SAcctObj { typedef struct { int8_t type; char db[TSDB_DB_NAME_LEN + 1]; - void * pNode; + void * pIter; int16_t numOfColumns; int32_t rowSize; int32_t numOfRows; diff --git a/src/mnode/inc/mgmtDnode.h b/src/mnode/inc/mgmtDnode.h index 1d7116c6c0..c9e062255b 100644 --- a/src/mnode/inc/mgmtDnode.h +++ b/src/mnode/inc/mgmtDnode.h @@ -34,7 +34,7 @@ char* mgmtGetDnodeStatusStr(int32_t dnodeStatus); void mgmtMonitorDnodeModule(); int32_t mgmtGetDnodesNum(); -void * mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode); +void * mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode); void mgmtIncDnodeRef(SDnodeObj *pDnode); void mgmtDecDnodeRef(SDnodeObj *pDnode); void * mgmtGetDnode(int32_t dnodeId); diff --git a/src/mnode/inc/mgmtMnode.h b/src/mnode/inc/mgmtMnode.h index cb1d009c8c..b9a135346d 100644 --- a/src/mnode/inc/mgmtMnode.h +++ b/src/mnode/inc/mgmtMnode.h @@ -37,7 +37,7 @@ void mgmtDropMnodeLocal(int32_t dnodeId); void * mgmtGetMnode(int32_t mnodeId); int32_t mgmtGetMnodesNum(); -void * mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode); +void * mgmtGetNextMnode(void *pIter, struct SMnodeObj **pMnode); void mgmtIncMnodeRef(struct SMnodeObj *pMnode); void mgmtDecMnodeRef(struct SMnodeObj *pMnode); diff --git a/src/mnode/inc/mgmtSdb.h b/src/mnode/inc/mgmtSdb.h index 2b96eb4005..975206d52e 100644 --- a/src/mnode/inc/mgmtSdb.h +++ b/src/mnode/inc/mgmtSdb.h @@ -35,7 +35,8 @@ typedef enum { typedef enum { SDB_KEY_STRING, SDB_KEY_INT, - SDB_KEY_AUTO + SDB_KEY_AUTO, + SDB_KEY_VAR_STRING, } ESdbKey; typedef enum { @@ -80,7 +81,8 @@ int32_t sdbDeleteRow(SSdbOper *pOper); int32_t sdbUpdateRow(SSdbOper *pOper); void *sdbGetRow(void *handle, void *key); -void *sdbFetchRow(void *handle, void *pNode, void **ppRow); +void *sdbFetchRow(void *handle, void *pIter, void **ppRow); +void sdbFreeIter(void *pIter); void sdbIncRef(void *thandle, void *pRow); void sdbDecRef(void *thandle, void *pRow); int64_t sdbGetNumOfRows(void *handle); diff --git a/src/mnode/inc/mgmtTable.h b/src/mnode/inc/mgmtTable.h index 9c4aa4a2a5..84220c9cf8 100644 --- a/src/mnode/inc/mgmtTable.h +++ b/src/mnode/inc/mgmtTable.h @@ -27,8 +27,8 @@ void mgmtCleanUpTables(); void * mgmtGetTable(char *tableId); void mgmtIncTableRef(void *pTable); void mgmtDecTableRef(void *pTable); -void * mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable); -void * mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable); +void * mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable); +void * mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable); void mgmtDropAllChildTables(SDbObj *pDropDb); void mgmtDropAllSuperTables(SDbObj *pDropDb); diff --git a/src/mnode/inc/mgmtUser.h b/src/mnode/inc/mgmtUser.h index 0a21b1f704..2edd71f3e7 100644 --- a/src/mnode/inc/mgmtUser.h +++ b/src/mnode/inc/mgmtUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mgmtInitUsers(); void mgmtCleanUpUsers(); SUserObj *mgmtGetUser(char *name); -void * mgmtGetNextUser(void *pNode, SUserObj **pUser); +void * mgmtGetNextUser(void *pIter, SUserObj **pUser); void mgmtIncUserRef(SUserObj *pUser); void mgmtDecUserRef(SUserObj *pUser); SUserObj *mgmtGetUserFromConn(void *pConn); diff --git a/src/mnode/inc/mgmtVgroup.h b/src/mnode/inc/mgmtVgroup.h index 21a2c9b896..7acf7112a4 100644 --- a/src/mnode/inc/mgmtVgroup.h +++ b/src/mnode/inc/mgmtVgroup.h @@ -35,7 +35,7 @@ void mgmtDecVgroupRef(SVgObj *pVgroup); void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg); void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode); -void * mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup); +void * mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup); void mgmtUpdateVgroup(SVgObj *pVgroup); void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload); diff --git a/src/mnode/src/mgmtAcct.c b/src/mnode/src/mgmtAcct.c index 1cacec96ee..27beff8944 100644 --- a/src/mnode/src/mgmtAcct.c +++ b/src/mnode/src/mgmtAcct.c @@ -126,8 +126,8 @@ void *mgmtGetAcct(char *name) { return sdbGetRow(tsAcctSdb, name); } -void *mgmtGetNextAcct(void *pNode, SAcctObj **pAcct) { - return sdbFetchRow(tsAcctSdb, pNode, (void **)pAcct); +void *mgmtGetNextAcct(void *pIter, SAcctObj **pAcct) { + return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); } void mgmtIncAcctRef(SAcctObj *pAcct) { diff --git a/src/mnode/src/mgmtBalance.c b/src/mnode/src/mgmtBalance.c index ff5b4008bb..f8410a207e 100644 --- a/src/mnode/src/mgmtBalance.c +++ b/src/mnode/src/mgmtBalance.c @@ -22,6 +22,7 @@ #include "mgmtInt.h" #include "mgmtMnode.h" #include "mgmtDnode.h" +#include "mgmtSdb.h" #include "mgmtVgroup.h" #ifndef _SYNC @@ -33,13 +34,13 @@ void balanceUpdateMgmt() {} void balanceReset() {} int32_t balanceAllocVnodes(SVgObj *pVgroup) { - void * pNode = NULL; + void * pIter = NULL; SDnodeObj *pDnode = NULL; SDnodeObj *pSelDnode = NULL; float vnodeUsage = 1000.0; while (1) { - pNode = mgmtGetNextDnode(pNode, &pDnode); + pIter = mgmtGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) { @@ -55,6 +56,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { mgmtDecDnodeRef(pDnode); } + sdbFreeIter(pIter); + if (pSelDnode == NULL) { mError("failed to alloc vnode to vgroup"); return TSDB_CODE_NO_ENOUGH_DNODES; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 2f17df92fa..3f43d4bf89 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -156,8 +156,8 @@ int32_t mgmtInitDbs() { return 0; } -void *mgmtGetNextDb(void *pNode, SDbObj **pDb) { - return sdbFetchRow(tsDbSdb, pNode, (void **)pDb); +void *mgmtGetNextDb(void *pIter, SDbObj **pDb) { + return sdbFetchRow(tsDbSdb, pIter, (void **)pDb); } SDbObj *mgmtGetDb(char *db) { @@ -583,7 +583,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (pUser == NULL) return 0; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb); + pShow->pIter = mgmtGetNextDb(pShow->pIter, &pDb); if (pDb == NULL) break; cols = 0; @@ -865,14 +865,15 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { } } - void *pNode = NULL; + void *pIter = NULL; while (1) { SVgObj *pVgroup = NULL; - pNode = mgmtGetNextVgroup(pNode, &pVgroup); + pIter = mgmtGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; mgmtSendCreateVgroupMsg(pVgroup, NULL); mgmtDecVgroupRef(pVgroup); } + sdbFreeIter(pIter); if (oldReplica != pDb->cfg.replications) { balanceNotify(); @@ -983,12 +984,12 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) { void mgmtDropAllDbs(SAcctObj *pAcct) { int32_t numOfDbs = 0; SDbObj *pDb = NULL; - void * pNode = NULL; + void * pIter = NULL; mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user); while (1) { - pNode = mgmtGetNextDb(pNode, &pDb); + pIter = mgmtGetNextDb(pIter, &pDb); if (pDb == NULL) break; if (pDb->pAcct == pAcct) { @@ -1005,5 +1006,7 @@ void mgmtDropAllDbs(SAcctObj *pAcct) { mgmtDecDbRef(pDb); } + sdbFreeIter(pIter); + mPrint("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index d60dae32ae..c3ae8b5ab1 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -74,6 +74,7 @@ static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) { SDnodeObj *pDnode = pOper->pObj; #ifndef _SYNC + //TODO: drop dnode local mgmtDropAllDnodeVgroups(pDnode); #endif mgmtDropMnodeLocal(pDnode->dnodeId); @@ -170,8 +171,8 @@ void mgmtCleanupDnodes() { sdbCloseTable(tsDnodeSdb); } -void *mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) { - return sdbFetchRow(tsDnodeSdb, pNode, (void **)pDnode); +void *mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode) { + return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); } int32_t mgmtGetDnodesNum() { @@ -184,17 +185,20 @@ void *mgmtGetDnode(int32_t dnodeId) { void *mgmtGetDnodeByEp(char *ep) { SDnodeObj *pDnode = NULL; - void * pNode = NULL; + void * pIter = NULL; while (1) { - pNode = mgmtGetNextDnode(pNode, &pDnode); + pIter = mgmtGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (strcmp(ep, pDnode->dnodeEp) == 0) { + sdbFreeIter(pIter); return pDnode; } mgmtDecDnodeRef(pDnode); } + sdbFreeIter(pIter); + return NULL; } @@ -530,7 +534,7 @@ static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->numOfRows = mgmtGetDnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = NULL; + pShow->pIter = NULL; mgmtDecUserRef(pUser); @@ -544,7 +548,7 @@ static int32_t mgmtRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, voi char *pWrite; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextDnode(pShow->pNode, &pDnode); + pShow->pIter = mgmtGetNextDnode(pShow->pIter, &pDnode); if (pDnode == NULL) break; cols = 0; @@ -636,7 +640,7 @@ static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->numOfRows = mgmtGetDnodesNum() * TSDB_MOD_MAX; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = NULL; + pShow->pIter = NULL; mgmtDecUserRef(pUser); return 0; @@ -648,7 +652,7 @@ int32_t mgmtRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pCo while (numOfRows < rows) { SDnodeObj *pDnode = NULL; - pShow->pNode = mgmtGetNextDnode(pShow->pNode, (SDnodeObj **)&pDnode); + pShow->pIter = mgmtGetNextDnode(pShow->pIter, (SDnodeObj **)&pDnode); if (pDnode == NULL) break; for (int32_t moduleType = 0; moduleType < TSDB_MOD_MAX; ++moduleType) { @@ -738,7 +742,7 @@ static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = NULL; + pShow->pIter = NULL; mgmtDecUserRef(pUser); return 0; @@ -821,7 +825,8 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo if (pShow->payloadLen > 0 ) { pDnode = mgmtGetDnodeByEp(pShow->payload); } else { - mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode); + void *pIter = mgmtGetNextDnode(NULL, (SDnodeObj **)&pDnode); + sdbFreeIter(pIter); } if (pDnode != NULL) { @@ -830,7 +835,7 @@ static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = pDnode; + pShow->pIter = pDnode; mgmtDecUserRef(pUser); return 0; @@ -844,12 +849,12 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi if (0 == rows) return 0; - pDnode = (SDnodeObj *)(pShow->pNode); + pDnode = (SDnodeObj *)(pShow->pIter); if (pDnode != NULL) { - void *pNode = NULL; + void *pIter = NULL; SVgObj *pVgroup; while (1) { - pNode = mgmtGetNextVgroup(pNode, &pVgroup); + pIter = mgmtGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { @@ -869,6 +874,7 @@ static int32_t mgmtRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, voi mgmtDecVgroupRef(pVgroup); } + sdbFreeIter(pIter); } else { numOfRows = 0; } diff --git a/src/mnode/src/mgmtMnode.c b/src/mnode/src/mgmtMnode.c index a78e5574c0..d9ddd465f6 100644 --- a/src/mnode/src/mgmtMnode.c +++ b/src/mnode/src/mgmtMnode.c @@ -95,11 +95,12 @@ static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) { static int32_t mgmtMnodeActionRestored() { if (mgmtGetMnodesNum() == 1) { SMnodeObj *pMnode = NULL; - mgmtGetNextMnode(NULL, &pMnode); + void *pIter = mgmtGetNextMnode(NULL, &pMnode); if (pMnode != NULL) { pMnode->role = TAOS_SYNC_ROLE_MASTER; mgmtDecMnodeRef(pMnode); } + sdbFreeIter(pIter); } return TSDB_CODE_SUCCESS; } @@ -157,8 +158,8 @@ void mgmtDecMnodeRef(SMnodeObj *pMnode) { sdbDecRef(tsMnodeSdb, pMnode); } -void *mgmtGetNextMnode(void *pNode, SMnodeObj **pMnode) { - return sdbFetchRow(tsMnodeSdb, pNode, (void **)pMnode); +void *mgmtGetNextMnode(void *pIter, SMnodeObj **pMnode) { + return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); } char *mgmtGetMnodeRoleStr(int32_t role) { @@ -177,10 +178,10 @@ char *mgmtGetMnodeRoleStr(int32_t role) { } void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) { - void *pNode = NULL; + void *pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; - pNode = mgmtGetNextMnode(pNode, &pMnode); + pIter = mgmtGetNextMnode(pIter, &pMnode); if (pMnode == NULL) break; strcpy(ipSet->fqdn[ipSet->numOfIps], pMnode->pDnode->dnodeFqdn); @@ -194,6 +195,7 @@ void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) { mgmtDecMnodeRef(pMnode); } + sdbFreeIter(pIter); } void mgmtGetMnodeInfos(void *param) { @@ -201,10 +203,10 @@ void mgmtGetMnodeInfos(void *param) { mnodes->inUse = 0; int32_t index = 0; - void *pNode = NULL; + void *pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; - pNode = mgmtGetNextMnode(pNode, &pMnode); + pIter = mgmtGetNextMnode(pIter, &pMnode); if (pMnode == NULL) break; mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId); @@ -216,6 +218,7 @@ void mgmtGetMnodeInfos(void *param) { index++; mgmtDecMnodeRef(pMnode); } + sdbFreeIter(pIter); mnodes->nodeNum = index; } @@ -317,7 +320,7 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo pShow->numOfRows = mgmtGetMnodesNum(); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pNode = NULL; + pShow->pIter = NULL; mgmtDecUserRef(pUser); return 0; @@ -330,7 +333,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi char *pWrite; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextMnode(pShow->pNode, &pMnode); + pShow->pIter = mgmtGetNextMnode(pShow->pIter, &pMnode); if (pMnode == NULL) break; cols = 0; diff --git a/src/mnode/src/mgmtProfile.c b/src/mnode/src/mgmtProfile.c index 64ac8ed710..6667bff052 100644 --- a/src/mnode/src/mgmtProfile.c +++ b/src/mnode/src/mgmtProfile.c @@ -140,7 +140,7 @@ int32_t mgmtGetQueries(SShowObj *pShow, void *pConn) { // // // sorting based on useconds // -// pShow->pNode = pQueryShow; +// pShow->pIter = pQueryShow; return 0; } @@ -187,7 +187,7 @@ int32_t mgmtGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; pShow->numOfRows = 1000000; - pShow->pNode = NULL; + pShow->pIter = NULL; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; mgmtGetQueries(pShow, pConn); @@ -252,7 +252,7 @@ int32_t mgmtRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pCo char *pWrite; int32_t cols = 0; - SQueryShow *pQueryShow = (SQueryShow *)pShow->pNode; + SQueryShow *pQueryShow = (SQueryShow *)pShow->pIter; if (rows > pQueryShow->numOfQueries - pQueryShow->index) rows = pQueryShow->numOfQueries - pQueryShow->index; @@ -339,7 +339,7 @@ int32_t mgmtGetStreams(SShowObj *pShow, void *pConn) { // // // sorting based on useconds // -// pShow->pNode = pStreamShow; +// pShow->pIter = pStreamShow; return 0; } @@ -397,7 +397,7 @@ int32_t mgmtGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { for (int32_t i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; pShow->numOfRows = 1000000; - pShow->pNode = NULL; + pShow->pIter = NULL; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; mgmtGetStreams(pShow, pConn); @@ -409,7 +409,7 @@ int32_t mgmtRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pCo char *pWrite; int32_t cols = 0; - SStreamShow *pStreamShow = (SStreamShow *)pShow->pNode; + SStreamShow *pStreamShow = (SStreamShow *)pShow->pIter; if (rows > pStreamShow->numOfStreams - pStreamShow->index) rows = pStreamShow->numOfStreams - pStreamShow->index; @@ -592,7 +592,7 @@ int mgmtGetConns(SShowObj *pShow, void *pConn) { // // // sorting based on useconds // - // pShow->pNode = pConnShow; + // pShow->pIter = pConnShow; return 0; } @@ -627,7 +627,7 @@ int32_t mgmtGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { for (int i = 1; i < cols; ++i) pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; pShow->numOfRows = 1000000; - pShow->pNode = NULL; + pShow->pIter = NULL; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; mgmtGetConns(pShow, pConn); @@ -639,7 +639,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn char *pWrite; int32_t cols = 0; - SConnShow *pConnShow = (SConnShow *)pShow->pNode; + SConnShow *pConnShow = (SConnShow *)pShow->pIter; if (rows > pConnShow->numOfConns - pConnShow->index) rows = pConnShow->numOfConns - pConnShow->index; diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index c25f4457a9..087c84effd 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" +#include "hash.h" #include "trpc.h" #include "tutil.h" #include "tbalance.h" @@ -23,8 +24,6 @@ #include "twal.h" #include "tsync.h" #include "tglobal.h" -#include "hashint.h" -#include "hashstr.h" #include "dnode.h" #include "mgmtDef.h" #include "mgmtInt.h" @@ -83,12 +82,6 @@ typedef struct { } SSdbRow; static SSdbObject tsSdbObj = {0}; -static void *(*sdbInitIndexFp[])(int32_t maxRows, int32_t dataSize) = {sdbOpenStrHash, sdbOpenIntHash, sdbOpenIntHash}; -static void *(*sdbAddIndexFp[])(void *handle, void *key, void *data) = {sdbAddStrHash, sdbAddIntHash, sdbAddIntHash}; -static void (*sdbDeleteIndexFp[])(void *handle, void *key) = {sdbDeleteStrHash, sdbDeleteIntHash, sdbDeleteIntHash}; -static void *(*sdbGetIndexFp[])(void *handle, void *key) = {sdbGetStrHashData, sdbGetIntHashData, sdbGetIntHashData}; -static void (*sdbCleanUpIndexFp[])(void *handle) = {sdbCloseStrHash, sdbCloseIntHash, sdbCloseIntHash}; -static void *(*sdbFetchRowFp[])(void *handle, void *ptr, void **ppRow) = {sdbFetchStrHashData, sdbFetchIntHashData, sdbFetchIntHashData}; static int sdbWrite(void *param, void *data, int type); int32_t sdbGetId(void *handle) { @@ -111,6 +104,14 @@ bool sdbIsServing() { return tsSdbObj.status == SDB_STATUS_SERVING; } +static void *sdbGetObjKey(SSdbTable *pTable, void *key) { + if (pTable->keyType == SDB_KEY_VAR_STRING) { + return *(char **)key; + } + + return key; +} + static char *sdbGetActionStr(int32_t action) { switch (action) { case SDB_ACTION_INSERT: @@ -123,20 +124,25 @@ static char *sdbGetActionStr(int32_t action) { return "invalid"; } -static char *sdbGetkeyStr(SSdbTable *pTable, void *row) { +static char *sdbGetKeyStr(SSdbTable *pTable, void *key) { static char str[16]; switch (pTable->keyType) { case SDB_KEY_STRING: - return (char *)row; + case SDB_KEY_VAR_STRING: + return (char *)key; case SDB_KEY_INT: case SDB_KEY_AUTO: - sprintf(str, "%d", *(int32_t *)row); + sprintf(str, "%d", *(int32_t *)key); return str; default: return "invalid"; } } +static char *sdbGetKeyStrFromObj(SSdbTable *pTable, void *key) { + return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key)); +} + static void *sdbGetTableFromId(int32_t tableId) { return tsSdbObj.tableList[tableId]; } @@ -192,7 +198,7 @@ void sdbUpdateMnodeRoles() { } } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) { +static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } @@ -244,10 +250,10 @@ void sdbUpdateSync() { } if (index == 0) { - void *pNode = NULL; + void *pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; - pNode = mgmtGetNextMnode(pNode, &pMnode); + pIter = mgmtGetNextMnode(pIter, &pMnode); if (pMnode == NULL) break; syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId; @@ -257,6 +263,7 @@ void sdbUpdateSync() { mgmtDecMnodeRef(pMnode); } + sdbFreeIter(pIter); } syncCfg.replica = index; @@ -338,46 +345,48 @@ void sdbCleanUp() { pthread_mutex_destroy(&tsSdbObj.mutex); } -void sdbIncRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); - atomic_add_fetch_32(pRefCount, 1); - if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { - sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), - *pRefCount); - } +void sdbIncRef(void *handle, void *pObj) { + if (pObj == NULL) return; + + SSdbTable *pTable = handle; + int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + atomic_add_fetch_32(pRefCount, 1); + if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { + sdbTrace("table:%s, add ref to record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); } } -void sdbDecRef(void *handle, void *pRow) { - if (pRow) { - SSdbTable *pTable = handle; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); - int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { - sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), - *pRefCount); - } - int8_t *updateEnd = pRow + pTable->refCountPos - 1; - if (refCount <= 0 && *updateEnd) { - sdbTrace("table:%s, record:%s:%s:%d is destroyed", pTable->tableName, pTable->tableName, - sdbGetkeyStr(pTable, pRow), *pRefCount); - SSdbOper oper = {.pObj = pRow}; - (*pTable->destroyFp)(&oper); - } +void sdbDecRef(void *handle, void *pObj) { + if (pObj == NULL) return; + + SSdbTable *pTable = handle; + int32_t * pRefCount = (int32_t *)(pObj + pTable->refCountPos); + int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); + if (0 && (pTable->tableId == SDB_TABLE_MNODE || pTable->tableId == SDB_TABLE_DNODE)) { + sdbTrace("table:%s, def ref of record:%s:%d", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); + } + + int8_t *updateEnd = pObj + pTable->refCountPos - 1; + if (refCount <= 0 && *updateEnd) { + sdbTrace("table:%s, record:%s:%d is destroyed", pTable->tableName, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); + SSdbOper oper = {.pObj = pObj}; + (*pTable->destroyFp)(&oper); } } -static SSdbRow *sdbGetRowMeta(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - SSdbRow * pMeta; +static SSdbRow *sdbGetRowMeta(SSdbTable *pTable, void *key) { + if (pTable == NULL) return NULL; - if (handle == NULL) return NULL; + int32_t keySize = sizeof(int32_t); + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); + } + + return taosHashGet(pTable->iHandle, key, keySize); +} - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); - - return pMeta; +static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) { + return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key)); } void *sdbGetRow(void *handle, void *key) { @@ -387,24 +396,41 @@ void *sdbGetRow(void *handle, void *key) { if (handle == NULL) return NULL; pthread_mutex_lock(&pTable->mutex); - pMeta = (*sdbGetIndexFp[pTable->keyType])(pTable->iHandle, key); + + int32_t keySize = sizeof(int32_t); + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); + } + pMeta = taosHashGet(pTable->iHandle, key, keySize); + if (pMeta) sdbIncRef(pTable, pMeta->row); pthread_mutex_unlock(&pTable->mutex); - if (pMeta == NULL) { - return NULL; - } + if (pMeta == NULL) return NULL; return pMeta->row; } +static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { + return sdbGetRow(pTable, sdbGetObjKey(pTable, key)); +} + static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { SSdbRow rowMeta; rowMeta.rowSize = pOper->rowSize; rowMeta.row = pOper->pObj; pthread_mutex_lock(&pTable->mutex); - (*sdbAddIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj, &rowMeta); + + void * key = sdbGetObjKey(pTable, pOper->pObj); + int32_t keySize = sizeof(int32_t); + + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); + } + + taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow)); + sdbIncRef(pTable, pOper->pObj); pTable->numOfRows++; @@ -417,7 +443,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, insert record:%s to hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->insertFp)(pOper); return TSDB_CODE_SUCCESS; @@ -427,12 +453,21 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { (*pTable->deleteFp)(pOper); pthread_mutex_lock(&pTable->mutex); - (*sdbDeleteIndexFp[pTable->keyType])(pTable->iHandle, pOper->pObj); + + void * key = sdbGetObjKey(pTable, pOper->pObj); + int32_t keySize = sizeof(int32_t); + + if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { + keySize = strlen((char *)key); + } + + taosHashRemove(pTable->iHandle, key, keySize); + pTable->numOfRows--; pthread_mutex_unlock(&pTable->mutex); sdbTrace("table:%s, delete record:%s from hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; *updateEnd = 1; @@ -443,7 +478,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) { sdbTrace("table:%s, update record:%s in hash, numOfRows:%d version:%" PRIu64, pTable->tableName, - sdbGetkeyStr(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); + sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, sdbGetVersion()); (*pTable->updateFp)(pOper); return TSDB_CODE_SUCCESS; @@ -474,7 +509,7 @@ static int sdbWrite(void *param, void *data, int type) { } else if (pHead->version != tsSdbObj.version + 1) { pthread_mutex_unlock(&tsSdbObj.mutex); sdbError("table:%s, failed to restore %s record:%s from wal, version:%" PRId64 " too large, sdb version:%" PRId64, - pTable->tableName, sdbGetActionStr(action), sdbGetkeyStr(pTable, pHead->cont), pHead->version, + pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, tsSdbObj.version); return TSDB_CODE_OTHERS; } else { @@ -526,8 +561,8 @@ int32_t sdbInsertRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - if (sdbGetRow(pTable, pOper->pObj)) { - sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetkeyStr(pTable, pOper->pObj)); + if (sdbGetRowFromObj(pTable, pOper->pObj)) { + sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj)); sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_ALREADY_THERE; } @@ -566,7 +601,7 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; @@ -576,25 +611,27 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { assert(pMetaRow != NULL); if (pOper->type == SDB_OPER_GLOBAL) { - int32_t rowSize = 0; + void * key = sdbGetObjKey(pTable, pOper->pObj); + int32_t keySize = 0; switch (pTable->keyType) { case SDB_KEY_STRING: - rowSize = strlen((char *)pOper->pObj) + 1; + case SDB_KEY_VAR_STRING: + keySize = strlen((char *)key) + 1; break; case SDB_KEY_INT: case SDB_KEY_AUTO: - rowSize = sizeof(uint64_t); + keySize = sizeof(uint32_t); break; default: return -1; } - int32_t size = sizeof(SWalHead) + rowSize; + int32_t size = sizeof(SWalHead) + keySize; SWalHead *pHead = taosAllocateQitem(size); pHead->version = 0; - pHead->len = rowSize; + pHead->len = keySize; pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE; - memcpy(pHead->cont, pOper->pObj, rowSize); + memcpy(pHead->cont, key, keySize); int32_t code = sdbWrite(pOper, pHead, pHead->msgType); taosFreeQitem(pHead); @@ -608,7 +645,7 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return -1; - SSdbRow *pMeta = sdbGetRowMeta(pTable, pOper->pObj); + SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); if (pMeta == NULL) { sdbTrace("table:%s, record is not there, delete failed", pTable->tableName); return -1; @@ -637,18 +674,35 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { SSdbTable *pTable = (SSdbTable *)handle; - SSdbRow * pMeta; - *ppRow = NULL; if (pTable == NULL) return NULL; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) return NULL; + SHashMutableIterator *pIter = pNode; + if (pIter == NULL) { + pIter = taosHashCreateIter(pTable->iHandle); + } + + if (!taosHashIterNext(pIter)) { + taosHashDestroyIter(pIter); + return NULL; + } + + SSdbRow *pMeta = taosHashIterGet(pIter); + if (pMeta == NULL) { + taosHashDestroyIter(pIter); + return NULL; + } *ppRow = pMeta->row; sdbIncRef(handle, pMeta->row); - return pNode; + return pIter; +} + +void sdbFreeIter(void *pIter) { + if (pIter != NULL) { + taosHashDestroyIter(pIter); + } } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -669,10 +723,12 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { pTable->decodeFp = pDesc->decodeFp; pTable->destroyFp = pDesc->destroyFp; pTable->restoredFp = pDesc->restoredFp; - - if (sdbInitIndexFp[pTable->keyType] != NULL) { - pTable->iHandle = (*sdbInitIndexFp[pTable->keyType])(pTable->maxRowSize, sizeof(SSdbRow)); + + _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + if (pTable->keyType == SDB_KEY_STRING) { + hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); } + pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true); pthread_mutex_init(&pTable->mutex, NULL); @@ -688,11 +744,10 @@ void sdbCloseTable(void *handle) { tsSdbObj.numOfTables--; tsSdbObj.tableList[pTable->tableId] = NULL; - void *pNode = NULL; - while (1) { - SSdbRow *pMeta; - pNode = (*sdbFetchRowFp[pTable->keyType])(pTable->iHandle, pNode, (void **)&pMeta); - if (pMeta == NULL) break; + SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); + while (taosHashIterNext(pIter)) { + SSdbRow *pMeta = taosHashIterGet(pIter); + if (pMeta == NULL) continue; SSdbOper oper = { .pObj = pMeta->row, @@ -702,9 +757,8 @@ void sdbCloseTable(void *handle) { (*pTable->destroyFp)(&oper); } - if (sdbCleanUpIndexFp[pTable->keyType]) { - (*sdbCleanUpIndexFp[pTable->keyType])(pTable->iHandle); - } + taosHashDestroyIter(pIter); + taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index cd74c166d1..d8bcf67242 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -41,7 +41,6 @@ typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); -//static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg); static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg); static void mgmtProcessShowMsg(SQueuedMsg *queuedMsg); @@ -49,6 +48,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *queuedMsg); static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg); static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg); static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg); +static void mgmtFreeShowObj(void *data); void *tsMgmtTmr; static void *tsMgmtTranQhandle = NULL; @@ -66,7 +66,7 @@ int32_t mgmtInitShell() { tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND"); tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT"); - tsQhandleCache = taosCacheInit(tsMgmtTmr, 10); + tsQhandleCache = taosCacheInitWithCb(tsMgmtTmr, 10, mgmtFreeShowObj); return 0; } @@ -124,6 +124,8 @@ void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg) { void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { + mTrace("%p, msg:%s will be processed", rpcMsg->ahandle, taosMsg[rpcMsg->msgType]); + if (rpcMsg->pCont == NULL) { mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_LEN); return; @@ -343,29 +345,6 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { rpcSendResponse(&rpcRsp); } -/* -static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - *spi = 1; - *encrypt = 0; - *ckey = 0; - - if (!sdbIsMaster()) { - *secret = 0; - return TSDB_CODE_NOT_READY; - } - - SUserObj *pUser = mgmtGetUser(user); - if (pUser == NULL) { - *secret = 0; - return TSDB_CODE_INVALID_USER; - } else { - memcpy(secret, pUser->pass, TSDB_KEY_LEN); - mgmtDecUserRef(pUser); - return TSDB_CODE_SUCCESS; - } -} -*/ - static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMConnectMsg *pConnectMsg = pMsg->pCont; @@ -500,10 +479,10 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) { bool mgmtCheckQhandle(uint64_t qhandle) { void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle); if (pSaved == (void *)qhandle) { - mTrace("qhandle:%p is retrived", qhandle); + mTrace("show:%p, is retrieved", qhandle); return true; } else { - mTrace("qhandle:%p is already freed", qhandle); + mTrace("show:%p, is already released", qhandle); return false; } } @@ -515,15 +494,21 @@ void* mgmtSaveQhandle(void *qhandle, int32_t size) { void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60); free(qhandle); - mTrace("qhandle:%p is saved", newQhandle); + mTrace("show:%p, is saved", newQhandle); return newQhandle; } return NULL; } +static void mgmtFreeShowObj(void *data) { + SShowObj *pShow = data; + sdbFreeIter(pShow->pIter); + mTrace("show:%p, is destroyed", pShow); +} + void mgmtFreeQhandle(void *qhandle, bool forceRemove) { - mTrace("qhandle:%p is freed", qhandle); + mTrace("show:%p, is released, force:%s", qhandle, forceRemove ? "true" : "false"); taosCacheRelease(tsQhandleCache, &qhandle, forceRemove); } diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 49311b0112..637201fdfb 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -84,6 +84,7 @@ static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); static void mgmtDestroyChildTable(SChildTableObj *pTable) { + tfree(pTable->info.tableId); tfree(pTable->schema); tfree(pTable->sql); tfree(pTable); @@ -180,6 +181,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { SChildTableObj *pNew = pOper->pObj; SChildTableObj *pTable = mgmtGetChildTable(pNew->info.tableId); if (pTable != pNew) { + void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; void *oldSchema = pTable->schema; memcpy(pTable, pNew, pOper->rowSize); @@ -188,6 +190,7 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { free(pNew); free(oldSql); free(oldSchema); + free(oldTableId); } mgmtDecTableRef(pTable); @@ -195,51 +198,66 @@ static int32_t mgmtChildTableActionUpdate(SSdbOper *pOper) { } static int32_t mgmtChildTableActionEncode(SSdbOper *pOper) { - const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; SChildTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); - if (pTable->info.type == TSDB_CHILD_TABLE) { - memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - pOper->rowSize = tsChildTableUpdateSize; - } else { + int32_t len = strlen(pTable->info.tableId); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + + memcpy(pOper->rowData, pTable->info.tableId, len); + memset(pOper->rowData + len, 0, 1); + len++; + + memcpy(pOper->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + len += tsChildTableUpdateSize; + + if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - if (maxRowSize < tsChildTableUpdateSize + schemaSize) { - return TSDB_CODE_INVALID_MSG_LEN; + memcpy(pOper->rowData + len, pTable->schema, schemaSize); + len += schemaSize; + + if (pTable->sqlLen != 0) { + memcpy(pOper->rowData + len, pTable->sql, pTable->sqlLen); + len += pTable->sqlLen; } - memcpy(pOper->rowData, pTable, tsChildTableUpdateSize); - memcpy(pOper->rowData + tsChildTableUpdateSize, pTable->schema, schemaSize); - memcpy(pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sql, pTable->sqlLen); - pOper->rowSize = tsChildTableUpdateSize + schemaSize + pTable->sqlLen; } + pOper->rowSize = len; + return TSDB_CODE_SUCCESS; } static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); - if (pTable == NULL) { - return TSDB_CODE_SERV_OUT_OF_MEMORY; - } + if (pTable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - memcpy(pTable, pOper->rowData, tsChildTableUpdateSize); + int32_t len = strlen(pOper->rowData); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + pTable->info.tableId = strdup(pOper->rowData); + len++; + + memcpy((char*)pTable + sizeof(char *), pOper->rowData + len, tsChildTableUpdateSize); + len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = (SSchema *)malloc(schemaSize); if (pTable->schema == NULL) { mgmtDestroyChildTable(pTable); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + return TSDB_CODE_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pOper->rowData + tsChildTableUpdateSize, schemaSize); + memcpy(pTable->schema, pOper->rowData + len, schemaSize); + len += schemaSize; - pTable->sql = (char *)malloc(pTable->sqlLen); - if (pTable->sql == NULL) { - mgmtDestroyChildTable(pTable); - return TSDB_CODE_SERV_OUT_OF_MEMORY; + if (pTable->sqlLen != 0) { + pTable->sql = malloc(pTable->sqlLen); + if (pTable->sql == NULL) { + mgmtDestroyChildTable(pTable); + return TSDB_CODE_SERV_OUT_OF_MEMORY; + } + memcpy(pTable->sql, pOper->rowData + len, pTable->sqlLen); } - memcpy(pTable->sql, pOper->rowData + tsChildTableUpdateSize + schemaSize, pTable->sqlLen); } pOper->pObj = pTable; @@ -247,25 +265,19 @@ static int32_t mgmtChildTableActionDecode(SSdbOper *pOper) { } static int32_t mgmtChildTableActionRestored() { - void *pNode = NULL; - void *pLastNode = NULL; + void *pIter = NULL; SChildTableObj *pTable = NULL; while (1) { - pLastNode = pNode; mgmtDecTableRef(pTable); - pNode = mgmtGetNextChildTable(pNode, &pTable); + pIter = mgmtGetNextChildTable(pIter, &pTable); if (pTable == NULL) break; SDbObj *pDb = mgmtGetDbByTableId(pTable->info.tableId); if (pDb == NULL) { mError("ctable:%s, failed to get db, discard it", pTable->info.tableId); - SSdbOper desc = {0}; - desc.type = SDB_OPER_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; + SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); - pNode = pLastNode; continue; } mgmtDecDbRef(pDb); @@ -274,12 +286,8 @@ static int32_t mgmtChildTableActionRestored() { if (pVgroup == NULL) { mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); pTable->vgId = 0; - SSdbOper desc = {0}; - desc.type = SDB_OPER_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; + SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); - pNode = pLastNode; continue; } mgmtDecVgroupRef(pVgroup); @@ -288,24 +296,16 @@ static int32_t mgmtChildTableActionRestored() { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); pTable->vgId = 0; - SSdbOper desc = {0}; - desc.type = SDB_OPER_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; + SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); - pNode = pLastNode; continue; } if (pVgroup->tableList == NULL) { mError("ctable:%s, vgId:%d tableList is null", pTable->info.tableId, pTable->vgId); pTable->vgId = 0; - SSdbOper desc = {0}; - desc.type = SDB_OPER_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; + SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); - pNode = pLastNode; continue; } @@ -314,32 +314,30 @@ static int32_t mgmtChildTableActionRestored() { if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; - SSdbOper desc = {0}; - desc.type = SDB_OPER_LOCAL; - desc.pObj = pTable; - desc.table = tsChildTableSdb; + SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); - pNode = pLastNode; continue; } mgmtDecTableRef(pSuperTable); } } + sdbFreeIter(pIter); + return 0; } static int32_t mgmtInitChildTables() { SChildTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = tsMaxTables, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_STRING, + .keyType = SDB_KEY_VAR_STRING, .insertFp = mgmtChildTableActionInsert, .deleteFp = mgmtChildTableActionDelete, .updateFp = mgmtChildTableActionUpdate, @@ -381,7 +379,7 @@ static void mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *p if (pStable->vgHash == NULL) return; SVgObj *pVgroup = mgmtGetVgroup(pCtable->vgId); - if (pVgroup != NULL) { + if (pVgroup == NULL) { taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); } mgmtDecVgroupRef(pVgroup); @@ -392,6 +390,7 @@ static void mgmtDestroySuperTable(SSuperTableObj *pStable) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; } + tfree(pStable->info.tableId); tfree(pStable->schema); tfree(pStable); } @@ -428,11 +427,13 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) { SSuperTableObj *pNew = pOper->pObj; SSuperTableObj *pTable = mgmtGetSuperTable(pNew->info.tableId); if (pTable != pNew) { + void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; memcpy(pTable, pNew, pOper->rowSize); pTable->schema = pNew->schema; free(pNew->vgHash); free(pNew); + free(oldTableId); free(oldSchema); } mgmtDecTableRef(pTable); @@ -440,40 +441,50 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) { } static int32_t mgmtSuperTableActionEncode(SSdbOper *pOper) { - const int32_t maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * TSDB_MAX_COLUMNS; - SSuperTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); + int32_t len = strlen(pStable->info.tableId); + if (len > TSDB_TABLE_ID_LEN) len = TSDB_CODE_INVALID_TABLE_ID; + + memcpy(pOper->rowData, pStable->info.tableId, len); + memset(pOper->rowData + len, 0, 1); + len++; + + memcpy(pOper->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + len += tsSuperTableUpdateSize; + int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); + memcpy(pOper->rowData + len, pStable->schema, schemaSize); + len += schemaSize; - if (maxRowSize < tsSuperTableUpdateSize + schemaSize) { - return TSDB_CODE_INVALID_MSG_LEN; - } - - memcpy(pOper->rowData, pStable, tsSuperTableUpdateSize); - memcpy(pOper->rowData + tsSuperTableUpdateSize, pStable->schema, schemaSize); - pOper->rowSize = tsSuperTableUpdateSize + schemaSize; + pOper->rowSize = len; return TSDB_CODE_SUCCESS; } static int32_t mgmtSuperTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY; - memcpy(pStable, pOper->rowData, tsSuperTableUpdateSize); + int32_t len = strlen(pOper->rowData); + if (len > TSDB_TABLE_ID_LEN) return TSDB_CODE_INVALID_TABLE_ID; + pStable->info.tableId = strdup(pOper->rowData); + len++; + + memcpy((char*)pStable + sizeof(char *), pOper->rowData + len, tsSuperTableUpdateSize); + len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); pStable->schema = malloc(schemaSize); if (pStable->schema == NULL) { mgmtDestroySuperTable(pStable); - return -1; + return TSDB_CODE_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pOper->rowData + tsSuperTableUpdateSize, schemaSize); + memcpy(pStable->schema, pOper->rowData + len, schemaSize); + pOper->pObj = pStable; return TSDB_CODE_SUCCESS; @@ -485,15 +496,15 @@ static int32_t mgmtSuperTableActionRestored() { static int32_t mgmtInitSuperTables() { SSuperTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_MAX_SUPER_TABLES, - .maxRowSize = tsSuperTableUpdateSize + sizeof(SSchema) * TSDB_MAX_COLUMNS, + .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, - .keyType = SDB_KEY_STRING, + .keyType = SDB_KEY_VAR_STRING, .insertFp = mgmtSuperTableActionInsert, .deleteFp = mgmtSuperTableActionDelete, .updateFp = mgmtSuperTableActionUpdate, @@ -560,17 +571,20 @@ static void *mgmtGetSuperTable(char *tableId) { static void *mgmtGetSuperTableByUid(uint64_t uid) { SSuperTableObj *pStable = NULL; - void * pNode = NULL; + void *pIter = NULL; while (1) { - pNode = mgmtGetNextSuperTable(pNode, &pStable); + pIter = mgmtGetNextSuperTable(pIter, &pStable); if (pStable == NULL) break; if (pStable->uid == uid) { + sdbFreeIter(pIter); return pStable; } mgmtDecTableRef(pStable); } + sdbFreeIter(pIter); + return NULL; } @@ -588,12 +602,12 @@ void *mgmtGetTable(char *tableId) { return NULL; } -void *mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable) { - return sdbFetchRow(tsChildTableSdb, pNode, (void **)pTable); +void *mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable) { + return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } -void *mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable) { - return sdbFetchRow(tsSuperTableSdb, pNode, (void **)pTable); +void *mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable) { + return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } void mgmtIncTableRef(void *p1) { @@ -737,18 +751,19 @@ static void mgmtProcessTableMetaMsg(SQueuedMsg *pMsg) { static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->pCont; - SSuperTableObj *pStable = (SSuperTableObj *)calloc(1, sizeof(SSuperTableObj)); + SSuperTableObj *pStable = calloc(1, sizeof(SSuperTableObj)); if (pStable == NULL) { mError("table:%s, failed to create, no enough memory", pCreate->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); return; } - strcpy(pStable->info.tableId, pCreate->tableId); + pStable->info.tableId = strdup(pCreate->tableId); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); pStable->uid = (((uint64_t) pStable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); pStable->sversion = 0; + pStable->tversion = 0; pStable->numOfColumns = htons(pCreate->numOfColumns); pStable->numOfTags = htons(pCreate->numOfTags); @@ -812,6 +827,7 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { dnodeSendMsgToDnode(&ipSet, &rpcMsg); mgmtDecVgroupRef(pVgroup); } + taosHashDestroyIter(pIter); mgmtDropAllChildTablesInStable(pStable); } @@ -867,7 +883,7 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i } pStable->numOfTags += ntags; - pStable->sversion++; + pStable->tversion++; SSdbOper oper = { .type = SDB_OPER_GLOBAL, @@ -894,7 +910,7 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1, sizeof(SSchema) * (pStable->numOfTags - col - 1)); pStable->numOfTags--; - pStable->sversion++; + pStable->tversion++; SSdbOper oper = { .type = SDB_OPER_GLOBAL, @@ -1121,7 +1137,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v while (numOfRows < rows) { mgmtDecTableRef(pTable); - pShow->pNode = mgmtGetNextSuperTable(pShow->pNode, &pTable); + pShow->pIter = mgmtGetNextSuperTable(pShow->pIter, &pTable); if (pTable == NULL) break; if (strncmp(pTable->info.tableId, prefix, prefixLen)) { continue; @@ -1171,8 +1187,7 @@ int32_t mgmtRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, v } void mgmtDropAllSuperTables(SDbObj *pDropDb) { - void *pNode = NULL; - void *pLastNode = NULL; + void * pIter= NULL; int32_t numOfTables = 0; int32_t dbNameLen = strlen(pDropDb->name); SSuperTableObj *pTable = NULL; @@ -1180,8 +1195,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name); while (1) { - pLastNode = pNode; - pNode = mgmtGetNextSuperTable(pNode, &pTable); + pIter = mgmtGetNextSuperTable(pIter, &pTable); if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { @@ -1191,20 +1205,23 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { .pObj = pTable, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfTables ++; } mgmtDecTableRef(pTable); } + sdbFreeIter(pIter); + mPrint("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; + assert(numOfCols <= TSDB_MAX_COLUMNS); + for (int32_t i = 0; i < numOfCols; ++i) { - strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN); + strncpy(pSchema->name, pTable->schema[i].name, TSDB_COL_NAME_LEN); pSchema->type = pTable->schema[i].type; pSchema->bytes = htons(pTable->schema[i].bytes); pSchema->colId = htons(pTable->schema[i].colId); @@ -1216,7 +1233,7 @@ static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTa static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); pMeta->uid = htobe64(pTable->uid); pMeta->sversion = htons(pTable->sversion); pMeta->precision = pMsg->pDb->cfg.precision; @@ -1288,6 +1305,8 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { mgmtDecVgroupRef(pVgroup); } + taosHashDestroyIter(pIter); + pVgroupInfo->numOfVgroups = htonl(vgSize); // one table is done, try the next table @@ -1371,7 +1390,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb } static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj *pVgroup, int32_t tid) { - SChildTableObj *pTable = (SChildTableObj *) calloc(1, sizeof(SChildTableObj)); + SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); if (pTable == NULL) { mError("table:%s, failed to alloc memory", pCreate->tableId); terrno = TSDB_CODE_SERV_OUT_OF_MEMORY; @@ -1384,10 +1403,10 @@ static SChildTableObj* mgmtDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgObj pTable->info.type = TSDB_NORMAL_TABLE; } - strcpy(pTable->info.tableId, pCreate->tableId); - pTable->createdTime = taosGetTimestampMs(); - pTable->sid = tid; - pTable->vgId = pVgroup->vgId; + pTable->info.tableId = strdup(pCreate->tableId); + pTable->createdTime = taosGetTimestampMs(); + pTable->sid = tid; + pTable->vgId = pVgroup->vgId; if (pTable->info.type == TSDB_CHILD_TABLE) { char *pTagData = (char *) pCreate->schema; // it is a tag key @@ -1473,15 +1492,15 @@ static void mgmtProcessCreateChildTableMsg(SQueuedMsg *pMsg) { return; } - int32_t sid = taosAllocateId(pVgroup->idPool); - if (sid <= 0) { - mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId); - mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb); - return; - } - if (pMsg->retry == 0) { if (pMsg->pTable == NULL) { + int32_t sid = taosAllocateId(pVgroup->idPool); + if (sid <= 0) { + mTrace("tables:%s, no enough sid in vgId:%d", pCreate->tableId, pVgroup->vgId); + mgmtCreateVgroup(mgmtCloneQueuedMsg(pMsg), pMsg->pDb); + return; + } + pMsg->pTable = (STableObj *)mgmtDoCreateChildTable(pCreate, pVgroup, sid); mgmtIncTableRef(pMsg->pTable); } @@ -1668,14 +1687,13 @@ static int32_t mgmtDoGetChildTableMeta(SQueuedMsg *pMsg, STableMetaMsg *pMeta) { pMeta->sid = htonl(pTable->sid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; - strncpy(pMeta->tableId, pTable->info.tableId, tListLen(pTable->info.tableId)); + strncpy(pMeta->tableId, pTable->info.tableId, strlen(pTable->info.tableId)); if (pTable->info.type == TSDB_CHILD_TABLE) { pMeta->sversion = htons(pTable->superTable->sversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; pMeta->numOfColumns = htons((int16_t)pTable->superTable->numOfColumns); pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable->superTable); - strncpy(pMeta->stableId, pTable->superTable->info.tableId, tListLen(pMeta->stableId)); } else { pMeta->sversion = htons(pTable->sversion); pMeta->numOfTags = 0; @@ -1731,7 +1749,7 @@ static void mgmtAutoCreateChildTable(SQueuedMsg *pMsg) { } static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { - STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS); + STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); if (pMeta == NULL) { mError("table:%s, failed to get table meta, no enough memory", pMsg->pTable->tableId); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY); @@ -1750,8 +1768,7 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg) { } void mgmtDropAllChildTables(SDbObj *pDropDb) { - void *pNode = NULL; - void *pLastNode = NULL; + void * pIter = NULL; int32_t numOfTables = 0; int32_t dbNameLen = strlen(pDropDb->name); SChildTableObj *pTable = NULL; @@ -1759,8 +1776,7 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name); while (1) { - pLastNode = pNode; - pNode = mgmtGetNextChildTable(pNode, &pTable); + pIter = mgmtGetNextChildTable(pIter, &pTable); if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { @@ -1770,26 +1786,25 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { .pObj = pTable, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfTables++; } mgmtDecTableRef(pTable); } + sdbFreeIter(pIter); + mPrint("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { - void *pNode = NULL; - void *pLastNode = NULL; + void * pIter = NULL; int32_t numOfTables = 0; SChildTableObj *pTable = NULL; mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables); while (1) { - pLastNode = pNode; - pNode = mgmtGetNextChildTable(pNode, &pTable); + pIter = mgmtGetNextChildTable(pIter, &pTable); if (pTable == NULL) break; if (pTable->superTable == pStable) { @@ -1799,13 +1814,14 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { .pObj = pTable, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfTables++; } mgmtDecTableRef(pTable); } + sdbFreeIter(pIter); + mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } @@ -1979,7 +1995,7 @@ static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { if (pMsg->pDb == NULL) continue; int availLen = totalMallocLen - pMultiMeta->contLen; - if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * TSDB_MAX_COLUMNS) { + if (availLen <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) { //TODO realloc //totalMallocLen *= 2; //pMultiMeta = rpcReMalloc(pMultiMeta, totalMallocLen); @@ -2076,7 +2092,7 @@ static int32_t mgmtRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, while (numOfRows < rows) { mgmtDecTableRef(pTable); - pShow->pNode = mgmtGetNextChildTable(pShow->pNode, &pTable); + pShow->pIter = mgmtGetNextChildTable(pShow->pIter, &pTable); if (pTable == NULL) break; // not belong to current db diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index 9630ab3d58..cc22ed2169 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -37,6 +37,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg); static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg); static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg); +static void mgmtProcessAuthMsg(SRpcMsg *rpcMsg); static int32_t mgmtUserActionDestroy(SSdbOper *pOper) { tfree(pOper->pObj); @@ -140,7 +141,8 @@ int32_t mgmtInitUsers() { mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mgmtProcessDropUserMsg); mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_USER, mgmtGetUserMeta); mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mgmtRetrieveUsers); - + dnodeAddServerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mgmtProcessAuthMsg); + mTrace("table:%s, hash is created", tableDesc.tableName); return 0; } @@ -153,8 +155,8 @@ SUserObj *mgmtGetUser(char *name) { return (SUserObj *)sdbGetRow(tsUserSdb, name); } -void *mgmtGetNextUser(void *pNode, SUserObj **pUser) { - return sdbFetchRow(tsUserSdb, pNode, (void **)pUser); +void *mgmtGetNextUser(void *pIter, SUserObj **pUser) { + return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); } void mgmtIncUserRef(SUserObj *pUser) { @@ -298,7 +300,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void char *pWrite; while (numOfRows < rows) { - pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser); + pShow->pIter = mgmtGetNextUser(pShow->pIter, &pUser); if (pUser == NULL) break; cols = 0; @@ -502,15 +504,13 @@ static void mgmtProcessDropUserMsg(SQueuedMsg *pMsg) { } void mgmtDropAllUsers(SAcctObj *pAcct) { - void * pNode = NULL; - void * pLastNode = NULL; + void * pIter = NULL; int32_t numOfUsers = 0; int32_t acctNameLen = strlen(pAcct->user); SUserObj *pUser = NULL; while (1) { - pLastNode = pNode; - pNode = mgmtGetNextUser(pNode, &pUser); + pIter = mgmtGetNextUser(pIter, &pUser); if (pUser == NULL) break; if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) { @@ -520,12 +520,50 @@ void mgmtDropAllUsers(SAcctObj *pAcct) { .pObj = pUser, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfUsers++; } mgmtDecUserRef(pUser); } + sdbFreeIter(pIter); + mTrace("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); } + +int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (!sdbIsMaster()) { + *secret = 0; + mTrace("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_NOT_READY)); + return TSDB_CODE_NOT_READY; + } + + SUserObj *pUser = mgmtGetUser(user); + if (pUser == NULL) { + *secret = 0; + mError("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_INVALID_USER)); + return TSDB_CODE_INVALID_USER; + } else { + *spi = 1; + *encrypt = 0; + *ckey = 0; + + memcpy(secret, pUser->pass, TSDB_KEY_LEN); + mgmtDecUserRef(pUser); + mTrace("user:%s, auth info is returned", user); + return TSDB_CODE_SUCCESS; + } +} + +static void mgmtProcessAuthMsg(SRpcMsg *rpcMsg) { + SRpcMsg rpcRsp = {.handle = rpcMsg->handle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; + + SDMAuthMsg *pAuthMsg = rpcMsg->pCont; + SDMAuthRsp *pAuthRsp = rpcMallocCont(sizeof(SDMAuthRsp)); + + rpcRsp.code = mgmtRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); + rpcRsp.pCont = pAuthRsp; + rpcRsp.contLen = sizeof(SDMAuthRsp); + + rpcSendResponse(&rpcRsp); +} diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index b87c6ad727..1416ed5f58 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -288,8 +288,8 @@ SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb) { return pDb->pHead; } -void *mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup) { - return sdbFetchRow(tsVgroupSdb, pNode, (void **)pVgroup); +void *mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup) { + return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); } void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) { @@ -429,10 +429,10 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { if (NULL == pTable) { pShow->numOfRows = pDb->numOfVgroups; - pShow->pNode = pDb->pHead; + pShow->pIter = pDb->pHead; } else { pShow->numOfRows = 1; - pShow->pNode = pVgroup; + pShow->pIter = pVgroup; } mgmtDecDbRef(pDb); @@ -457,9 +457,9 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo } while (numOfRows < rows) { - pVgroup = (SVgObj *) pShow->pNode; + pVgroup = (SVgObj *) pShow->pIter; if (pVgroup == NULL) break; - pShow->pNode = (void *) pVgroup->next; + pShow->pIter = (void *) pVgroup->next; cols = 0; @@ -666,7 +666,6 @@ static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) { } void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { - mTrace("vgId:%d, send drop vnode msg, ahandle:%p", vgId, ahandle); SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .handle = ahandle, @@ -682,6 +681,7 @@ static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { mTrace("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + mTrace("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle); mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); } } @@ -749,14 +749,12 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { } void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) { - void * pNode = NULL; - void * pLastNode = NULL; + void * pIter = NULL; SVgObj *pVgroup = NULL; int32_t numOfVgroups = 0; while (1) { - pLastNode = pNode; - pNode = mgmtGetNextVgroup(pNode, &pVgroup); + pIter = mgmtGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) { @@ -766,24 +764,23 @@ void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) { .pObj = pVgroup, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfVgroups++; continue; } mgmtDecVgroupRef(pVgroup); } + + sdbFreeIter(pIter); } void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) { - void *pNode = NULL; - void *pLastNode = NULL; + void * pIter = NULL; int32_t numOfVgroups = 0; SVgObj *pVgroup = NULL; mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name); while (1) { - pLastNode = pNode; - pNode = mgmtGetNextVgroup(pNode, &pVgroup); + pIter = mgmtGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; if (pVgroup->pDb == pDropDb) { @@ -793,7 +790,6 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) { .pObj = pVgroup, }; sdbDeleteRow(&oper); - pNode = pLastNode; numOfVgroups++; if (sendMsg) { @@ -804,5 +800,7 @@ void mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) { mgmtDecVgroupRef(pVgroup); } + sdbFreeIter(pIter); + mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index cab899753d..40fcf83431 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -253,6 +253,17 @@ void taosBlockSIGPIPE(); #define BUILDIN_CLZ(val) __builtin_clz(val) #define BUILDIN_CTZ(val) __builtin_ctz(val) +#undef threadlocal +#ifdef _ISOC11_SOURCE + #define threadlocal _Thread_local +#elif defined(__APPLE__) + #define threadlocal +#elif defined(__GNUC__) && !defined(threadlocal) + #define threadlocal __thread +#else + #define threadlocal +#endif + #ifdef __cplusplus } #endif diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c index 72da97287f..73cccd020a 100644 --- a/src/os/linux/src/linuxPlatform.c +++ b/src/os/linux/src/linuxPlatform.c @@ -157,7 +157,7 @@ void *taosProcessAlarmSignal(void *tharg) { void (*callback)(int) = tharg; static timer_t timerId; - struct sigevent sevent = {0}; + struct sigevent sevent = {{0}}; #ifdef _ALPINE sevent.sigev_notify = SIGEV_THREAD; diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 4eaff3660a..8323ef7f4e 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -19,7 +19,6 @@ #include "tglobal.h" #include "tsocket.h" #include "ttimer.h" -#include "shash.h" #include "http.h" #include "httpLog.h" #include "httpCode.h" diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 4b095aeb77..e80d6f26b7 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "shash.h" +#include "hash.h" #include "taos.h" #include "ttime.h" #include "ttimer.h" @@ -25,7 +25,6 @@ #include "httpHandle.h" #include "httpResp.h" - void httpAccessSession(HttpContext *pContext) { HttpServer *server = pContext->pThread->pServer; pthread_mutex_lock(&server->serverMutex); @@ -50,8 +49,11 @@ void httpCreateSession(HttpContext *pContext, void *taos) { session.taos = taos; session.expire = (int)taosGetTimestampSec() + server->sessionExpire; session.access = 1; - snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = (HttpSession *)taosAddStrHash(server->pSessionHash, session.id, (char *)(&session)); + int sessionIdLen = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + + taosHashPut(server->pSessionHash, session.id, sessionIdLen, (char *)(&session), sizeof(HttpSession)); + pContext->session = taosHashGet(server->pSessionHash, session.id, sessionIdLen); + if (pContext->session == NULL) { httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, httpMsg[HTTP_SESSION_FULL]); @@ -71,9 +73,9 @@ void httpFetchSessionImp(HttpContext *pContext) { pthread_mutex_lock(&server->serverMutex); char sessionId[HTTP_SESSION_ID_LEN]; - snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); + int sessonIdLen = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = (HttpSession *)taosGetStrHashData(server->pSessionHash, sessionId); + pContext->session = taosHashGet(server->pSessionHash, sessionId, sessonIdLen); if (pContext->session != NULL && pContext->session == pContext->session->signature) { pContext->session->access++; httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d", @@ -120,8 +122,7 @@ void httpRestoreSession(HttpContext *pContext) { pthread_mutex_unlock(&server->serverMutex); } -void httpResetSession(char *session) { - HttpSession *pSession = (HttpSession *)session; +void httpResetSession(HttpSession *pSession) { httpTrace("close session:%p:%p", pSession, pSession->taos); if (pSession->taos != NULL) { taos_close(pSession->taos); @@ -131,15 +132,20 @@ void httpResetSession(char *session) { } void httpRemoveAllSessions(HttpServer *pServer) { - if (pServer->pSessionHash != NULL) { - taosCleanUpStrHashWithFp(pServer->pSessionHash, httpResetSession); - pServer->pSessionHash = NULL; + SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash); + + while (taosHashIterNext(pIter)) { + HttpSession *pSession = taosHashIterGet(pIter); + if (pSession == NULL) continue; + httpResetSession(pSession); } + + taosHashDestroyIter(pIter); } bool httpInitAllSessions(HttpServer *pServer) { if (pServer->pSessionHash == NULL) { - pServer->pSessionHash = taosInitStrHash(100, sizeof(HttpSession), taosHashStringStep1); + pServer->pSessionHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); } if (pServer->pSessionHash == NULL) { httpError("http init session pool failed"); @@ -152,46 +158,41 @@ bool httpInitAllSessions(HttpServer *pServer) { return true; } -int httpSessionExpired(char *session) { - HttpSession *pSession = (HttpSession *)session; - time_t cur = taosGetTimestampSec(); +bool httpSessionExpired(HttpSession *pSession) { + time_t cur = taosGetTimestampSec(); if (pSession->taos != NULL) { if (pSession->expire > cur) { - return 0; // un-expired, so return false + return false; // un-expired, so return false } if (pSession->access > 0) { httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos, pSession->access); - return 0; // still used, so return false + return false; // still used, so return false } httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d", pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire); } - return 1; + return true; } -void httpRemoveExpireSessions(HttpServer *pServer) { - int expiredNum = 0; - do { +void httpRemoveExpireSessions(HttpServer *pServer) { + SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash); + + while (taosHashIterNext(pIter)) { + HttpSession *pSession = taosHashIterGet(pIter); + if (pSession == NULL) continue; + pthread_mutex_lock(&pServer->serverMutex); - - HttpSession *pSession = (HttpSession *)taosVisitStrHashWithFp(pServer->pSessionHash, httpSessionExpired); - if (pSession == NULL) { - pthread_mutex_unlock(&pServer->serverMutex); - break; + if (httpSessionExpired(pSession)) { + httpResetSession(pSession); + taosHashRemove(pServer->pSessionHash, pSession->id, strlen(pSession->id)); } - - httpResetSession((char *)pSession); - taosDeleteStrHashNode(pServer->pSessionHash, pSession->id, pSession); - pthread_mutex_unlock(&pServer->serverMutex); + } - if (++expiredNum > 10) { - break; - } - } while (true); + taosHashDestroyIter(pIter); } void httpProcessSessionExpire(void *handle, void *tmrId) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 8b95077632..e7a5344be5 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "shash.h" #include "taos.h" #include "tglobal.h" #include "tsocket.h" diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index fc5f771c69..77fc399272 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tmd5.h" -#include "shash.h" #include "taos.h" #include "http.h" #include "httpLog.h" diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index c854e0adc3..1a55c325d5 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" -#include "shash.h" #include "taosdef.h" #include "taosmsg.h" #include "tgHandle.h" diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 76b322d01b..ae8038e444 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -109,6 +109,11 @@ static void monitorStartSystemRetry() { } static void monitorInitConn(void *para, void *unused) { + if (dnodeGetDnodeId() <= 0) { + monitorStartSystemRetry(); + return; + } + monitorPrint("starting to initialize monitor service .."); tsMonitorConn.state = MONITOR_STATE_INITIALIZING; diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 9bc36413de..410b2ac9d2 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -50,11 +50,11 @@ typedef struct tQueryInfo { SSchema sch; // schema of tags char* q; __compar_fn_t compare; // filter function - void* param; // STSchema, + void* param; // STSchema } tQueryInfo; typedef struct SExprTraverseSupp { - __result_filter_fn_t fp; + __result_filter_fn_t nodeFilterFn; __do_filter_suppl_fn_t setupInfoFn; void * pExtInfo; } SExprTraverseSupp; diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qextbuffer.h index a73948fe3b..9389e5a5ab 100644 --- a/src/query/inc/qextbuffer.h +++ b/src/query/inc/qextbuffer.h @@ -120,12 +120,6 @@ typedef struct tExtMemBuffer { EXT_BUFFER_FLUSH_MODEL flushModel; } tExtMemBuffer; -//typedef struct tTagSchema { -// struct SSchema *pSchema; -// int32_t numOfCols; -// int32_t colOffset[]; -//} tTagSchema; - /** * * @param inMemSize diff --git a/src/query/inc/qsqltype.h b/src/query/inc/qsqltype.h deleted file mode 100644 index 08d30be925..0000000000 --- a/src/query/inc/qsqltype.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QSQLCMD_H -#define TDENGINE_QSQLCMD_H - -#ifdef __cplusplus -extern "C" { -#endif - -enum _sql_type { - TSDB_SQL_SELECT = 1, - TSDB_SQL_FETCH, - TSDB_SQL_INSERT, - - TSDB_SQL_MGMT, // the SQL below is for mgmt node - TSDB_SQL_CREATE_DB, - TSDB_SQL_CREATE_TABLE, - TSDB_SQL_DROP_DB, - TSDB_SQL_DROP_TABLE, - TSDB_SQL_CREATE_ACCT, - TSDB_SQL_CREATE_USER, // 10 - TSDB_SQL_DROP_ACCT, - TSDB_SQL_DROP_USER, - TSDB_SQL_ALTER_USER, - TSDB_SQL_ALTER_ACCT, - TSDB_SQL_ALTER_TABLE, - TSDB_SQL_ALTER_DB, - TSDB_SQL_CREATE_MNODE, - TSDB_SQL_DROP_MNODE, - TSDB_SQL_CREATE_DNODE, - TSDB_SQL_DROP_DNODE, // 20 - TSDB_SQL_CFG_DNODE, - TSDB_SQL_CFG_MNODE, - TSDB_SQL_SHOW, - TSDB_SQL_RETRIEVE, - TSDB_SQL_KILL_QUERY, - TSDB_SQL_KILL_STREAM, - TSDB_SQL_KILL_CONNECTION, - - TSDB_SQL_READ, // SQL below is for read operation - TSDB_SQL_CONNECT, - TSDB_SQL_USE_DB, // 30 - TSDB_SQL_META, - TSDB_SQL_STABLEVGROUP, - TSDB_SQL_MULTI_META, - TSDB_SQL_HB, - - TSDB_SQL_LOCAL, // SQL below for client local - TSDB_SQL_DESCRIBE_TABLE, - TSDB_SQL_RETRIEVE_LOCALMERGE, - TSDB_SQL_METRIC_JOIN_RETRIEVE, - - /* - * build empty result instead of accessing dnode to fetch result - * reset the client cache - */ - TSDB_SQL_RETRIEVE_EMPTY_RESULT, - - TSDB_SQL_RESET_CACHE, // 40 - TSDB_SQL_SERV_STATUS, - TSDB_SQL_CURRENT_DB, - TSDB_SQL_SERV_VERSION, - TSDB_SQL_CLI_VERSION, - TSDB_SQL_CURRENT_USER, - TSDB_SQL_CFG_LOCAL, - - TSDB_SQL_MAX // 47 -}; - - -// create table operation type -enum TSQL_TYPE { - TSQL_CREATE_TABLE = 0x1, - TSQL_CREATE_STABLE = 0x2, - TSQL_CREATE_TABLE_FROM_STABLE = 0x3, - TSQL_CREATE_STREAM = 0x4, -}; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QSQLCMD_H diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 2088e5a49d..991b3b73f7 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -182,6 +182,7 @@ typedef struct SQInfo { SQueryRuntimeEnv runtimeEnv; int32_t groupIndex; int32_t offset; // offset in group result set of subgroup, todo refactor + SArray* arrTableIdInfo; T_REF_DECLARE() /* diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 43bdf57ed7..f727acb720 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -760,7 +760,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); - return param->fp(pItem, pExpr->_node.info); + return param->nodeFilterFn(pItem, pExpr->_node.info); } /** @@ -801,33 +801,33 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki tSkipListDestroyIter(iter); } -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { bool addToResult = false; SSkipListNode *pNode = tSkipListIterGet(iter); - char* pTable = SL_GET_NODE_DATA(pNode); - - //todo refactor: - tstr* name = ((STableIndexElem*) pTable)->pTable->name; -// char* name = NULL; -// tsdbGetTableName(tsdb, pTable, &name); + char * pData = SL_GET_NODE_DATA(pNode); + + // todo refactor: + tstr *name = ((STableIndexElem *)pData)->pTable->name; + // char* name = NULL; +// tsdbGetTableName(pQueryInfo->, pTable, &name); // todo speed up by using hash if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { addToResult = !pQueryInfo->compare(name, pQueryInfo->q); } } else { - // TODO: other columns + addToResult = filterFp(pNode, pQueryInfo); } if (addToResult) { - taosArrayPush(result, pTable); + taosArrayPush(res, pData); } } @@ -851,7 +851,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { - tArrayTraverse(pExpr, param->fp, result); + tArrayTraverse(pExpr, param->nodeFilterFn, result); return; } @@ -859,7 +859,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); } return; diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index b8d1189129..9589be86e4 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -820,7 +820,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt = *pDB; pInfo->pDCLInfo->dbOpt.dbname = *pToken; - pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists->z != NULL); + pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 52cfa582d0..ef6ff8a70b 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -113,7 +113,6 @@ static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void createTableQueryInfo(SQInfo *pQInfo); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, STableId *pTableId, STableQueryInfo *pTableQueryInfo); @@ -507,7 +506,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.ekey = pQuery->window.ekey; } - assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + assert(ts >= w.skey && ts <= w.ekey); return w; } @@ -624,7 +623,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); } else { // set the current index to be the last unclosed window int32_t i = 0; - int64_t skey = 0; + int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; @@ -642,7 +641,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, } // all windows are closed, set the last one to be the skey - if (skey == 0) { + if (skey == TSKEY_INITIAL_VAL) { assert(i == pWindowResInfo->size); pWindowResInfo->curIndex = pWindowResInfo->size - 1; } else { @@ -660,7 +659,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, qTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, n); } - assert(pWindowResInfo->prevSKey != 0); + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -2399,7 +2398,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { TSKEY skey1, ekey1; - STimeWindow w = {0}; + STimeWindow w = TSWINDOW_INITIALIZER; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -2436,8 +2435,16 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // set the pCtx output buffer position pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; + + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + } } - + + qTrace("QInfo: %p realloc output buffer, new size: %d rows, old:%d, remain:%d", GET_QINFO_ADDR(pRuntimeEnv), + newSize, pRec->capacity, newSize - pRec->rows); + pRec->capacity = newSize; } } @@ -2641,26 +2648,21 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t } } -void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { -#if 0 +void UNUSED_FUNC displayInterResult(SData **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); - SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); - SMeterObj *pMeterObj = pQInfo->pObj; - for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; - int32_t type = 0; - - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].base.colInfo.flag)) { - type = pQuery->pSelectExpr[i].type; - } else { - type = pMeterObj->schema[colIndex].type; - } +// int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; + int32_t type = pQuery->pSelectExpr[i].type; +// } else { +// type = pMeterObj->schema[colIndex].type; +// } printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; @@ -2682,7 +2684,6 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } printf("\n"); } -#endif } typedef struct SCompSupporter { @@ -2950,7 +2951,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -3078,6 +3079,9 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { int32_t functId = pQuery->pSelectExpr[j].base.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; + if (pCtx->resultInfo == NULL) { + continue; // resultInfo is NULL, means no data checked in previous scan + } if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { @@ -3458,7 +3462,11 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -STableQueryInfo *createTableQueryInfoImpl(SQueryRuntimeEnv *pRuntimeEnv, STableId tableId, STimeWindow win) { +static STableQueryInfo *createTableQueryInfo( + SQueryRuntimeEnv *pRuntimeEnv, + STableId tableId, + STimeWindow win +) { STableQueryInfo *pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); pTableQueryInfo->win = win; @@ -3588,7 +3596,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { - pQuery->window.skey = key; + pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; // for too small query range, no data in this interval. @@ -3608,20 +3616,18 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); - pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp + pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp - if (pWindowResInfo->prevSKey == 0) { - if (QUERY_IS_ASC_QUERY(pQuery)) { - pWindowResInfo->prevSKey = w.skey; - } else { + if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.skey); - pWindowResInfo->prevSKey = w.skey; } + + pWindowResInfo->prevSKey = w.skey; } pTableQueryInfo->queryRangeSet = 1; - pTableQueryInfo->lastKey = pQuery->window.skey; - pTableQueryInfo->win.skey = pQuery->window.skey; + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; } } @@ -3726,7 +3732,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde qTrace("QInfo:%p copy data to query buf completed", pQInfo); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, numOfResult); + displayInterResult(pQuery->sdata, pRuntimeEnv, numOfResult); #endif return numOfResult; } @@ -3867,7 +3873,18 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data data += bytes * numOfRows; } - + int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo); + *(int32_t*)data = htonl(numOfTables); + data += sizeof(int32_t); + for(int32_t i = 0; i < numOfTables; i++) { + STableIdInfo* pSrc = taosArrayGet(pQInfo->arrTableIdInfo, i); + STableIdInfo* pDst = (STableIdInfo*)data; + pDst->uid = htobe64(pSrc->uid); + pDst->tid = htonl(pSrc->tid); + pDst->key = htobe64(pSrc->key); + data += sizeof(STableIdInfo); + } + // all data returned, set query over if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery && isIntervalQuery(pQuery)) { @@ -4054,10 +4071,11 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is * not valid. otherwise, we only forward pQuery->limit.offset number of points */ - assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = TSWINDOW_INITIALIZER; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4145,6 +4163,42 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { return true; } + +static void setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + if (onlyQueryTags(pQuery)) { + return; + } + + if (isSTableQuery && (!isIntervalQuery(pQuery)) && (!isFixedOutputQuery(pQuery))) { + return; + } + + STsdbQueryCond cond = { + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, + .numOfCols = pQuery->numOfCols, + }; + + if (!isSTableQuery + && (pQInfo->groupInfo.numOfTables == 1) + && (cond.order == TSDB_ORDER_ASC) + && (!isIntervalQuery(pQuery)) + && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) + && (!isFixedOutputQuery(pQuery)) + ) { + SArray* pa = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + SGroupItem* pItem = taosArrayGet(pa, 0); + cond.twindow = pItem->info->win; + } + + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); +} + + int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; @@ -4153,26 +4207,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); - - STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - - // normal query setup the queryhandle here - if (!onlyQueryTags(pQuery)) { - if (!isSTableQuery && isFirstLastRowQuery(pQuery)) { // in case of last_row query, invoke a different API. - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableIdGroupInfo); - } else if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableIdGroupInfo); - } - - // create the table query support structures - createTableQueryInfo(pQInfo); - } + setupQueryHandle(tsdb, pQInfo, isSTableQuery); pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4591,6 +4626,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * to ensure that, we can reset the query range once query on a meter is completed. */ pQInfo->tableIndex++; + + STableIdInfo tidInfo; + tidInfo.uid = item->id.uid; + tidInfo.tid = item->id.tid; + tidInfo.key = pQuery->current->lastKey; + taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); + // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { @@ -4660,35 +4702,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->limit.offset); } -static void createTableQueryInfo(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - // todo make sure the table are added the reference count to gauranteed that all involved tables are valid - size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - - int32_t index = 0; - for (int32_t i = 0; i < numOfGroups; ++i) { // load all meter meta info - SArray *group = *(SArray **)taosArrayGet(pQInfo->groupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(group); - for (int32_t j = 0; j < s; ++j) { - SGroupItem* item = (SGroupItem *)taosArrayGet(group, j); - - // STableQueryInfo has been created for each table - if (item->info != NULL) { - return; - } - - STableQueryInfo* pInfo = createTableQueryInfoImpl(&pQInfo->runtimeEnv, item->id, pQuery->window); - pInfo->groupIdx = i; - pInfo->tableIndex = index; - - item->info = pInfo; - index += 1; - } - } -} - static void doSaveContext(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4727,7 +4740,7 @@ static void doRestoreContext(SQInfo *pQInfo) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; + SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } switchCtxOrder(pRuntimeEnv); @@ -4766,7 +4779,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4819,7 +4832,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } } else { // not a interval query @@ -4910,6 +4923,12 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->current->lastKey, pQuery->window.ekey); + } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + STableIdInfo tidInfo; + tidInfo.uid = pQuery->current->id.uid; + tidInfo.tid = pQuery->current->id.tid; + tidInfo.key = pQuery->current->lastKey; + taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); } if (!isTSCompQuery(pQuery)) { @@ -5193,20 +5212,10 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { assert(pQueryMsg->numOfTables > 0); - *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableIdInfo->tid); - pTableIdInfo->uid = htobe64(pTableIdInfo->uid); - pTableIdInfo->key = htobe64(pTableIdInfo->key); - - STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->tid}; - taosArrayPush(*pTableIdList, &id); - - pMsg += sizeof(STableIdInfo); - - for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { - pTableIdInfo = (STableIdInfo *)pMsg; + for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) { + STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableIdInfo->tid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); @@ -5276,26 +5285,26 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - - pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; + + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; + pColFilter->filterstr = htons(pFilterMsg->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterstr) { - pDestFilterInfo->len = htobe64(pFilterInfo->len); + if (pColFilter->filterstr) { + pColFilter->len = htobe64(pFilterMsg->len); - pDestFilterInfo->pz = (int64_t) calloc(1, pDestFilterInfo->len); - memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len); - pMsg += (pDestFilterInfo->len); + pColFilter->pz = (int64_t) calloc(1, pColFilter->len); + memcpy((void *)pColFilter->pz, pMsg, pColFilter->len); + pMsg += (pColFilter->len + 1); } else { - pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); - pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); + pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); + pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); } - pDestFilterInfo->lowerRelOptr = htons(pFilterInfo->lowerRelOptr); - pDestFilterInfo->upperRelOptr = htons(pFilterInfo->upperRelOptr); + pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); + pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); } } @@ -5657,7 +5666,16 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + +static int compareTableIdInfo( const void* a, const void* b ) { + const STableIdInfo* x = (const STableIdInfo*)a; + const STableIdInfo* y = (const STableIdInfo*)b; + if (x->uid > y->uid) return 1; + if (x->uid < y->uid) return -1; + return 0; +} + +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, STableGroupInfo *groupInfo, SColumnInfo* pTagCols) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { @@ -5692,9 +5710,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; - - SColumnInfo *pColInfo = &pQuery->colList[i]; - pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); + pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); } pQuery->tagColList = pTagCols; @@ -5752,6 +5768,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->groupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->groupInfo.numOfTables = groupInfo->numOfTables; + int tableIndex = 0; + STimeWindow window = pQueryMsg->window; + taosArraySort( pTableIdList, compareTableIdInfo ); for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(groupInfo->pGroupList, i); size_t s = taosArrayGetSize(pa); @@ -5759,13 +5778,26 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou SArray* p1 = taosArrayInit(s, sizeof(SGroupItem)); for(int32_t j = 0; j < s; ++j) { - SGroupItem item = { .id = *(STableId*) taosArrayGet(pa, j), .info = NULL, }; + STableId id = *(STableId*) taosArrayGet(pa, j); + SGroupItem item = { .id = id }; + // NOTE: compare STableIdInfo with STableId + // not a problem at present because we only use their 1st int64_t field + STableIdInfo* pTableId = taosArraySearch( pTableIdList, compareTableIdInfo, &id ); + if (pTableId != NULL ) { + window.skey = pTableId->key; + } else { + window.skey = pQueryMsg->window.skey; + } + item.info = createTableQueryInfo(&pQInfo->runtimeEnv, item.id, window); + item.info->groupIdx = i; + item.info->tableIndex = tableIndex++; taosArrayPush(p1, &item); } - taosArrayPush(pQInfo->groupInfo.pGroupList, &p1); } + pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); + pQuery->pos = -1; pQuery->window = pQueryMsg->window; @@ -5917,6 +5949,7 @@ static void freeQInfo(SQInfo *pQInfo) { } taosArrayDestroy(pQInfo->tableIdGroupInfo.pGroupList); + taosArrayDestroy(pQInfo->arrTableIdInfo); if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); @@ -6044,32 +6077,48 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_TABLE_QUERY)) { isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - STableId *id = taosArrayGet(pTableIdList, 0); + STableIdInfo *id = taosArrayGet(pTableIdList, 0); if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { goto _over; } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; - STableId *id = taosArrayGet(pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - // todo handle the error - /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, - numOfGroupByCols); - if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query - code = TSDB_CODE_SUCCESS; - goto _over; + // TODO: need a macro from TSDB to check if table is super table, + // also note there's possiblity that only one table in the super table + if (taosArrayGetSize(pTableIdList) == 1) { + STableIdInfo *id = taosArrayGet(pTableIdList, 0); + // if array size is 1 and assert super table + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + // todo handle the error + /*int32_t ret =*/tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, + numOfGroupByCols); + if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query + code = TSDB_CODE_SUCCESS; + goto _over; + } + } else { + groupInfo.numOfTables = taosArrayGetSize(pTableIdList); + SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); + + SArray* sa = taosArrayInit(groupInfo.numOfTables, sizeof(STableId)); + for(int32_t i = 0; i < groupInfo.numOfTables; ++i) { + STableIdInfo* tableId = taosArrayGet(pTableIdList, i); + taosArrayPush(sa, tableId); + } + taosArrayPush(pTableGroup, &sa); + groupInfo.pGroupList = pTableGroup; } } else { assert(0); } - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); + (*pQInfo) = createQInfoImpl(pQueryMsg, pTableIdList, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; goto _over; @@ -6167,6 +6216,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co SQuery *pQuery = pQInfo->runtimeEnv.pQuery; size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + size += sizeof(int32_t); + size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); *contLen = size + sizeof(SRetrieveTableRsp); // todo handle failed to allocate memory diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c index 3218b26179..41a888e92d 100644 --- a/src/query/src/queryFilterFunc.c +++ b/src/query/src/queryFilterFunc.c @@ -183,33 +183,33 @@ bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) == 0; } bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) == 0; } //////////////////////////////////////////////////////////////// bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; + return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; } bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; } //////////////////////////////////////////////////////////////// @@ -270,11 +270,11 @@ bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return true; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) != 0; } bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { @@ -282,7 +282,7 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return true; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } //////////////////////////////////////////////////////////////// diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index 51d3286722..6f8d579936 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -17,7 +17,7 @@ #include "hash.h" #include "hashfunc.h" #include "os.h" -#include "shash.h" +#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index 41156ce8ff..fa2b58a10b 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -76,4 +76,8 @@ TEST(testCase, patternMatchTest) { str = "carzero"; ret = patternMatch("%o", str, strlen(str), &info); EXPECT_EQ(ret, TSDB_PATTERN_MATCH); + + str = "19"; + ret = patternMatch("%9", str, 2, &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h index 8b5410a596..520edadc7d 100644 --- a/src/rpc/inc/rpcHead.h +++ b/src/rpc/inc/rpcHead.h @@ -49,6 +49,7 @@ typedef struct { char encrypt:3; // encrypt algorithm, 0: no encryption uint16_t tranId; // transcation ID uint32_t linkUid; // for unique connection ID assigned by client + uint64_t ahandle; // ahandle assigned by client uint32_t sourceId; // source ID, an index for connection list uint32_t destId; // destination ID, an index for connection list uint32_t destIp; // destination IP address, for NAT scenario diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index edbb9b3e12..7a96571ab9 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -146,7 +146,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in rpcUnlockCache(pCache->lockedBy+hash); pCache->total++; - tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); return; } @@ -200,9 +200,9 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy rpcUnlockCache(pCache->lockedBy+hash); if (pData) { - tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); + //tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); } else { - tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); + //tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); } return pData; @@ -240,8 +240,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pNext = pNode->next; pCache->total--; pCache->count[hash]--; - tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, - pCache->count[hash]); + //tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, + // pCache->count[hash]); taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); pNode = pNext; } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index ca4b211be8..445e542387 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -87,6 +87,7 @@ typedef struct { } SRpcReqContext; typedef struct SRpcConn { + char info[50];// debug info: label + pConn + ahandle int sid; // session ID uint32_t ownId; // own link ID uint32_t peerId; // peer link ID @@ -275,7 +276,7 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - tTrace("%s RPC is openned, numOfThreads:%d", pRpc->label, pRpc->numOfThreads); + tTrace("%s rpc is openned, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); return pRpc; } @@ -299,7 +300,7 @@ void rpcClose(void *param) { tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); - tTrace("%s RPC is closed", pRpc->label); + tTrace("%s rpc is closed", pRpc->label); tfree(pRpc); } @@ -361,9 +362,10 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; - if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH || - type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || - type == TSDB_MSG_TYPE_CM_SHOW ) + if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP + || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META + || type == TSDB_MSG_TYPE_CM_SHOW ) pContext->connType = RPC_CONN_TCPC; rpcSendReqToServer(pRpc, pContext); @@ -374,8 +376,6 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) void rpcSendResponse(const SRpcMsg *pRsp) { int msgLen = 0; SRpcConn *pConn = (SRpcConn *)pRsp->handle; - SRpcInfo *pRpc = pConn->pRpc; - SRpcMsg rpcMsg = *pRsp; SRpcMsg *pMsg = &rpcMsg; @@ -393,7 +393,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { rpcLockConn(pConn); if ( pConn->inType == 0 || pConn->user[0] == 0 ) { - tTrace("%s %p, connection is already released, rsp wont be sent", pRpc->label, pConn); + tTrace("%s, connection is already released, rsp wont be sent", pConn->info); rpcUnlockConn(pConn); return; } @@ -409,7 +409,8 @@ void rpcSendResponse(const SRpcMsg *pRsp) { pHead->linkUid = pConn->linkUid; pHead->port = htons(pConn->localPort); pHead->code = htonl(pMsg->code); - + pHead->ahandle = (uint64_t) pConn->ahandle; + // set pConn parameters pConn->inType = 0; @@ -491,6 +492,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); if (peerIp == -1) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -506,11 +508,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, if (taosOpenConn[connType]) { void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle; pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); - if (pConn->chandle) { - tTrace("%s %p, rpc connection is set up, sid:%d id:%s %s:%hu connType:%d", pRpc->label, - pConn, pConn->sid, pRpc->user, peerFqdn, pConn->peerPort, pConn->connType); - } else { - tError("%s %p, failed to set up connection to %s:%hu", pRpc->label, pConn, peerFqdn, pConn->peerPort); + if (pConn->chandle == NULL) { terrno = TSDB_CODE_NETWORK_UNAVAIL; rpcCloseConn(pConn); pConn = NULL; @@ -557,7 +555,7 @@ static void rpcCloseConn(void *thandle) { taosFreeId(pRpc->idPool, pConn->sid); pConn->pContext = NULL; - tTrace("%s %p, rpc connection is closed", pRpc->label, pConn); + tTrace("%s, rpc connection is closed", pConn->info); rpcUnlockConn(pConn); } @@ -612,9 +610,13 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { pConn->ownId = htonl(pConn->sid); pConn->linkUid = pHead->linkUid; if (pRpc->afp) { - terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); + if (pConn->user[0] == 0) { + terrno = TSDB_CODE_AUTH_REQUIRED; + } else { + terrno = (*pRpc->afp)(pConn->user, &pConn->spi, &pConn->encrypt, pConn->secret, pConn->ckey); + } + if (terrno != 0) { - tWarn("%s %p, user not there or server not ready", pRpc->label, pConn); taosFreeId(pRpc->idPool, sid); // sid shall be released pConn = NULL; } @@ -629,8 +631,6 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tTrace("%s %p, rpc connection is allocated, sid:%d id:%s port:%u", - pRpc->label, pConn, sid, pConn->user, pConn->localPort); } return pConn; @@ -655,7 +655,6 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { if (pConn) { if (pConn->linkUid != pHead->linkUid) { - tTrace("%s %p, linkUid:0x%x not matched, received:0x%x", pRpc->label, pConn, pConn->linkUid, pHead->linkUid); terrno = TSDB_CODE_MISMATCHED_METER_ID; pConn = NULL; } @@ -672,21 +671,25 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); if ( pConn == NULL || pConn->user[0] == 0) { pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); + } + + if (pConn) { + pConn->ahandle = pContext->ahandle; + sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle); } else { - tTrace("%s %p, connection is retrieved from cache", pRpc->label, pConn); + tError("%s %p, failed to set up connection(%s)", pRpc->label, pContext->ahandle, tstrerror(terrno)); } return pConn; } static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { - SRpcInfo *pRpc= pConn->pRpc; if (pConn->peerId == 0) { pConn->peerId = pHead->sourceId; } else { if (pConn->peerId != pHead->sourceId) { - tTrace("%s %p, source Id is changed, old:0x%08x new:0x%08x", pRpc->label, pConn, + tTrace("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId); return TSDB_CODE_INVALID_VALUE; } @@ -695,17 +698,16 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { if (pConn->inTranId == pHead->tranId) { if (pConn->inType == pHead->msgType) { if (pHead->code == 0) { - tTrace("%s %p, %s is retransmitted", pRpc->label, pConn, taosMsg[pHead->msgType]); + tTrace("%s, %s is retransmitted", pConn->info, taosMsg[pHead->msgType]); rpcSendQuickRsp(pConn, TSDB_CODE_ACTION_IN_PROGRESS); } else { // do nothing, it is heart beat from client } } else if (pConn->inType == 0) { - tTrace("%s %p, %s is already processed, tranId:%d", pRpc->label, pConn, - taosMsg[pHead->msgType], pConn->inTranId); + tTrace("%s, %s is already processed, tranId:%d", pConn->info, taosMsg[pHead->msgType], pConn->inTranId); rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response } else { - tTrace("%s %p, mismatched message %s and tranId", pRpc->label, pConn, taosMsg[pHead->msgType]); + tTrace("%s, mismatched message %s and tranId", pConn->info, taosMsg[pHead->msgType]); } // do not reply any message @@ -713,7 +715,7 @@ static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { } if (pConn->inType != 0) { - tTrace("%s %p, last session is not finished, inTranId:%d tranId:%d", pRpc->label, pConn, + tTrace("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId); return TSDB_CODE_LAST_SESSION_NOT_FINISHED; } @@ -745,7 +747,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_ACTION_IN_PROGRESS) { if (pConn->tretry <= tsRpcMaxRetry) { - tTrace("%s %p, peer is still processing the transaction", pRpc->label, pConn); + tTrace("%s, peer is still processing the transaction", pConn->info); pConn->tretry++; rpcSendReqHead(pConn); taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer); @@ -784,7 +786,15 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { } pConn = rpcGetConnObj(pRpc, sid, pRecv); - if (pConn == NULL) return NULL; + if (pConn == NULL) { + tError("%s %p, failed to get connection obj(%s)", pRpc->label, pHead->ahandle, tstrerror(terrno)); + return NULL; + } else { + if (rpcIsReq(pHead->msgType)) { + pConn->ahandle = (void *)pHead->ahandle; + sprintf(pConn->info, "%s %p %p", pRpc->label, pConn, pConn->ahandle); + } + } rpcLockConn(pConn); sid = pConn->sid; @@ -821,7 +831,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) { static void rpcProcessBrokenLink(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; - tTrace("%s %p, link is broken", pRpc->label, pConn); + tTrace("%s, link is broken", pConn->info); // pConn->chandle = NULL; if (pConn->outType) { @@ -832,7 +842,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { if (pConn->inType) { // if there are pending request, notify the app - tTrace("%s %p, connection is gone, notify the app", pRpc->label, pConn); + tTrace("%s, connection is gone, notify the app", pConn->info); /* SRpcMsg rpcMsg; rpcMsg.pCont = NULL; @@ -867,17 +877,17 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { pConn = rpcProcessMsgHead(pRpc, pRecv); if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) { - tTrace("%s %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", - pRpc->label, pConn, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, + tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", + pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); } int32_t code = terrno; if (code != TSDB_CODE_ALREADY_PROCESSED) { if (code != 0) { // parsing error - if ( rpcIsReq(pHead->msgType) ) { + if (rpcIsReq(pHead->msgType)) { rpcSendErrorMsgToPeer(pRecv, code); - tTrace("%s %p, %s is sent with error code:%x", pRpc->label, pConn, taosMsg[pHead->msgType+1], code); + tTrace("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); } } else { // parsing OK rpcProcessIncomingMsg(pConn, pHead); @@ -893,9 +903,9 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { if (pContext->pRsp) { // for synchronous API - tsem_post(pContext->pSem); memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); + tsem_post(pContext->pSem); } else { // for asynchronous API SRpcIpSet *pIpSet = NULL; @@ -919,6 +929,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; rpcMsg.code = pHead->code; + rpcMsg.ahandle = pConn->ahandle; if ( rpcIsReq(pHead->msgType) ) { rpcMsg.handle = pConn; @@ -930,6 +941,12 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { rpcMsg.handle = pContext->ahandle; pConn->pContext = NULL; + if (pHead->code == TSDB_CODE_AUTH_REQUIRED) { + pConn->secured = 0; + rpcSendReqToServer(pRpc, pContext); + return; + } + // for UDP, port may be changed by server, the port in ipSet shall be used for cache rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType); @@ -937,14 +954,14 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { pContext->redirect++; if (pContext->redirect > TSDB_MAX_REPLICA) { pHead->code = TSDB_CODE_NETWORK_UNAVAIL; - tWarn("%s %p, too many redirects, quit", pRpc->label, pConn); + tWarn("%s, too many redirects, quit", pConn->info); } } if (pHead->code == TSDB_CODE_REDIRECT) { pContext->numOfTry = 0; memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); - tTrace("%s %p, redirect is received, numOfIps:%d", pRpc->label, pConn, pContext->ipSet.numOfIps); + tTrace("%s, redirect is received, numOfIps:%d", pConn->info, pContext->ipSet.numOfIps); for (int i=0; iipSet.numOfIps; ++i) pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]); rpcSendReqToServer(pRpc, pContext); @@ -1050,6 +1067,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { return; } + pConn->ahandle = pContext->ahandle; rpcLockConn(pConn); // set the message header @@ -1063,6 +1081,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { pHead->destId = pConn->peerId; pHead->port = 0; pHead->linkUid = pConn->linkUid; + pHead->ahandle = (uint64_t)pConn->ahandle; if (!pConn->secured) memcpy(pHead->user, pConn->user, tListLen(pHead->user)); // set the connection parameters @@ -1080,29 +1099,28 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { int writtenLen = 0; - SRpcInfo *pRpc = pConn->pRpc; SRpcHead *pHead = (SRpcHead *)msg; msgLen = rpcAddAuthPart(pConn, msg, msgLen); if ( rpcIsReq(pHead->msgType)) { if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) - tTrace("%s %p, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerFqdn, - pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", + pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, + msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) - tTrace( "%s %p, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pRpc->label, pConn, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, + tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", + pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } + //tTrace("connection type is: %d", pConn->connType); writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); if (writtenLen != msgLen) { - tError("%s %p, failed to send, dataLen:%d writtenLen:%d, reason:%s", pRpc->label, pConn, - msgLen, writtenLen, strerror(errno)); + tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno)); } tDump(msg, msgLen); @@ -1117,7 +1135,7 @@ static void rpcProcessConnError(void *param, void *id) { return; } - tTrace("%s connection error happens", pRpc->label); + tTrace("%s %p, connection error happens", pRpc->label, pContext->ahandle); if (pContext->numOfTry >= pContext->ipSet.numOfIps) { rpcMsg.msgType = pContext->msgType+1; @@ -1143,23 +1161,21 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { rpcLockConn(pConn); if (pConn->outType && pConn->user[0]) { - tTrace("%s %p, expected %s is not received", pRpc->label, pConn, taosMsg[(int)pConn->outType + 1]); + tTrace("%s, expected %s is not received", pConn->info, taosMsg[(int)pConn->outType + 1]); pConn->pTimer = NULL; pConn->retry++; if (pConn->retry < 4) { - tTrace("%s %p, re-send msg:%s to %s:%hu", pRpc->label, pConn, - taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); + tTrace("%s, re-send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); taosTmrReset(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl, &pConn->pTimer); } else { // close the connection - tTrace("%s %p, failed to send msg:%s to %s:%hu", pRpc->label, pConn, - taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); + tTrace("%s, failed to send msg:%s to %s:%hu", pConn->info, taosMsg[pConn->outType], pConn->peerFqdn, pConn->peerPort); reportDisc = 1; } } else { - tTrace("%s %p, retry timer not processed", pRpc->label, pConn); + tTrace("%s, retry timer not processed", pConn->info); } rpcUnlockConn(pConn); @@ -1176,10 +1192,10 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) { SRpcInfo *pRpc = pConn->pRpc; if (pConn->user[0]) { - tTrace("%s %p, close the connection since no activity", pRpc->label, pConn); + tTrace("%s, close the connection since no activity", pConn->info); if (pConn->inType && pRpc->cfp) { // if there are pending request, notify the app - tTrace("%s %p, notify the app, connection is gone", pRpc->label, pConn); + tTrace("%s, notify the app, connection is gone", pConn->info); /* SRpcMsg rpcMsg; rpcMsg.pCont = NULL; @@ -1192,7 +1208,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) { } rpcCloseConn(pConn); } else { - tTrace("%s %p, idle timer:%p not processed", pRpc->label, pConn, tmrId); + tTrace("%s, idle timer:%p not processed", pConn->info, tmrId); } } @@ -1203,11 +1219,11 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) { rpcLockConn(pConn); if (pConn->inType && pConn->user[0]) { - tTrace("%s %p, progress timer expired, send progress", pRpc->label, pConn); + tTrace("%s, progress timer expired, send progress", pConn->info); rpcSendQuickRsp(pConn, TSDB_CODE_ACTION_IN_PROGRESS); taosTmrReset(rpcProcessProgressTimer, tsRpcTimer/2, pConn, pRpc->tmrCtrl, &pConn->pTimer); } else { - tTrace("%s %p, progress timer:%p not processed", pRpc->label, pConn, tmrId); + tTrace("%s, progress timer:%p not processed", pConn->info, tmrId); } rpcUnlockConn(pConn); @@ -1241,7 +1257,7 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { memcpy(pCont + overhead, buf, compLen); pHead->comp = 1; - tTrace("compress rpc msg, before:%d, after:%d", contLen, compLen); + //tTrace("compress rpc msg, before:%d, after:%d", contLen, compLen); finalLen = compLen + overhead; } else { finalLen = contLen; @@ -1275,7 +1291,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { pNewHead->msgLen = rpcMsgLenFromCont(origLen); rpcFreeMsg(pHead); // free the compressed message buffer pHead = pNewHead; - tTrace("decompress rpc msg, compLen:%d, after:%d", compLen, contLen); + //tTrace("decompress rpc msg, compLen:%d, after:%d", compLen, contLen); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); } @@ -1332,7 +1348,6 @@ static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen) { static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { SRpcHead *pHead = (SRpcHead *)msg; - SRpcInfo *pRpc = pConn->pRpc; int code = 0; if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){ @@ -1360,20 +1375,20 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { delta = (int32_t)htonl(pDigest->timeStamp); delta -= (int32_t)taosGetTimestampSec(); if (abs(delta) > 900) { - tWarn("%s %p, time diff:%d is too big, msg discarded", pRpc->label, pConn, delta); + tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); code = TSDB_CODE_INVALID_TIME_STAMP; } else { if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { - tError("%s %p, authentication failed, msg discarded", pRpc->label, pConn); + tError("%s, authentication failed, msg discarded", pConn->info); code = TSDB_CODE_AUTH_FAILURE; } else { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1; // link is secured for client - tTrace("%s %p, message is authenticated", pRpc->label, pConn); + //tTrace("%s, message is authenticated", pConn->info); } } } else { - tTrace("%s %p, auth spi:%d not matched with received:%d", pRpc->label, pConn, pConn->spi, pHead->spi); + tError("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); code = TSDB_CODE_AUTH_FAILURE; } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 8d5721a437..677187e3b9 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -18,7 +18,6 @@ #include "tsystem.h" #include "ttimer.h" #include "tutil.h" -#include "thash.h" #include "rpcLog.h" #include "rpcHaship.h" #include "rpcUdp.h" @@ -219,7 +218,7 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); port = ntohs(sourceAdd.sin_port); - tTrace("%s msg is recv from 0x%x:%hu len:%d", pConn->label, sourceAdd.sin_addr.s_addr, port, dataLen); + //tTrace("%s msg is recv from 0x%x:%hu len:%d", pConn->label, sourceAdd.sin_addr.s_addr, port, dataLen); if (dataLen < sizeof(SRpcHead)) { tError("%s recvfrom failed, reason:%s\n", pConn->label, strerror(errno)); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 2c3aff4b84..c5a6ed54c3 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -29,23 +29,23 @@ extern "C" { extern int tsdbDebugFlag; -#define tsdbError(...) \ - if (tsdbDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("ERROR TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbError(...) \ + if (tsdbDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("ERROR TDB ", tsdbDebugFlag, __VA_ARGS__); \ } -#define tsdbWarn(...) \ - if (tsdbDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WARN TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbWarn(...) \ + if (tsdbDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WARN TDB ", tsdbDebugFlag, __VA_ARGS__); \ } -#define tsdbTrace(...) \ - if (tsdbDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("TSDB ", tsdbDebugFlag, __VA_ARGS__); \ +#define tsdbTrace(...) \ + if (tsdbDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); \ } #define tsdbPrint(...) \ - { taosPrintLog("TSDB ", 255, __VA_ARGS__); } + { taosPrintLog("TDB ", 255, __VA_ARGS__); } // ------------------------------ TSDB META FILE INTERFACES ------------------------------ -#define TSDB_META_FILE_NAME "META" +#define TSDB_META_FILE_NAME "meta" #define TSDB_META_HASH_FRACTION 1.1 typedef int (*iterFunc)(void *, void *cont, int contLen); @@ -151,8 +151,6 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable); STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo); -int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg); -int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); @@ -216,12 +214,12 @@ typedef struct { int64_t tombSize; // unused file size int32_t totalBlocks; int32_t totalSubBlocks; -} SFileInfo; +} STsdbFileInfo; typedef struct { int fd; char fname[128]; - SFileInfo info; + STsdbFileInfo info; } SFile; #define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) @@ -327,6 +325,13 @@ typedef struct { int16_t len; // Column length // TODO: int16_t is not enough int32_t type : 8; int32_t offset : 24; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + char padding[2]; } SCompCol; // TODO: Take recover into account @@ -345,7 +350,7 @@ SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); // TSDB repository definition -typedef struct _tsdb_repo { +typedef struct STsdbRepo { char *rootDir; // TSDB configuration STsdbCfg config; @@ -496,9 +501,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper); int tsdbWriteCompIdx(SRWHelper *pHelper); // --------- Other functions need to further organize -void tsdbFitRetention(STsdbRepo *pRepo); -int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); -void tsdbAdjustCacheBlocks(STsdbCache *pCache); +void tsdbFitRetention(STsdbRepo *pRepo); +int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); +void tsdbAdjustCacheBlocks(STsdbCache *pCache); +int32_t tsdbGetMetaFileName(char *rootDir, char *fname); #ifdef __cplusplus } diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 84f8a81eea..2761ed5e8e 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -154,7 +154,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { for (int i = 0; i < blocksToAdd; i++) { if (tsdbAddCacheBlockToPool(pCache) < 0) { tsdbUnLockRepo((TsdbRepoT *)pRepo); - tsdbError("tsdbId %d: failed to add cache block to cache pool", pRepo->config.tsdbId); + tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); return -1; } } @@ -164,7 +164,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { } tsdbUnLockRepo((TsdbRepoT *)pRepo); - tsdbTrace("tsdbId %d: tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); + tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); return 0; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 653ec50af8..c63de165d3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -7,6 +7,7 @@ #include "tscompression.h" #include "tchecksum.h" #include "ttime.h" +#include int tsdbDebugFlag = 135; @@ -17,7 +18,7 @@ int tsdbDebugFlag = 135; #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX -#define TSDB_CFG_FILE_NAME "CONFIG" +#define TSDB_CFG_FILE_NAME "config" #define TSDB_DATA_DIR_NAME "data" #define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 #define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M @@ -28,7 +29,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo); static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo); // static int tsdbOpenMetaFile(char *tsdbDir); -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now); +static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); @@ -90,7 +91,7 @@ void tsdbFreeCfg(STsdbCfg *pCfg) { int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) { if (mkdir(rootDir, 0755) != 0) { - tsdbError("id %d: failed to create rootDir! rootDir %s, reason %s", pCfg->tsdbId, rootDir, strerror(errno)); + tsdbError("vgId:%d, failed to create rootDir! rootDir:%s, reason:%s", pCfg->tsdbId, rootDir, strerror(errno)); if (errno == EACCES) { return TSDB_CODE_NO_DISK_PERMISSIONS; } else if (errno == ENOSPC) { @@ -133,6 +134,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) */ int32_t tsdbDropRepo(TsdbRepoT *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; + int id = pRepo->config.tsdbId; pRepo->state = TSDB_REPO_STATE_CLOSED; @@ -148,6 +150,8 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) { free(pRepo->rootDir); free(pRepo); + tsdbTrace("vgId:%d, tsdb repository is dropped!", id); + return 0; } @@ -157,7 +161,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { SFileGroup *pFGroup = NULL; SFileGroupIter iter; - SRWHelper rhelper = {0}; + SRWHelper rhelper = {{0}}; if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); @@ -165,6 +169,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; for (int i = 1; i < pRepo->config.maxTables; i++) { STable * pTable = pMeta->tables[i]; + if (pTable == NULL) continue; SCompIdx *pIdx = &rhelper.pCompIdx[i]; if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; @@ -238,6 +243,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { pRepo->state = TSDB_REPO_STATE_ACTIVE; + tsdbTrace("vgId:%d, open tsdb repository successfully!", pRepo->config.tsdbId); return (TsdbRepoT *)pRepo; } @@ -253,9 +259,10 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { * * @return 0 for success, -1 for failure and the error number is set */ -int32_t tsdbCloseRepo(TsdbRepoT *repo) { +int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit) { STsdbRepo *pRepo = (STsdbRepo *)repo; if (pRepo == NULL) return 0; + int id = pRepo->config.tsdbId; pRepo->state = TSDB_REPO_STATE_CLOSED; tsdbLockRepo(repo); @@ -278,7 +285,8 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) { pRepo->tsdbCache->curBlock = NULL; tsdbUnLockRepo(repo); - tsdbCommitData((void *)repo); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); + if (toCommit) tsdbCommitData((void *)repo); tsdbCloseFileH(pRepo->tsdbFileH); @@ -289,6 +297,8 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) { tfree(pRepo->rootDir); tfree(pRepo); + tsdbTrace("vgId:%d, repository is closed!", id); + return 0; } @@ -322,7 +332,7 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) { int32_t tsdbTriggerCommit(TsdbRepoT *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; - if (pRepo->appH.walCallBack) pRepo->appH.walCallBack(pRepo->appH.appH); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); tsdbLockRepo(repo); if (pRepo->commit) { @@ -349,6 +359,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); + tsdbTrace("vgId:%d, start to commit!", pRepo->config.tsdbId); return 0; } @@ -376,11 +387,6 @@ STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) { return NULL; } -int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - return tsdbCreateTableImpl(pRepo->tsdbMeta, pCfg); -} - int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) { // TODO return 0; @@ -395,35 +401,29 @@ TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) { return TSDB_GET_TABLE_LAST_KEY(pTable); } -int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { - if (repo == NULL) return -1; - STsdbRepo *pRepo = (STsdbRepo *)repo; - - return tsdbDropTableImpl(pRepo->tsdbMeta, tableId); -} - STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) { // TODO return NULL; } // TODO: need to return the number of data inserted -int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg) { +int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) { SSubmitMsgIter msgIter; STsdbRepo *pRepo = (STsdbRepo *)repo; tsdbInitSubmitMsgIter(pMsg, &msgIter); SSubmitBlk *pBlock = NULL; int32_t code = TSDB_CODE_SUCCESS; + int32_t affectedrows = 0; TSKEY now = taosGetTimestamp(pRepo->config.precision); while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) { - if ((code = tsdbInsertDataToTable(repo, pBlock, now)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) { return code; } } - + pRsp->affectedRows = htonl(affectedrows); return code; } @@ -613,7 +613,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { pCfg->precision = TSDB_DEFAULT_PRECISION; } else { if (!IS_VALID_PRECISION(pCfg->precision)) { - tsdbError("id %d: invalid precision configuration! precision %d", pCfg->tsdbId, pCfg->precision); + tsdbError("vgId:%d, invalid precision configuration! precision:%d", pCfg->tsdbId, pCfg->precision); return -1; } } @@ -623,7 +623,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { pCfg->compression = TSDB_DEFAULT_COMPRESSION; } else { if (!IS_VALID_COMPRESSION(pCfg->compression)) { - tsdbError("id %d: invalid compression configuration! compression %d", pCfg->tsdbId, pCfg->precision); + tsdbError("vgId:%d: invalid compression configuration! compression:%d", pCfg->tsdbId, pCfg->precision); return -1; } } @@ -636,7 +636,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { pCfg->maxTables = TSDB_DEFAULT_TABLES; } else { if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { - tsdbError("id %d: invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", + tsdbError("vgId:%d: invalid maxTables configuration! maxTables:%d TSDB_MIN_TABLES:%d TSDB_MAX_TABLES:%d", pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); return -1; } @@ -648,7 +648,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { tsdbError( - "id %d: invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " + "vgId:%d, invalid daysPerFile configuration! daysPerFile:%d TSDB_MIN_DAYS_PER_FILE:%d TSDB_MAX_DAYS_PER_FILE:" "%d", pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); return -1; @@ -661,8 +661,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { tsdbError( - "id %d: invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " - "TSDB_MAX_MIN_ROW_FBLOCK %d", + "vgId:%d, invalid minRowsPerFileBlock configuration! minRowsPerFileBlock:%d TSDB_MIN_MIN_ROW_FBLOCK:%d " + "TSDB_MAX_MIN_ROW_FBLOCK:%d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return -1; } @@ -673,8 +673,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { tsdbError( - "id %d: invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " - "TSDB_MAX_MAX_ROW_FBLOCK %d", + "vgId:%d, invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock:%d TSDB_MIN_MAX_ROW_FBLOCK:%d " + "TSDB_MAX_MAX_ROW_FBLOCK:%d", pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return -1; } @@ -688,8 +688,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { tsdbError( - "id %d: invalid keep configuration! keep %d TSDB_MIN_KEEP %d " - "TSDB_MAX_KEEP %d", + "vgId:%d, invalid keep configuration! keep:%d TSDB_MIN_KEEP:%d " + "TSDB_MAX_KEEP:%d", pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); return -1; } @@ -756,13 +756,13 @@ static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) { if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1; if (mkdir(dirName, 0755) < 0) { - tsdbError("id %d: failed to create repository directory! reason %s", pRepo->config.tsdbId, strerror(errno)); + tsdbError("vgId:%d, failed to create repository directory! reason:%s", pRepo->config.tsdbId, strerror(errno)); return -1; } - tsdbError( - "id %d: set up tsdb environment succeed! cacheBlockSize %d, totalBlocks %d, maxTables %d, daysPerFile %d, keep " - "%d, minRowsPerFileBlock %d, maxRowsPerFileBlock %d, precision %d, compression%d", + tsdbTrace( + "vgId:%d, set up tsdb environment succeed! cacheBlockSize:%d, totalBlocks:%d, maxTables:%d, daysPerFile:%d, keep:" + "%d, minRowsPerFileBlock:%d, maxRowsPerFileBlock:%d, precision:%d, compression:%d", pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); return 0; @@ -842,16 +842,19 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); + tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId, + pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row)); + return 0; } -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now) { +static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { STsdbRepo *pRepo = (STsdbRepo *)repo; STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); if (pTable == NULL) { - tsdbError("id %d: failed to get table for insert, uid:%" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid, + tsdbError("vgId:%d, failed to get table for insert, uid:" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid, pBlock->tid); return TSDB_CODE_INVALID_TABLE_ID; } @@ -865,15 +868,16 @@ static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { - tsdbError( - "tsdbId: %d, table tid: %d, talbe uid: %ld timestamp is out of range. now: %ld maxKey: %ld, minKey: %ld", - pRepo->config.tsdbId, pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey); + tsdbError("vgId:%d, table:%s, tid:%d, talbe uid:%ld timestamp is out of range. now:" PRId64 ", maxKey:" PRId64 + ", minKey:" PRId64, + pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey); return TSDB_CODE_TIMESTAMP_OUT_OF_RANGE; } if (tdInsertRowToTable(pRepo, row, pTable) < 0) { return -1; } + (*affectedrows)++; } return TSDB_CODE_SUCCESS; @@ -942,15 +946,16 @@ static void tsdbFreeMemTable(SMemTable *pMemTable) { // Commit to file static void *tsdbCommitData(void *arg) { - printf("Starting to commit....\n"); STsdbRepo * pRepo = (STsdbRepo *)arg; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCache *pCache = pRepo->tsdbCache; STsdbCfg * pCfg = &(pRepo->config); SDataCols * pDataCols = NULL; - SRWHelper whelper = {0}; + SRWHelper whelper = {{0}}; if (pCache->imem == NULL) return NULL; + tsdbPrint("vgId: %d, starting to commit....", pRepo->config.tsdbId); + // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); if (iters == NULL) { @@ -974,6 +979,7 @@ static void *tsdbCommitData(void *arg) { // Do retention actions tsdbFitRetention(pRepo); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); _exit: tdFreeDataCols(pDataCols); @@ -1015,10 +1021,16 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // Create and open files for commit tsdbGetDataDirName(pRepo, dataDir); - if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) goto _err; + if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { + tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); + goto _err; + } // Open files for write/read - if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err; + if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { + tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); + goto _err; + } // Loop to commit data in each table for (int tid = 1; tid < pCfg->maxTables; tid++) { @@ -1055,13 +1067,22 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters ASSERT(pDataCols->numOfPoints == 0); // Move the last block to the new .l file if neccessary - if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) goto _err; + if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { + tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); + goto _err; + } // Write the SCompBlock part - if (tsdbWriteCompInfo(pHelper) < 0) goto _err; + if (tsdbWriteCompInfo(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); + goto _err; + } } - if (tsdbWriteCompIdx(pHelper) < 0) goto _err; + if (tsdbWriteCompIdx(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); + goto _err; + } tsdbCloseHelperFile(pHelper, 0); // TODO: make it atomic with some methods @@ -1104,11 +1125,14 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK } static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { + int8_t oldCompRession = pRepo->config.compression; pRepo->config.compression = compression; + tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression); } static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { STsdbCfg *pCfg = &pRepo->config; + int oldKeep = pCfg->keep; int maxFiles = keep / pCfg->maxTables + 3; if (pRepo->config.keep > keep) { @@ -1120,8 +1144,57 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { } pRepo->tsdbFileH->maxFGroups = maxFiles; } + tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); } static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { // TODO -} \ No newline at end of file + int oldMaxTables = pRepo->config.maxTables; + tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); +} + +uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) { + // TODO: need to refactor this function + + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char fname[256] = "\0"; + + struct stat fState; + char *spath = strdup(pRepo->rootDir); + char *prefixDir = dirname(spath); + + if (name[0] == 0) { + // Map index to the file name + int fid = (*index) / 3; + + if (fid > pFileH->numOfFGroups) { + // return meta data file + if ((*index) % 3 > 0) { // it is finished + tfree(spath); + return 0; + } else { + tsdbGetMetaFileName(pRepo->rootDir, fname); + } + } else { + // return data file name + strcpy(fname, pFileH->fGroup[fid].files[(*index) % 3].fname); + } + strcpy(name, fname + strlen(spath)); + } else { + // Name is provided, need to get the file info + sprintf(fname, "%s/%s", prefixDir, name); + } + + if (stat(fname, &fState) < 0) { + tfree(spath); + return 0; + } + + *size = fState.st_size; + magic = *size; + + return magic; +} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index c7e260ae50..5ecad18443 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -6,7 +6,7 @@ #include "tsdbMain.h" #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here -#define TSDB_META_FILE_NAME "META" +// #define TSDB_META_FILE_NAME "META" const int32_t DEFAULT_TAG_INDEX_COLUMN = 0; // skip list built based on the first column of tags @@ -80,7 +80,7 @@ STable *tsdbDecodeTable(void *cont, int contLen) { T_READ_MEMBER(ptr, int8_t, pTable->type); int len = *(int *)ptr; ptr = (char *)ptr + sizeof(int); - pTable->name = calloc(1, len + VARSTR_HEADER_SIZE); + pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1); if (pTable->name == NULL) return NULL; varDataSetLen(pTable->name, len); @@ -234,7 +234,6 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { } } -// todo refactor table name definition int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t* type, int16_t* bytes, char** val) { STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); @@ -242,6 +241,7 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); STColumn* pCol = NULL; + // todo binary search for(int32_t col = 0; col < schemaNCols(pSchema); ++col) { STColumn* p = schemaColAt(pSchema, col); if (p->colId == colId) { @@ -283,7 +283,10 @@ char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes) { } } -int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { +int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + if (tsdbCheckTableCfg(pCfg) < 0) return -1; STable *super = NULL; @@ -307,7 +310,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { // todo refactor extract method size_t size = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN); - super->name = malloc(size + VARSTR_HEADER_SIZE); + super->name = calloc(1, size + VARSTR_HEADER_SIZE + 1); STR_WITH_SIZE_TO_VARSTR(super->name, pCfg->sname, size); // index the first tag column @@ -336,7 +339,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->tableId = pCfg->tableId; size_t size = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN); - table->name = malloc(size + VARSTR_HEADER_SIZE); + table->name = calloc(1, size + VARSTR_HEADER_SIZE + 1); STR_WITH_SIZE_TO_VARSTR(table->name, pCfg->name, size); table->lastKey = 0; @@ -351,8 +354,14 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { } // Register to meta - if (newSuper) tsdbAddTableToMeta(pMeta, super, true); + if (newSuper) { + tsdbAddTableToMeta(pMeta, super, true); + tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name), + super->tableId.uid); + } tsdbAddTableToMeta(pMeta, table, true); + tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name), + table->tableId.tid, table->tableId.uid); // Write to meta file int bufLen = 0; @@ -385,13 +394,25 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId) { return pTable; } -int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { +// int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { +int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + if (pRepo == NULL) return -1; + + STsdbMeta *pMeta = pRepo->tsdbMeta; if (pMeta == NULL) return -1; STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); - if (pTable == NULL) return -1; + if (pTable == NULL) { + tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId, + tableId.tid, tableId.uid); + return -1; + } + tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), + tableId.tid, tableId.uid); if (tsdbRemoveTableFromMeta(pMeta, pTable) < 0) return -1; + return 0; } @@ -483,7 +504,9 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable) { if (pTable->type == TSDB_SUPER_TABLE) { SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STableIndexElem *pEle = (STableIndexElem *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = pEle->pTable; + ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE); pMeta->tables[tTable->tableId.tid] = NULL; diff --git a/src/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c index 6821fc2d98..f9d10ec579 100644 --- a/src/tsdb/src/tsdbMetaFile.c +++ b/src/tsdb/src/tsdbMetaFile.c @@ -28,7 +28,7 @@ typedef struct { int64_t uid; } SRecordInfo; -static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); +// static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); // static int32_t tsdbCheckMetaHeader(int fd); static int32_t tsdbWriteMetaHeader(int fd); static int tsdbCreateMetaFile(char *fname); @@ -180,7 +180,7 @@ void tsdbCloseMetaFile(SMetaFile *mfh) { tfree(mfh); } -static int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { +int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { if (rootDir == NULL) return -1; sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index e32a646296..829d7ab3fd 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -701,8 +701,15 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa continue; } + memset(pCompCol, 0, sizeof(*pCompCol)); + pCompCol->colId = pDataCol->colId; pCompCol->type = pDataCol->type; + if (tDataTypeDesc[pDataCol->type].getStatisFunc) { + (*tDataTypeDesc[pDataCol->type].getStatisFunc)( + (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max), + &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull)); + } nColsNotAllNull++; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 49b6d2984f..a5cba70219 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -54,12 +54,12 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SFileGroup* fileGroup; int32_t slot; - int32_t sid; + int32_t tid; SArray* pLoadedCols; } SDataBlockLoadInfo; typedef struct SLoadCompBlockInfo { - int32_t sid; /* table sid */ + int32_t tid; /* table tid */ int32_t fileId; int32_t fileListIndex; } SLoadCompBlockInfo; @@ -127,12 +127,12 @@ static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; - pBlockLoadInfo->sid = -1; + pBlockLoadInfo->tid = -1; pBlockLoadInfo->fileGroup = NULL; } static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { - pCompBlockLoadInfo->sid = -1; + pCompBlockLoadInfo->tid = -1; pCompBlockLoadInfo->fileId = -1; pCompBlockLoadInfo->fileListIndex = -1; } @@ -414,6 +414,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheck static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { + STsdbRepo *pRepo = pQueryHandle->pTsdb; SCompData* data = calloc(1, sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols); data->numOfCols = pBlock->numOfCols; @@ -422,8 +423,8 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo bool blockLoaded = false; SArray* sa = getDefaultLoadColumns(pQueryHandle, true); - if (pCheckInfo->pDataCols == NULL) { - pCheckInfo->pDataCols = tdNewDataCols(1000, 100, 4096); //todo fix me + if (pCheckInfo->pDataCols == NULL) { // todo: why not the real data? + pCheckInfo->pDataCols = tdNewDataCols(pRepo->tsdbMeta->maxRowBytes, pRepo->tsdbMeta->maxCols, pRepo->config.maxRowsPerFileBlock); } tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); @@ -433,7 +434,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->sid = pCheckInfo->pTableObj->tableId.tid; + pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; blockLoaded = true; } @@ -613,7 +614,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); int32_t bytes = pCol->info.bytes; - for (int32_t j = 0; j < numOfCols; ++j) { + for (int32_t j = 0; j < numOfCols; ++j) { //todo opt performance SDataCol* src = &pQueryHandle->rhelper.pDataCols[0]->cols[j]; if (pCol->info.colId == src->colId) { @@ -1196,7 +1197,9 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } else { // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; - if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->sid == pCheckInfo->pTableObj->tableId.tid) { + + if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid && + pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { SCompBlock* pBlock = pBlockInfoEx->pBlock.compBlock; @@ -1346,7 +1349,7 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { int32_t type = 0; int32_t bytes = 0; - if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor extract method , to queryExecutor to generate tags values f1 = (char*) pTable1->name; f2 = (char*) pTable2->name; type = TSDB_DATA_TYPE_BINARY; @@ -1354,7 +1357,8 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { } else { STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); bytes = pCol->bytes; - + type = pCol->type; + f1 = tdGetRowDataOfCol(pTable1->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); f2 = tdGetRowDataOfCol(pTable2->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); } @@ -1432,7 +1436,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -bool tSkipListNodeFilterCallback(const void* pNode, void* param) { +bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -1497,7 +1501,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .fp = (__result_filter_fn_t) tSkipListNodeFilterCallback, + .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; @@ -1521,7 +1525,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagC } if (pTable->type != TSDB_SUPER_TABLE) { - uError("%p query normal tag not allowed, uid:%, tid:%d, name:%s" PRIu64, + uError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, pTable->name); return TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client diff --git a/src/util/inc/exception.h b/src/util/inc/exception.h index 41f01d68dd..52cd03d830 100644 --- a/src/util/inc/exception.h +++ b/src/util/inc/exception.h @@ -73,6 +73,7 @@ void cleanupPush_void_ptr_bool ( bool failOnly, void* func, void* arg1, bool ar void cleanupPush_void_ptr ( bool failOnly, void* func, void* arg ); void cleanupPush_int_int ( bool failOnly, void* func, int arg ); void cleanupPush_void ( bool failOnly, void* func ); +void cleanupPush_int_ptr ( bool failOnly, void* func, void* arg ); int32_t cleanupGetActionCount(); void cleanupExecuteTo( int32_t anchor, bool failed ); @@ -83,8 +84,10 @@ void cleanupExecute( SExceptionNode* node, bool failed ); #define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) ) #define CLEANUP_PUSH_INT_INT( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) ) #define CLEANUP_PUSH_VOID( failOnly, func ) cleanupPush_void( (failOnly), (void*)(func) ) +#define CLEANUP_PUSH_INT_PTR( failOnly, func, arg ) cleanupPush_int_ptr( (failOnly), (void*)(func), (void*)(arg) ) #define CLEANUP_PUSH_FREE( failOnly, arg ) cleanupPush_void_ptr( (failOnly), free, (void*)(arg) ) #define CLEANUP_PUSH_CLOSE( failOnly, arg ) cleanupPush_int_int( (failOnly), close, (int)(arg) ) +#define CLEANUP_PUSH_FCLOSE( failOnly, arg ) cleanupPush_int_ptr( (failOnly), fclose, (void*)(arg) ) #define CLEANUP_GET_ANCHOR() cleanupGetActionCount() #define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) ) @@ -95,7 +98,7 @@ void cleanupExecute( SExceptionNode* node, bool failed ); void exceptionPushNode( SExceptionNode* node ); int32_t exceptionPopNode(); -void exceptionThrow( int code ); +void exceptionThrow( int32_t code ); #define TRY(maxCleanupActions) do { \ SExceptionNode exceptionNode = { 0 }; \ @@ -106,10 +109,10 @@ void exceptionThrow( int code ); int caughtException = setjmp( exceptionNode.jb ); \ if( caughtException == 0 ) -#define CATCH( code ) int code = exceptionPopNode(); \ +#define CATCH( code ) int32_t code = exceptionPopNode(); \ if( caughtException == 1 ) -#define FINALLY( code ) int code = exceptionPopNode(); +#define FINALLY( code ) int32_t code = exceptionPopNode(); #define END_TRY } while( 0 ); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 1bbc8dcf5c..0a2cbe8baf 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -94,7 +94,7 @@ size_t taosHashGetSize(const SHashObj *pHashObj); * @param size * @return */ -int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *data, size_t size); +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); /** * return the payload data with the specified key @@ -104,7 +104,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *da * @param keyLen * @return */ -void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen); +void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); /** * remove item with the specified key @@ -112,7 +112,7 @@ void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen); * @param key * @param keyLen */ -void taosHashRemove(SHashObj *pHashObj, const char *key, size_t keyLen); +void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); /** * clean up hash table diff --git a/src/util/inc/ihash.h b/src/util/inc/ihash.h deleted file mode 100644 index f283abe737..0000000000 --- a/src/util/inc/ihash.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_IHASH_H -#define TDENGINE_IHASH_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void *, uint64_t)); - -void taosCleanUpIntHash(void *handle); - -char *taosGetIntHashData(void *handle, uint64_t key); - -void taosDeleteIntHash(void *handle, uint64_t key); - -char *taosAddIntHash(void *handle, uint64_t key, char *pData); - -int32_t taosHashInt(void *handle, uint64_t key); - -void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)); - -void taosVisitIntHashWithFp(void *handle, void (*fp)(char *, void *), void *param); - -int32_t taosGetIntHashSize(void *handle); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_IHASH_H diff --git a/src/util/inc/shash.h b/src/util/inc/shash.h deleted file mode 100644 index 2651a6b349..0000000000 --- a/src/util/inc/shash.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TSHASH_H -#define TDENGINE_TSHASH_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -void *taosInitStrHash(uint32_t maxSessions, uint32_t dataSize, uint32_t (*fp)(void *, char *)); - -void taosCleanUpStrHash(void *handle); - -void *taosGetStrHashData(void *handle, char *string); - -void taosDeleteStrHash(void *handle, char *string); - -void taosDeleteStrHashNode(void *handle, char *string, void *pDeleteNode); - -void *taosAddStrHash(void *handle, char *string, char *pData); - -void *taosAddStrHashWithSize(void *handle, char *string, char *pData, int dataSize); - -uint32_t taosHashString(void *handle, char *string); - -uint32_t taosHashStringStep1(void *handle, char *string); - -char *taosVisitStrHashWithFp(void *handle, int (*fp)(char *)); - -void taosCleanUpStrHashWithFp(void *handle, void (*fp)(char *)); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TSHASH_H diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 7edd032034..866bde0938 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -106,6 +106,12 @@ void taosArrayCopy(SArray* pDst, const SArray* pSrc); */ SArray* taosArrayClone(const SArray* pSrc); +/** + * clear the array (remove all element) + * @param pArray + */ +void taosArrayClear(SArray* pArray); + /** * destroy array list * @param pArray diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 2e26f8f0e3..6dd707b763 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -67,6 +67,7 @@ typedef struct { void * pTimer; SCacheStatis statistics; SHashObj * pHashTable; + _hash_free_fn_t freeFp; int numOfElemsInTrash; // number of element in trash int16_t deleting; // set the deleting flag to stop refreshing ASAP. T_REF_DECLARE() @@ -88,6 +89,7 @@ typedef struct { * @return */ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTimeInSeconds); +SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTimeInSeconds, void (*freeCb)(void *data)); /** * add data into cache diff --git a/src/util/inc/thash.h b/src/util/inc/thash.h deleted file mode 100644 index 592db62fe5..0000000000 --- a/src/util/inc/thash.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_THASH_H -#define TDENGINE_THASH_H - -#ifdef __cplusplus -extern "C" { -#endif - -void *taosOpenHash(int maxSessions, int (*fp)(void *, uint64_t)); - -void taosCloseHash(void *handle); - -int taosAddHash(void *handle, uint64_t, uint32_t id); - -void taosDeleteHash(void *handle, uint64_t); - -int32_t taosGetIdFromHash(void *handle, uint64_t); - -int taosHashLong(void *, uint64_t ip); - -uint64_t taosHashUInt64(uint64_t handle); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_THASH_H diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 759ecbb7ed..686e5ab313 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -174,7 +174,7 @@ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSiz SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); /** - * get only *one* node of which key is equalled to pKey, even there are more than one nodes are of the same key + * get *all* nodes which key are equivalent to pKey * * @param pSkipList * @param pKey @@ -234,14 +234,13 @@ SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); void *tSkipListDestroyIter(SSkipListIterator *iter); /* - * remove only one node of the pKey value. - * If more than one node has the same value, any one will be removed + * remove nodes of the pKey value. + * If more than one node has the same value, all will be removed * * @Return - * true: one node has been removed - * false: no node has been removed + * the count of removed nodes */ -bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); /* * remove the specified node in parameters diff --git a/src/util/src/exception.c b/src/util/src/exception.c index 7f8f91c784..e1f6516fe7 100644 --- a/src/util/src/exception.c +++ b/src/util/src/exception.c @@ -1,7 +1,8 @@ +#include "os.h" #include "exception.h" -static _Thread_local SExceptionNode* expList; +static threadlocal SExceptionNode* expList; void exceptionPushNode( SExceptionNode* node ) { node->prev = expList; @@ -14,7 +15,7 @@ int32_t exceptionPopNode() { return node->code; } -void exceptionThrow( int code ) { +void exceptionThrow( int32_t code ) { expList->code = code; longjmp( expList->jb, 1 ); } @@ -38,21 +39,27 @@ static void cleanupWrapper_void_ptr( SCleanupAction* ca ) { static void cleanupWrapper_int_int( SCleanupAction* ca ) { int (*func)( int ) = ca->func; - func( (int)(intptr_t)(ca->arg1.Int) ); + func( ca->arg1.Int ); } -static void cleanupWrapper_void_void( SCleanupAction* ca ) { +static void cleanupWrapper_void( SCleanupAction* ca ) { void (*func)() = ca->func; func(); } +static void cleanupWrapper_int_ptr( SCleanupAction* ca ) { + int (*func)( void* ) = ca->func; + func( ca->arg1.Ptr ); +} + typedef void (*wrapper)(SCleanupAction*); static wrapper wrappers[] = { cleanupWrapper_void_ptr_ptr, cleanupWrapper_void_ptr_bool, cleanupWrapper_void_ptr, cleanupWrapper_int_int, - cleanupWrapper_void_void, + cleanupWrapper_void, + cleanupWrapper_int_ptr, }; @@ -107,6 +114,15 @@ void cleanupPush_void( bool failOnly, void* func ) { ca->func = func; } +void cleanupPush_int_ptr( bool failOnly, void* func, void* arg ) { + assert( expList->numCleanupAction < expList->maxCleanupAction ); + + SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 5; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} int32_t cleanupGetActionCount() { @@ -118,8 +134,9 @@ static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed while( node->numCleanupAction > anchor ) { --node->numCleanupAction; SCleanupAction *ca = node->cleanupActions + node->numCleanupAction; - if( failed || !(ca->failOnly) ) + if( failed || !(ca->failOnly) ) { wrappers[ca->wrapper]( ca ); + } } } @@ -129,4 +146,4 @@ void cleanupExecuteTo( int32_t anchor, bool failed ) { void cleanupExecute( SExceptionNode* node, bool failed ) { doExecuteCleanup( node, 0, failed ); -} \ No newline at end of file +} diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 37e8123170..12f7b43b79 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -17,7 +17,6 @@ #include "hash.h" #include "tulog.h" -#include "ttime.h" #include "tutil.h" static FORCE_INLINE void __wr_lock(void *lock) { @@ -90,154 +89,65 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { /** * inplace update node in hash table - * @param pHashObj hash table object - * @param pNode data node + * @param pHashObj hash table object + * @param pNode hash data node */ -static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) { - if (pNode->prev1) { - pNode->prev1->next = pNode; - } - - if (pNode->next) { - (pNode->next)->prev = pNode; - } -} +static void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode); /** - * get SHashNode from hashlist, nodes from trash are not included. + * Get SHashNode from hashlist, nodes from trash are not included. * @param pHashObj Cache objection * @param key key for hash * @param keyLen key length + * @param hashVal hash value by hash function * @return */ -static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const char *key, uint32_t keyLen, uint32_t *hashVal) { - uint32_t hash = (*pHashObj->hashFp)(key, keyLen); - - int32_t slot = HASH_INDEX(hash, pHashObj->capacity); - SHashEntry *pEntry = pHashObj->hashList[slot]; - - SHashNode *pNode = pEntry->next; - while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { - break; - } - - pNode = pNode->next; - } - - if (pNode) { - assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot); - } - - // return the calculated hash value, to avoid calculating it again in other functions - if (hashVal != NULL) { - *hashVal = hash; - } - - return pNode; -} +static SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal); /** - * resize the hash list if the threshold is reached + * Resize the hash list if the threshold is reached * * @param pHashObj */ -static void taosHashTableResize(SHashObj *pHashObj) { - if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) { - return; - } - - // double the original capacity - SHashNode *pNode = NULL; - SHashNode *pNext = NULL; - - int32_t newSize = pHashObj->capacity << 1u; - if (newSize > HASH_MAX_CAPACITY) { -// uTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); - return; - } - -// int64_t st = taosGetTimestampUs(); - - SHashEntry **pNewEntry = realloc(pHashObj->hashList, sizeof(SHashEntry *) * newSize); - if (pNewEntry == NULL) { -// uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); - return; - } - - pHashObj->hashList = pNewEntry; - for (int32_t i = pHashObj->capacity; i < newSize; ++i) { - pHashObj->hashList[i] = calloc(1, sizeof(SHashEntry)); - } - - pHashObj->capacity = newSize; - - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - - pNode = pEntry->next; - if (pNode != NULL) { - assert(pNode->prev1 == pEntry && pEntry->num > 0); - } - - while (pNode) { - int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j == i) { // this key resides in the same slot, no need to relocate it - pNode = pNode->next; - } else { - pNext = pNode->next; - - // remove from current slot - assert(pNode->prev1 != NULL); - - if (pNode->prev1 == pEntry) { // first node of the overflow linked list - pEntry->next = pNode->next; - } else { - pNode->prev->next = pNode->next; - } - - pEntry->num--; - assert(pEntry->num >= 0); - - if (pNode->next != NULL) { - (pNode->next)->prev = pNode->prev; - } - - // added into new slot - pNode->next = NULL; - pNode->prev1 = NULL; - - SHashEntry *pNewIndexEntry = pHashObj->hashList[j]; - - if (pNewIndexEntry->next != NULL) { - assert(pNewIndexEntry->next->prev1 == pNewIndexEntry); - - pNewIndexEntry->next->prev = pNode; - } - - pNode->next = pNewIndexEntry->next; - pNode->prev1 = pNewIndexEntry; - - pNewIndexEntry->next = pNode; - pNewIndexEntry->num++; - - // continue - pNode = pNext; - } - } - } - -// int64_t et = taosGetTimestampUs(); -// uTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); -} +static void taosHashTableResize(SHashObj *pHashObj); /** - * @param capacity maximum slots available for hash elements - * @param fn hash function + * @param key key of object for hash, usually a null-terminated string + * @param keyLen length of key + * @param pData actually data. Requires a consecutive memory block, no pointer is allowed in pData. + * Pointer copy causes memory access error. + * @param dsize size of data + * @return SHashNode + */ +static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal); + +/** + * Update the hash node + * + * @param pNode hash node + * @param key key for generate hash value + * @param keyLen key length + * @param pData actual data + * @param dsize size of actual data + * @return hash node + */ +static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize); + +/** + * insert the hash node at the front of the linked list + * + * @param pHashObj + * @param pNode + */ +static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); + +/** + * Get the next element in hash table for iterator + * @param pIter * @return */ +static SHashNode *getNextHashNode(SHashMutableIterator *pIter); + SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { if (capacity == 0 || fn == NULL) { return NULL; @@ -285,79 +195,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { return pHashObj; } -/** - * @param key key of object for hash, usually a null-terminated string - * @param keyLen length of key - * @param pData actually data. required a consecutive memory block, no pointer is allowed - * in pData. Pointer copy causes memory access error. - * @param size size of block - * @return SHashNode - */ -static SHashNode *doCreateHashNode(const char *key, size_t keyLen, const char *pData, size_t dataSize, - uint32_t hashVal) { - size_t totalSize = dataSize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null - - SHashNode *pNewNode = calloc(1, totalSize); - if (pNewNode == NULL) { - uError("failed to allocate memory, reason:%s", strerror(errno)); - return NULL; - } - - memcpy(pNewNode->data, pData, dataSize); - - pNewNode->key = pNewNode->data + dataSize; - memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = keyLen; - - pNewNode->hashVal = hashVal; - - return pNewNode; -} - -static SHashNode *doUpdateHashNode(SHashNode *pNode, const char *key, size_t keyLen, const char *pData, - size_t dataSize) { - size_t size = dataSize + sizeof(SHashNode) + keyLen; - - SHashNode *pNewNode = (SHashNode *)realloc(pNode, size); - if (pNewNode == NULL) { - return NULL; - } - - memcpy(pNewNode->data, pData, dataSize); - - pNewNode->key = pNewNode->data + dataSize; - - assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen); - - memcpy(pNewNode->key, key, keyLen); - return pNewNode; -} - -/** - * insert the hash node at the front of the linked list - * - * @param pHashObj - * @param pNode - */ -static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) { - assert(pNode != NULL); - - int32_t index = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - SHashEntry *pEntry = pHashObj->hashList[index]; - - pNode->next = pEntry->next; - - if (pEntry->next) { - pEntry->next->prev = pNode; - } - - pEntry->next = pNode; - pNode->prev1 = pEntry; - - pEntry->num++; - pHashObj->size++; -} - size_t taosHashGetSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; @@ -366,12 +203,7 @@ size_t taosHashGetSize(const SHashObj *pHashObj) { return pHashObj->size; } -/** - * add data node into hash table - * @param pHashObj hash object - * @param pNode hash node - */ -int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *data, size_t size) { +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { __wr_lock(pHashObj->lock); uint32_t hashVal = 0; @@ -402,7 +234,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const char *key, size_t keyLen, void *da return 0; } -void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen) { +void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { __rd_lock(pHashObj->lock); uint32_t hashVal = 0; @@ -419,12 +251,7 @@ void *taosHashGet(SHashObj *pHashObj, const char *key, size_t keyLen) { } } -/** - * remove node in hash list - * @param pHashObj - * @param pNode - */ -void taosHashRemove(SHashObj *pHashObj, const char *key, size_t keyLen) { +void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { __wr_lock(pHashObj->lock); uint32_t val = 0; @@ -518,23 +345,6 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { return pIter; } -static SHashNode *getNextHashNode(SHashMutableIterator *pIter) { - assert(pIter != NULL); - - pIter->entryIndex++; - while (pIter->entryIndex < pIter->pHashObj->capacity) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->next == NULL) { - pIter->entryIndex++; - continue; - } - - return pEntry->next; - } - - return NULL; -} - bool taosHashIterNext(SHashMutableIterator *pIter) { if (pIter == NULL) { return false; @@ -617,3 +427,205 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { return num; } + +void doUpdateHashTable(SHashObj *pHashObj, SHashNode *pNode) { + if (pNode->prev1) { + pNode->prev1->next = pNode; + } + + if (pNode->next) { + (pNode->next)->prev = pNode; + } +} + +SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) { + uint32_t hash = (*pHashObj->hashFp)(key, keyLen); + + int32_t slot = HASH_INDEX(hash, pHashObj->capacity); + SHashEntry *pEntry = pHashObj->hashList[slot]; + + SHashNode *pNode = pEntry->next; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + break; + } + + pNode = pNode->next; + } + + if (pNode) { + assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot); + } + + // return the calculated hash value, to avoid calculating it again in other functions + if (hashVal != NULL) { + *hashVal = hash; + } + + return pNode; +} + +void taosHashTableResize(SHashObj *pHashObj) { + if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) { + return; + } + + // double the original capacity + SHashNode *pNode = NULL; + SHashNode *pNext = NULL; + + int32_t newSize = pHashObj->capacity << 1u; + if (newSize > HASH_MAX_CAPACITY) { +// uTrace("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", +// pHashObj->capacity, HASH_MAX_CAPACITY); + return; + } + +// int64_t st = taosGetTimestampUs(); + + SHashEntry **pNewEntry = realloc(pHashObj->hashList, sizeof(SHashEntry *) * newSize); + if (pNewEntry == NULL) { +// uTrace("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + return; + } + + pHashObj->hashList = pNewEntry; + for (int32_t i = pHashObj->capacity; i < newSize; ++i) { + pHashObj->hashList[i] = calloc(1, sizeof(SHashEntry)); + } + + pHashObj->capacity = newSize; + + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + + pNode = pEntry->next; + if (pNode != NULL) { + assert(pNode->prev1 == pEntry && pEntry->num > 0); + } + + while (pNode) { + int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); + if (j == i) { // this key resides in the same slot, no need to relocate it + pNode = pNode->next; + } else { + pNext = pNode->next; + + // remove from current slot + assert(pNode->prev1 != NULL); + + if (pNode->prev1 == pEntry) { // first node of the overflow linked list + pEntry->next = pNode->next; + } else { + pNode->prev->next = pNode->next; + } + + pEntry->num--; + assert(pEntry->num >= 0); + + if (pNode->next != NULL) { + (pNode->next)->prev = pNode->prev; + } + + // added into new slot + pNode->next = NULL; + pNode->prev1 = NULL; + + SHashEntry *pNewIndexEntry = pHashObj->hashList[j]; + + if (pNewIndexEntry->next != NULL) { + assert(pNewIndexEntry->next->prev1 == pNewIndexEntry); + + pNewIndexEntry->next->prev = pNode; + } + + pNode->next = pNewIndexEntry->next; + pNode->prev1 = pNewIndexEntry; + + pNewIndexEntry->next = pNode; + pNewIndexEntry->num++; + + // continue + pNode = pNext; + } + } + } + +// int64_t et = taosGetTimestampUs(); +// uTrace("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, +// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); +} + +SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { + size_t totalSize = dsize + sizeof(SHashNode) + keyLen + 1; // one extra byte for null + + SHashNode *pNewNode = calloc(1, totalSize); + if (pNewNode == NULL) { + uError("failed to allocate memory, reason:%s", strerror(errno)); + return NULL; + } + + memcpy(pNewNode->data, pData, dsize); + + pNewNode->key = pNewNode->data + dsize; + memcpy(pNewNode->key, key, keyLen); + pNewNode->keyLen = keyLen; + + pNewNode->hashVal = hashVal; + + return pNewNode; +} + +SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize) { + size_t size = dsize + sizeof(SHashNode) + keyLen; + + SHashNode *pNewNode = (SHashNode *)realloc(pNode, size); + if (pNewNode == NULL) { + return NULL; + } + + memcpy(pNewNode->data, pData, dsize); + + pNewNode->key = pNewNode->data + dsize; + + assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen); + + memcpy(pNewNode->key, key, keyLen); + return pNewNode; +} + +void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) { + assert(pNode != NULL); + + int32_t index = HASH_INDEX(pNode->hashVal, pHashObj->capacity); + SHashEntry *pEntry = pHashObj->hashList[index]; + + pNode->next = pEntry->next; + + if (pEntry->next) { + pEntry->next->prev = pNode; + } + + pEntry->next = pNode; + pNode->prev1 = pEntry; + + pEntry->num++; + pHashObj->size++; +} + +SHashNode *getNextHashNode(SHashMutableIterator *pIter) { + assert(pIter != NULL); + + pIter->entryIndex++; + while (pIter->entryIndex < pIter->pHashObj->capacity) { + SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + if (pEntry->next == NULL) { + pIter->entryIndex++; + continue; + } + + return pEntry->next; + } + + return NULL; +} diff --git a/src/util/src/hashint.c b/src/util/src/hashint.c deleted file mode 100644 index 87a6da4fa8..0000000000 --- a/src/util/src/hashint.c +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "tmempool.h" -#include "taosdef.h" - -typedef struct _long_hash_t { - uint32_t key; - int hash; - struct _long_hash_t *prev; - struct _long_hash_t *next; - char data[]; -} SLongHash; - -typedef struct { - SLongHash **longHashList; - mpool_h longHashMemPool; - int maxSessions; - int dataSize; -} SHashObj; - -int sdbHashLong(void *handle, uint32_t ip) { - SHashObj *pObj = (SHashObj *)handle; - int hash = 0; - - hash = ip >> 16; - hash += (ip & 0xFFFF); - - hash = hash % pObj->maxSessions; - - return hash; -} - -void *sdbAddIntHash(void *handle, void *pKey, void *data) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - uint32_t key = *((uint32_t *)pKey); - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = sdbHashLong(pObj, key); - pNode = (SLongHash *)taosMemPoolMalloc(pObj->longHashMemPool); - pNode->key = key; - memcpy(pNode->data, data, pObj->dataSize); - pNode->prev = 0; - pNode->next = pObj->longHashList[hash]; - pNode->hash = hash; - - if (pObj->longHashList[hash] != 0) (pObj->longHashList[hash])->prev = pNode; - pObj->longHashList[hash] = pNode; - - return pObj; -} - -void sdbDeleteIntHash(void *handle, void *pKey) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - uint32_t key = *((uint32_t *)pKey); - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - hash = sdbHashLong(pObj, key); - - pNode = pObj->longHashList[hash]; - while (pNode) { - if (pNode->key == key) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->longHashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - taosMemPoolFree(pObj->longHashMemPool, (char *)pNode); - } -} - -void *sdbGetIntHashData(void *handle, void *pKey) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - uint32_t key = *((uint32_t *)pKey); - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = sdbHashLong(pObj, key); - pNode = pObj->longHashList[hash]; - - while (pNode) { - if (pNode->key == key) { - break; - } - pNode = pNode->next; - } - - if (pNode) return pNode->data; - - return NULL; -} - -void *sdbOpenIntHash(int maxSessions, int dataSize) { - SLongHash **longHashList; - mpool_h longHashMemPool; - SHashObj * pObj; - - longHashMemPool = taosMemPoolInit(maxSessions, sizeof(SLongHash) + dataSize); - if (longHashMemPool == 0) return NULL; - - longHashList = calloc(sizeof(SLongHash *), maxSessions); - if (longHashList == 0) { - taosMemPoolCleanUp(longHashMemPool); - return NULL; - } - - pObj = malloc(sizeof(SHashObj)); - if (pObj == NULL) { - taosMemPoolCleanUp(longHashMemPool); - free(longHashList); - return NULL; - } - - pObj->maxSessions = maxSessions; - pObj->longHashMemPool = longHashMemPool; - pObj->longHashList = longHashList; - pObj->dataSize = dataSize; - - return pObj; -} - -void sdbCloseIntHash(void *handle) { - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - if (pObj->longHashMemPool) taosMemPoolCleanUp(pObj->longHashMemPool); - - if (pObj->longHashList) free(pObj->longHashList); - - memset(pObj, 0, sizeof(SHashObj)); - free(pObj); -} - -void *sdbFetchIntHashData(void *handle, void *ptr, void **ppMeta) { - SHashObj * pObj = (SHashObj *)handle; - SLongHash *pNode = (SLongHash *)ptr; - int hash = 0; - - *ppMeta = NULL; - if (pObj == NULL || pObj->maxSessions <= 0) return NULL; - if (pObj->longHashList == NULL) return NULL; - - if (pNode) { - hash = pNode->hash + 1; - pNode = pNode->next; - } - - if (pNode == NULL) { - for (int i = hash; i < pObj->maxSessions; ++i) { - pNode = pObj->longHashList[i]; - if (pNode) break; - } - } - - if (pNode) *ppMeta = pNode->data; - - return pNode; -} diff --git a/src/util/src/hashstr.c b/src/util/src/hashstr.c deleted file mode 100644 index d87e605176..0000000000 --- a/src/util/src/hashstr.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" - -#define MAX_STR_LEN 40 - -typedef struct _str_node_t { - char string[TSDB_TABLE_ID_LEN]; - int hash; - struct _str_node_t *prev; - struct _str_node_t *next; - char data[]; -} SHashNode; - -typedef struct { - SHashNode **hashList; - int maxSessions; - int dataSize; -} SHashObj; - -int sdbHashString(void *handle, char *string) { - SHashObj * pObj = (SHashObj *)handle; - unsigned int hash = 0, hashv; - char * c; - int len = strlen(string); - - c = string; - - while (len >= 4) { - hash += *((int *)c); - c += 4; - len -= 4; - } - - while (len > 0) { - hash += *c; - c++; - len--; - } - - hashv = hash / pObj->maxSessions; - hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions; - - return hash; -} - -void *sdbAddStrHash(void *handle, void *key, void *pData) { - int hash; - SHashNode *pNode; - SHashObj * pObj; - char * string = (char *)key; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = sdbHashString(pObj, string); - - int size = sizeof(SHashNode) + pObj->dataSize; - pNode = (SHashNode *)malloc(size); - memset(pNode, 0, size); - strcpy(pNode->string, string); - memcpy(pNode->data, pData, pObj->dataSize); - pNode->prev = 0; - pNode->next = pObj->hashList[hash]; - pNode->hash = hash; - - if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode; - pObj->hashList[hash] = pNode; - - return pNode->data; -} - -void sdbDeleteStrHash(void *handle, void *key) { - int hash; - SHashNode *pNode; - SHashObj * pObj; - char * string = (char *)key; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - hash = sdbHashString(pObj, string); - pNode = pObj->hashList[hash]; - while (pNode) { - if (strcmp(pNode->string, string) == 0) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->hashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - memset(pNode, 0, sizeof(SHashNode)); - free(pNode); - } -} - -void *sdbGetStrHashData(void *handle, void *key) { - int hash; - SHashNode *pNode; - SHashObj * pObj; - char * string = (char *)key; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = sdbHashString(pObj, string); - pNode = pObj->hashList[hash]; - - while (pNode) { - if (strcmp(pNode->string, string) == 0) { - break; - } - pNode = pNode->next; - } - - if (pNode) return pNode->data; - - return NULL; -} - -void *sdbOpenStrHash(int maxSessions, int dataSize) { - SHashObj *pObj; - - pObj = (SHashObj *)malloc(sizeof(SHashObj)); - if (pObj == NULL) { - return NULL; - } - - memset(pObj, 0, sizeof(SHashObj)); - pObj->maxSessions = maxSessions; - pObj->dataSize = dataSize; - - pObj->hashList = (SHashNode **)malloc(sizeof(SHashNode *) * maxSessions); - if (pObj->hashList == NULL) { - free(pObj); - return NULL; - } - memset(pObj->hashList, 0, sizeof(SHashNode *) * maxSessions); - - return (void *)pObj; -} - -void sdbCloseStrHash(void *handle) { - SHashObj *pObj; - SHashNode *pNode, *pNext; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return; - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - free(pNode); - pNode = pNext; - } - } - - free(pObj->hashList); - } - - memset(pObj, 0, sizeof(SHashObj)); - free(pObj); -} - -void *sdbFetchStrHashData(void *handle, void *ptr, void **ppMeta) { - SHashObj *pObj = (SHashObj *)handle; - SHashNode *pNode = (SHashNode *)ptr; - int hash = 0; - - *ppMeta = NULL; - if (pObj == NULL || pObj->maxSessions <= 0) return NULL; - if (pObj->hashList == NULL) return NULL; - - if (pNode) { - hash = pNode->hash + 1; - pNode = pNode->next; - } - - if (pNode == NULL) { - for (int i = hash; i < pObj->maxSessions; ++i) { - pNode = pObj->hashList[i]; - if (pNode) break; - } - } - - if (pNode) *ppMeta = pNode->data; - - return pNode; -} diff --git a/src/util/src/ihash.c b/src/util/src/ihash.c deleted file mode 100644 index 2cfadad964..0000000000 --- a/src/util/src/ihash.c +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -typedef struct _str_node_t { - uint64_t key; - struct _str_node_t *prev; - struct _str_node_t *next; - char data[]; -} IHashNode; - -typedef struct { - IHashNode **hashList; - int64_t *lockedBy; - int32_t maxSessions; - int32_t dataSize; - int32_t (*hashFp)(void *, uint64_t key); -} IHashObj; - -int32_t taosHashInt(void *handle, uint64_t key) { - IHashObj *pObj = (IHashObj *)handle; - int32_t hash = key % pObj->maxSessions; - return hash; -} - -static void taosLockIntHash(IHashObj *pObj, int hash); -static void taosUnlockIntHash(IHashObj *pObj, int hash); - -char *taosAddIntHash(void *handle, uint64_t key, char *pData) { - int32_t hash; - IHashNode *pNode; - IHashObj * pObj; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = (*pObj->hashFp)(pObj, key); - - pNode = (IHashNode *)malloc(sizeof(IHashNode) + (size_t)pObj->dataSize); - if (pNode == NULL) - return NULL; - - taosLockIntHash(pObj, hash); - - pNode->key = key; - if (pData != NULL) { - memcpy(pNode->data, pData, (size_t)pObj->dataSize); - } - pNode->prev = 0; - pNode->next = pObj->hashList[hash]; - - if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode; - pObj->hashList[hash] = pNode; - - taosUnlockIntHash(pObj, hash); - - return (char *)pNode->data; -} - -void taosDeleteIntHash(void *handle, uint64_t key) { - int32_t hash; - IHashNode *pNode; - IHashObj * pObj; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - hash = (*(pObj->hashFp))(pObj, key); - - taosLockIntHash(pObj, hash); - - pNode = pObj->hashList[hash]; - while (pNode) { - if (pNode->key == key) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->hashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - free(pNode); - } - - taosUnlockIntHash(pObj, hash); -} - -char *taosGetIntHashData(void *handle, uint64_t key) { - int32_t hash; - IHashNode *pNode; - IHashObj * pObj; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - - hash = (*pObj->hashFp)(pObj, key); - - taosLockIntHash(pObj, hash); - - pNode = pObj->hashList[hash]; - - while (pNode) { - if (pNode->key == key) { - break; - } - - pNode = pNode->next; - } - - taosUnlockIntHash(pObj, hash); - - if (pNode) return pNode->data; - - return NULL; -} - -void *taosInitIntHash(int32_t maxSessions, int32_t dataSize, int32_t (*fp)(void *, uint64_t)) { - IHashObj *pObj; - - pObj = (IHashObj *)malloc(sizeof(IHashObj)); - if (pObj == NULL) { - return NULL; - } - - memset(pObj, 0, sizeof(IHashObj)); - pObj->maxSessions = maxSessions; - pObj->dataSize = dataSize; - pObj->hashFp = fp; - - pObj->hashList = (IHashNode **)malloc(sizeof(IHashNode *) * (size_t)maxSessions); - if (pObj->hashList == NULL) { - free(pObj); - return NULL; - } - memset(pObj->hashList, 0, sizeof(IHashNode *) * (size_t)maxSessions); - - pObj->lockedBy = (int64_t *)calloc(sizeof(int64_t), maxSessions); - if (pObj->lockedBy == NULL) { - free(pObj); - free(pObj->hashList); - pObj = NULL; - } - - return pObj; -} - -void taosCleanUpIntHash(void *handle) { - IHashObj * pObj; - IHashNode *pNode, *pNext; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return; - - if (pObj->hashList) { - for (int32_t i = 0; i < pObj->maxSessions; ++i) { - taosLockIntHash(pObj, i); - - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - free(pNode); - pNode = pNext; - } - - taosUnlockIntHash(pObj, i); - } - - free(pObj->hashList); - } - - free(pObj->lockedBy); - free(pObj); -} - -void taosCleanUpIntHashWithFp(void *handle, void (*fp)(char *)) { - IHashObj * pObj; - IHashNode *pNode, *pNext; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return; - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - taosLockIntHash(pObj, i); - - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - if (fp != NULL) (*fp)(pNode->data); - free(pNode); - pNode = pNext; - } - - taosUnlockIntHash(pObj, i); - } - - free(pObj->hashList); - } - - memset(pObj, 0, sizeof(IHashObj)); - free(pObj); -} - -void taosVisitIntHashWithFp(void *handle, int (*fp)(char *, void *), void *param) { - IHashObj * pObj; - IHashNode *pNode, *pNext; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return; - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - taosLockIntHash(pObj, i); - - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - (*fp)(pNode->data, param); - pNode = pNext; - } - - taosUnlockIntHash(pObj, i); - } - } -} - -int32_t taosGetIntHashSize(void *handle) { - IHashObj * pObj; - IHashNode *pNode, *pNext; - int32_t num = 0; - - pObj = (IHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return 0; - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - taosLockIntHash(pObj, i); - - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - num++; - pNode = pNext; - } - - taosUnlockIntHash(pObj, i); - } - } - - return num; -} - -static void taosLockIntHash(IHashObj *pObj, int hash) { - int64_t tid = taosGetPthreadId(); - int i = 0; - while (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), 0, tid) != 0) { - if (++i % 1000 == 0) { - sched_yield(); - } - } -} - -static void taosUnlockIntHash(IHashObj *pObj, int hash) { - int64_t tid = taosGetPthreadId(); - if (atomic_val_compare_exchange_64(&(pObj->lockedBy[hash]), tid, 0) != tid) { - assert(false); - } -} - diff --git a/src/util/src/shash.c b/src/util/src/shash.c deleted file mode 100644 index 807ba58951..0000000000 --- a/src/util/src/shash.c +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "shash.h" -#include "tulog.h" - -typedef struct _str_node_t { - char * string; - struct _str_node_t *prev; - struct _str_node_t *next; - char data[]; -} SHashNode; - -typedef struct { - SHashNode **hashList; - uint32_t maxSessions; - uint32_t dataSize; - uint32_t (*hashFp)(void *, char *string); - pthread_mutex_t mutex; -} SHashObj; - -uint32_t taosHashString(void *handle, char *string) { - SHashObj *pObj = (SHashObj *)handle; - uint32_t hash = 0, hashv; - char * c; - - c = string; - while (*c) { - hash += *((int *)c); - c += 4; - } - - hashv = hash / pObj->maxSessions; - hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions; - - return hash; -} - -uint32_t taosHashStringStep1(void *handle, char *string) { - SHashObj *pObj = (SHashObj *)handle; - uint32_t hash = 0, hashv; - char * c; - - c = string; - while (*c) { - hash += *c; - c++; - } - - hashv = hash / pObj->maxSessions; - hash = (hashv + hash % pObj->maxSessions) % pObj->maxSessions; - - return hash; -} - -void *taosAddStrHashWithSize(void *handle, char *string, char *pData, int dataSize) { - uint32_t hash; - SHashNode *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - if (string == NULL || string[0] == 0) return NULL; - - hash = (*pObj->hashFp)(pObj, string); - - pthread_mutex_lock(&pObj->mutex); - - pNode = (SHashNode *)malloc(sizeof(SHashNode) + (size_t)dataSize + strlen(string) + 1); - memcpy(pNode->data, pData, (size_t)dataSize); - pNode->prev = 0; - pNode->next = pObj->hashList[hash]; - pNode->string = pNode->data + dataSize; - strcpy(pNode->string, string); - - if (pObj->hashList[hash] != 0) (pObj->hashList[hash])->prev = pNode; - pObj->hashList[hash] = pNode; - - pthread_mutex_unlock(&pObj->mutex); - - uTrace("hash:%d:%s is added", hash, string); - - return pNode->data; -} - -void *taosAddStrHash(void *handle, char *string, char *pData) { - if (string == NULL || string[0] == 0) return NULL; - - SHashObj *pObj = (SHashObj *)handle; - return taosAddStrHashWithSize(handle, string, pData, pObj->dataSize); -} - -void taosDeleteStrHashNode(void *handle, char *string, void *pDeleteNode) { - uint32_t hash; - SHashNode *pNode; - SHashObj * pObj; - bool find = false; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - if (string == NULL || string[0] == 0) return; - - hash = (*(pObj->hashFp))(pObj, string); - - pthread_mutex_lock(&pObj->mutex); - - pNode = pObj->hashList[hash]; - - while (pNode) { - if (strcmp(pNode->string, string) != 0) continue; - if (pNode->data == pDeleteNode) { - find = true; - break; - } - - pNode = pNode->next; - } - - if (find) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->hashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - uTrace("hash:%d:%s:%p is removed", hash, string, pNode); - - free(pNode); - } - - pthread_mutex_unlock(&pObj->mutex); -} - -void taosDeleteStrHash(void *handle, char *string) { - uint32_t hash; - SHashNode *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - if (string == NULL || string[0] == 0) return; - - hash = (*(pObj->hashFp))(pObj, string); - - pthread_mutex_lock(&pObj->mutex); - - pNode = pObj->hashList[hash]; - while (pNode) { - if (strcmp(pNode->string, string) == 0) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->hashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - uTrace("hash:%d:%s:%p is removed", hash, string, pNode); - - free(pNode); - } - - pthread_mutex_unlock(&pObj->mutex); -} - -void *taosGetStrHashData(void *handle, char *string) { - uint32_t hash; - SHashNode *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return NULL; - if (string == NULL || string[0] == 0) return NULL; - - hash = (*pObj->hashFp)(pObj, string); - - pthread_mutex_lock(&pObj->mutex); - - pNode = pObj->hashList[hash]; - - while (pNode) { - if (strcmp(pNode->string, string) == 0) { - uTrace("hash:%d:%s is retrieved", hash, string); - break; - } - - pNode = pNode->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pNode) return pNode->data; - - return NULL; -} - -void *taosInitStrHash(uint32_t maxSessions, uint32_t dataSize, uint32_t (*fp)(void *, char *)) { - SHashObj *pObj; - - pObj = (SHashObj *)malloc(sizeof(SHashObj)); - if (pObj == NULL) { - return NULL; - } - - memset(pObj, 0, sizeof(SHashObj)); - pObj->maxSessions = maxSessions; - pObj->dataSize = dataSize; - pObj->hashFp = fp; - - pObj->hashList = (SHashNode **)malloc(sizeof(SHashNode *) * (size_t)maxSessions); - if (pObj->hashList == NULL) { - free(pObj); - return NULL; - } - memset(pObj->hashList, 0, sizeof(SHashNode *) * (size_t)maxSessions); - - pthread_mutex_init(&pObj->mutex, NULL); - - return pObj; -} - -void taosCleanUpStrHashWithFp(void *handle, void (*fp)(char *)) { - SHashObj * pObj; - SHashNode *pNode, *pNext; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return; - - pthread_mutex_lock(&pObj->mutex); - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - if (fp != NULL) fp(pNode->data); - free(pNode); - pNode = pNext; - } - } - - free(pObj->hashList); - } - - pthread_mutex_unlock(&pObj->mutex); - - pthread_mutex_destroy(&pObj->mutex); - - memset(pObj, 0, sizeof(SHashObj)); - free(pObj); -} - -void taosCleanUpStrHash(void *handle) { taosCleanUpStrHashWithFp(handle, NULL); } - -char *taosVisitStrHashWithFp(void *handle, int (*fp)(char *)) { - SHashObj * pObj; - SHashNode *pNode, *pNext; - char * pData = NULL; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions <= 0) return NULL; - - pthread_mutex_lock(&pObj->mutex); - - if (pObj->hashList) { - for (int i = 0; i < pObj->maxSessions; ++i) { - pNode = pObj->hashList[i]; - while (pNode) { - pNext = pNode->next; - int flag = fp(pNode->data); - if (flag) { - pData = pNode->data; - goto VisitEnd; - } - - pNode = pNext; - } - } - } - -VisitEnd: - - pthread_mutex_unlock(&pObj->mutex); - return pData; -} diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 5ef4417710..5198597ff7 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -176,6 +176,11 @@ SArray* taosArrayClone(const SArray* pSrc) { return dst; } +void taosArrayClear(SArray* pArray) { + assert( pArray != NULL ); + pArray->size = 0; +} + void taosArrayDestroy(SArray* pArray) { if (pArray == NULL) { return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 02a5b93141..2641d2eacb 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -149,6 +149,7 @@ static void taosRemoveFromTrash(SCacheObj *pCacheObj, STrashElem *pElem) { } pElem->pData->signature = 0; + if (pCacheObj->freeFp) pCacheObj->freeFp(pElem->pData->data); free(pElem->pData); free(pElem); } @@ -210,7 +211,8 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo int32_t size = pNode->size; taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - uTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size); + uTrace("key:%s is removed from cache,total:%d,size:%ldbytes", pNode->key, pCacheObj->totalSize, size); + if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -380,7 +382,7 @@ static void taosCacheRefresh(void *handle, void *tmrId) { } } -SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { +SCacheObj *taosCacheInitWithCb(void *tmrCtrl, int64_t refreshTime, void (*freeCb)(void *data)) { if (tmrCtrl == NULL || refreshTime <= 0) { return NULL; } @@ -401,6 +403,7 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { // set free cache node callback function for hash table taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode); + pCacheObj->freeFp = freeCb; pCacheObj->refreshTime = refreshTime * 1000; pCacheObj->tmrCtrl = tmrCtrl; @@ -419,6 +422,10 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { return pCacheObj; } +SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) { + return taosCacheInitWithCb(tmrCtrl, refreshTime, NULL); +} + void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, size_t dataSize, int duration) { SCacheDataNode *pNode; @@ -439,6 +446,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const char *key, const void *pData, siz uTrace("key:%s %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", total:%d, size:%" PRId64 " bytes", key, pNode, pNode->addedTime, pNode->expiredTime, pCacheObj->totalSize, dataSize); + } else { + uError("key:%s failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 0abf1e6be3..4bac669dc0 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -51,12 +51,21 @@ int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { } } +int32_t compareFloatVal(const void *pLeft, const void *pRight) { + float ret = GET_FLOAT_VAL(pLeft) - GET_FLOAT_VAL(pRight); + if (fabs(ret) < FLT_EPSILON) { + return 0; + } else { + return ret > 0? 1 : -1; + } +} + int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); if (fabs(ret) < FLT_EPSILON) { return 0; } else { - return ret > 0 ? 1 : -1; + return ret > 0? 1 : -1; } } @@ -127,7 +136,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat size_t n = strcspn(str, next); str += n; - if (str[0] == 0 || (n >= size - 1)) { + if (str[0] == 0 || (n >= size)) { break; } @@ -175,10 +184,10 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcsspn(str, accept); + size_t n = wcscspn(str, accept); str += n; - if (str[0] == 0 || (n >= size - 1)) { + if (str[0] == 0 || (n >= size)) { break; } @@ -257,7 +266,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_FLOAT: { - comparFn = compareDoubleVal; break; + comparFn = compareFloatVal; break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -313,6 +322,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { comparFn = compareInt64Val; break; case TSDB_DATA_TYPE_FLOAT: + comparFn = compareFloatVal; + break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 61ae647bf4..32a3df844c 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -25,7 +25,7 @@ #include "tsystem.h" #include "tutil.h" -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {0}; +SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; int32_t tsGlobalConfigNum = 0; static char *tsGlobalUnit[] = { diff --git a/src/util/src/tdes.c b/src/util/src/tdes.c index 6f72dec574..00474e4ae2 100644 --- a/src/util/src/tdes.c +++ b/src/util/src/tdes.c @@ -32,7 +32,7 @@ char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mo unsigned int number_of_blocks = len / 8; unsigned char data_block[9] = {0}; unsigned char processed_block[9] = {0}; - key_set key_sets[17] = {0}; + key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); char* dest = calloc(len + 1, 1); generate_sub_keys(key, key_sets); diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 628f7a2d5c..c040a11362 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -27,10 +27,11 @@ typedef struct { } STaosError; +#include "os.h" #include "taoserror.h" -static _Thread_local int32_t tsErrno; +static threadlocal int32_t tsErrno; int32_t* taosGetErrno() { return &tsErrno; } diff --git a/src/util/src/thash.c b/src/util/src/thash.c deleted file mode 100644 index e3c6fe26b4..0000000000 --- a/src/util/src/thash.c +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmempool.h" - -typedef struct _long_hash_t { - unsigned int id; - struct _long_hash_t *prev; - struct _long_hash_t *next; - uint64_t cont; -} SLongHash; - -typedef struct { - SLongHash **longHashList; - mpool_h longHashMemPool; - int (*hashFp)(void *, uint64_t); - int maxSessions; - pthread_mutex_t mutex; -} SHashObj; - -uint64_t taosHashUInt64(uint64_t handle) { - uint64_t hash = handle >> 16; - hash += handle & 0xFFFF; - return hash; -} - -int taosHashLong(void *handle, uint64_t ip) { - SHashObj *pObj = (SHashObj *)handle; - int hash = 0; - - hash = (int)(ip >> 16); - hash += (int)(ip & 0xFFFF); - - hash = hash % pObj->maxSessions; - - return hash; -} - -int taosAddHash(void *handle, uint64_t cont, unsigned int id) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return -1; - - pthread_mutex_lock(&pObj->mutex); - - hash = (*pObj->hashFp)(pObj, cont); - - pNode = (SLongHash *)taosMemPoolMalloc(pObj->longHashMemPool); - pNode->cont = cont; - pNode->id = id; - pNode->prev = 0; - pNode->next = pObj->longHashList[hash]; - - if (pObj->longHashList[hash] != 0) (pObj->longHashList[hash])->prev = pNode; - pObj->longHashList[hash] = pNode; - - pthread_mutex_unlock(&pObj->mutex); - - return 0; -} - -void taosDeleteHash(void *handle, uint64_t cont) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - hash = (*pObj->hashFp)(pObj, cont); - - pthread_mutex_lock(&pObj->mutex); - - pNode = pObj->longHashList[hash]; - while (pNode) { - if (pNode->cont == cont) break; - - pNode = pNode->next; - } - - if (pNode) { - if (pNode->prev) { - pNode->prev->next = pNode->next; - } else { - pObj->longHashList[hash] = pNode->next; - } - - if (pNode->next) { - pNode->next->prev = pNode->prev; - } - - taosMemPoolFree(pObj->longHashMemPool, (char *)pNode); - } - - pthread_mutex_unlock(&pObj->mutex); -} - -int32_t taosGetIdFromHash(void *handle, uint64_t cont) { - int hash; - SLongHash *pNode; - SHashObj * pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return -1; - - hash = (*pObj->hashFp)(pObj, cont); - - pthread_mutex_lock(&pObj->mutex); - - pNode = pObj->longHashList[hash]; - - while (pNode) { - if (pNode->cont == cont) { - break; - } - - pNode = pNode->next; - } - - pthread_mutex_unlock(&pObj->mutex); - - if (pNode) return (int32_t)pNode->id; - - return -1; -} - -void *taosOpenHash(int maxSessions, int (*fp)(void *, uint64_t)) { - SLongHash **longHashList; - mpool_h longHashMemPool; - SHashObj * pObj; - - longHashMemPool = taosMemPoolInit(maxSessions, sizeof(SLongHash)); - if (longHashMemPool == 0) return NULL; - - longHashList = calloc(sizeof(SLongHash *), (size_t)maxSessions); - if (longHashList == 0) { - taosMemPoolCleanUp(longHashMemPool); - return NULL; - } - - pObj = malloc(sizeof(SHashObj)); - if (pObj == NULL) { - taosMemPoolCleanUp(longHashMemPool); - free(longHashList); - return NULL; - } - - pObj->maxSessions = maxSessions; - pObj->longHashMemPool = longHashMemPool; - pObj->longHashList = longHashList; - pObj->hashFp = fp; - - pthread_mutex_init(&pObj->mutex, NULL); - - return pObj; -} - -void taosCloseHash(void *handle) { - SHashObj *pObj; - - pObj = (SHashObj *)handle; - if (pObj == NULL || pObj->maxSessions == 0) return; - - pthread_mutex_lock(&pObj->mutex); - - if (pObj->longHashMemPool) taosMemPoolCleanUp(pObj->longHashMemPool); - - if (pObj->longHashList) free(pObj->longHashList); - - pthread_mutex_unlock(&pObj->mutex); - - pthread_mutex_destroy(&pObj->mutex); - - memset(pObj, 0, sizeof(SHashObj)); - free(pObj); -} diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 25ea49b60d..b72db6a8d8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -74,7 +74,47 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode); static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode); static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); -static SSkipListNode* tSkipListDoGet(SSkipList *pSkipList, SSkipListKey key); + + +// when order is TSDB_ORDER_ASC, return the last node with key less than val +// when order is TSDB_ORDER_DESC, return the first node with key large than val +static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order) { + __compar_fn_t comparFn = pSkipList->comparFn; + SSkipListNode *pNode = NULL; + + if (order == TSDB_ORDER_ASC) { + pNode = pSkipList->pHead; + for (int32_t i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if (comparFn(key, val) < 0) { + pNode = p; + p = SL_GET_FORWARD_POINTER(p, i); + } else { + break; + } + } + } + } else { + pNode = pSkipList->pTail; + for (int32_t i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i); + while (p != pSkipList->pHead) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if (comparFn(key, val) > 0) { + pNode = p; + p = SL_GET_BACKWARD_POINTER(p, i); + } else { + break; + } + } + } + } + + return pNode; +} + static bool initForwardBackwardPtr(SSkipList* pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; @@ -110,7 +150,11 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui maxLevel = MAX_SKIP_LIST_LEVEL; } - pSkipList->keyInfo = (SSkipListKeyInfo){.type = keyType, .len = keyLen, .dupKey = dupKey, .freeNode = freeNode}; + pSkipList->keyInfo.type = keyType; + pSkipList->keyInfo.len = keyLen; + pSkipList->keyInfo.dupKey = dupKey; + pSkipList->keyInfo.freeNode = freeNode; + pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparFunc(keyType); pSkipList->maxLevel = maxLevel; @@ -240,13 +284,37 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { return pNode; } + + SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { SArray* sa = taosArrayInit(1, POINTER_BYTES); - SSkipListNode* pNode = tSkipListDoGet(pSkipList, key); - taosArrayPush(sa, &pNode); + + if (pSkipList->lock) { + pthread_rwlock_wrlock(pSkipList->lock); + } + + SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + taosArrayPush(sa, &p); + pNode = p; + } + + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + return sa; } + + size_t tSkipListGetSize(const SSkipList* pSkipList) { if (pSkipList == NULL) { return 0; @@ -375,14 +443,52 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) { // return true; //} -bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { - SSkipListNode* pNode = tSkipListDoGet(pSkipList, key); - if (pNode != NULL) { - tSkipListRemoveNode(pSkipList, pNode); - return true; +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { + uint32_t count = 0; + + if (pSkipList->lock) { + pthread_rwlock_wrlock(pSkipList->lock); } - - return false; + + SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + + for (int32_t j = p->level - 1; j >= 0; --j) { + SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j); + SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j); + SL_GET_FORWARD_POINTER(prev, j) = next; + SL_GET_BACKWARD_POINTER(next, j) = prev; + } + + if (pSkipList->keyInfo.freeNode) { + tfree(p); + } + + ++count; + } + + // compress the minimum level of skip list + while (pSkipList->level > 0) { + if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) { + break; + } + pSkipList->level--; + } + + pSkipList->size -= count; + + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + + return count; } void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { @@ -425,54 +531,25 @@ SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { } SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { - if (pSkipList == NULL) { - return NULL; - } - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - + assert(pSkipList != NULL); + + SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); if (val == NULL) { - return doCreateSkipListIterator(pSkipList, order); - } else { - - SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - - int32_t ret = -1; - __compar_fn_t filterComparFn = getKeyComparFunc(pSkipList->keyInfo.type); - SSkipListNode* pNode = pSkipList->pHead; - - for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - - if ((ret = filterComparFn(key, val)) < 0) { - pNode = p; - p = SL_GET_FORWARD_POINTER(p, i); - } else { - break; - } - } - - forward[i] = pNode; - } - - SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); - - // set the initial position - if (order == TSDB_ORDER_ASC) { - iter->cur = forward[0]; // greater equals than the value - } else { - iter->cur = SL_GET_FORWARD_POINTER(forward[0], 0); - - if (ret == 0) { - assert(iter->cur != pSkipList->pTail); - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - } - } - return iter; } + + if (pSkipList->lock) { + pthread_rwlock_rdlock(pSkipList->lock); + } + + iter->cur = getPriorNode(pSkipList, val, order); + + if (pSkipList->lock) { + pthread_rwlock_unlock(pSkipList->lock); + } + + return iter; } bool tSkipListIterNext(SSkipListIterator *iter) { @@ -487,17 +564,9 @@ bool tSkipListIterNext(SSkipListIterator *iter) { } if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - if (iter->cur == NULL) { - iter->cur = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - } else { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - } + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); } else { // descending order iterate - if (iter->cur == NULL) { - iter->cur = SL_GET_BACKWARD_POINTER(pSkipList->pTail, 0); - } else { - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - } + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); } if (pSkipList->lock) { @@ -638,57 +707,16 @@ SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) { return pNode; } -SSkipListNode* tSkipListDoGet(SSkipList *pSkipList, SSkipListKey skey) { - SSkipListNode *pNode = pSkipList->pHead; - SSkipListNode *pRes = NULL; - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.queryCount++; -#endif - - __compar_fn_t cmparFn = getComparFunc(pSkipList->keyInfo.type, 0); - - int32_t ret = -1; - for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - - if ((ret = cmparFn(key, skey)) < 0) { - pNode = p; - p = SL_GET_FORWARD_POINTER(p, i); - } else { - break; - } - } - - // find the qualified key - if (ret == 0) { - pRes = SL_GET_FORWARD_POINTER(pNode, i); - break; - // skip list does not allowed duplicated key, abort further retrieve data -// if (!pSkipList->keyInfo.dupKey) { -// break; -// } - } - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return pRes; -} - SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); iter->pSkipList = pSkipList; iter->order = order; + if(order == TSDB_ORDER_ASC) { + iter->cur = pSkipList->pHead; + } else { + iter->cur = pSkipList->pTail; + } return iter; } \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 3713e71a01..70445a3d65 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -281,34 +281,55 @@ void skiplistPerformanceTest() { // todo not support duplicated key yet void duplicatedKeyTest() { -#if 0 - SSkipListKey key; - key.nType = TSDB_DATA_TYPE_INT; + SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int), true, false, true, getkey); - SSkipListNode **pNodes = NULL; - - SSkipList *pSkipList = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_INT, sizeof(int)); - - for (int32_t i = 0; i < 10000; ++i) { + for (int32_t i = 0; i < 200; ++i) { for (int32_t j = 0; j < 5; ++j) { - key.i64Key = i; - tSkipListPut(pSkipList, "", &key, 1); + int32_t level, size; + tSkipListNewNodeInfo(pSkipList, &level, &size); + SSkipListNode* d = (SSkipListNode*)calloc(1, size + sizeof(int32_t)); + d->level = level; + int32_t* key = (int32_t*)SL_GET_NODE_KEY(pSkipList, d); + key[0] = i; + tSkipListPut(pSkipList, d); } } - tSkipListPrint(pSkipList, 1); - for (int32_t i = 0; i < 100; ++i) { - key.i64Key = rand() % 1000; - int32_t size = tSkipListGets(pSkipList, &key, &pNodes); - - assert(size == 5); - - tfree(pNodes); + SSkipListKey key; + SArray* nodes = tSkipListGet(pSkipList, (char*)(&i)); + assert( taosArrayGetSize(nodes) == 5 ); + taosArrayDestroy(nodes); } + int32_t key = 101; + uint32_t num = tSkipListRemove(pSkipList, (char*)(&key)); + assert(num == 5); + + SArray* nodes = tSkipListGet(pSkipList, (char*)(&key)); + assert( taosArrayGetSize(nodes) == 0 ); + taosArrayDestroy(nodes); + + key = 102; + SSkipListIterator* iter = tSkipListCreateIterFromVal(pSkipList, (char*)(&key), TSDB_DATA_TYPE_INT, TSDB_ORDER_ASC); + for(int i = 0; i < 6; i++) { + assert(tSkipListIterNext(iter) == true); + SSkipListNode* node = tSkipListIterGet(iter); + int32_t* val = (int32_t*)SL_GET_NODE_KEY(pSkipList, node); + assert((i < 5) == ((*val) == key)); + } + tSkipListDestroyIter(iter); + + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(&key), TSDB_DATA_TYPE_INT, TSDB_ORDER_DESC); + for(int i = 0; i < 6; i++) { + assert(tSkipListIterNext(iter) == true); + SSkipListNode* node = tSkipListIterGet(iter); + int32_t* val = (int32_t*)SL_GET_NODE_KEY(pSkipList, node); + assert((i < 5) == ((*val) == key)); + } + tSkipListDestroyIter(iter); + tSkipListDestroy(pSkipList); -#endif } } // namespace diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index d4842fdf7f..dea9369dd8 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -37,7 +37,8 @@ typedef struct { int32_t refCount; // reference count int status; int8_t role; - int64_t version; + int64_t version; // current version + int64_t fversion; // version on saved data file void *wqueue; void *rqueue; void *wal; @@ -45,10 +46,11 @@ typedef struct { void *sync; void *events; void *cq; // continuous query - int32_t cfgVersion; - STsdbCfg tsdbCfg; - SSyncCfg syncCfg; - SWalCfg walCfg; + int32_t cfgVersion; + STsdbCfg tsdbCfg; + SSyncCfg syncCfg; + SWalCfg walCfg; + char *rootDir; } SVnodeObj; int vnodeWriteToQueue(void *param, void *pHead, int type); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index d3e9751738..6dabc98ae8 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "ihash.h" +#include "hash.h" #include "taoserror.h" #include "taosmsg.h" #include "tutil.h" @@ -32,17 +32,15 @@ static int32_t tsOpennedVnodes; static void *tsDnodeVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); -static void vnodeBuildVloadMsg(char *pNode, void * param); -static int vnodeWalCallback(void *arg); static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg); static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); static bool vnodeReadVersion(SVnodeObj *pVnode); -static int vnodeWalCallback(void *arg); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size); +static int vnodeProcessTsdbStatus(void *arg, int status); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); -static void vnodeNotifyFileSynced(void *ahandle); +static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); static pthread_once_t vnodeModuleInit = PTHREAD_ONCE_INIT; @@ -59,7 +57,7 @@ static void vnodeInit() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsDnodeVnodesHash = taosInitIntHash(TSDB_MAX_VNODES, sizeof(SVnodeObj *), taosHashInt); + tsDnodeVnodesHash = taosHashInit(TSDB_MAX_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); if (tsDnodeVnodesHash == NULL) { vError("failed to init vnode list"); } @@ -69,7 +67,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { int32_t code; pthread_once(&vnodeModuleInit, vnodeInit); - SVnodeObj *pTemp = (SVnodeObj *)taosGetIntHashData(tsDnodeVnodesHash, pVnodeCfg->cfg.vgId); + SVnodeObj *pTemp = (SVnodeObj *)taosHashGet(tsDnodeVnodesHash, (const char *)&pVnodeCfg->cfg.vgId, sizeof(int32_t)); if (pTemp != NULL) { vPrint("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp); return TSDB_CODE_SUCCESS; @@ -121,12 +119,17 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { } int32_t vnodeDrop(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); - if (ppVnode == NULL || *ppVnode == NULL) { + if (tsDnodeVnodesHash == NULL) { vTrace("vgId:%d, failed to drop, vgId not exist", vgId); return TSDB_CODE_INVALID_VGROUP_ID; } + SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); + if (ppVnode == NULL || *ppVnode == NULL) { + vTrace("vgId:%d, failed to drop, vgId not find", vgId); + return TSDB_CODE_INVALID_VGROUP_ID; + } + SVnodeObj *pVnode = *ppVnode; vTrace("vgId:%d, vnode will be dropped", pVnode->vgId); pVnode->status = TAOS_VN_STATUS_DELETING; @@ -148,7 +151,7 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) { code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to read cfg file", pVnode->vgId); - taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); + taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); return code; } @@ -182,16 +185,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->refCount = 1; pVnode->version = 0; pVnode->tsdbCfg.tsdbId = pVnode->vgId; - taosAddIntHash(tsDnodeVnodesHash, pVnode->vgId, (char *)(&pVnode)); - + pVnode->rootDir = strdup(rootDir); + taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + int32_t code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to read cfg file", pVnode->vgId); - taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); + taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); return code; } vnodeReadVersion(pVnode); + pVnode->fversion = pVnode->version; pVnode->wqueue = dnodeAllocateWqueue(pVnode); pVnode->rqueue = dnodeAllocateRqueue(pVnode); @@ -205,14 +210,14 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { STsdbAppH appH = {0}; appH.appH = (void *)pVnode; - appH.walCallBack = vnodeWalCallback; + appH.notifyStatus = vnodeProcessTsdbStatus; appH.cqH = pVnode->cq; sprintf(temp, "%s/tsdb", rootDir); pVnode->tsdb = tsdbOpenRepo(temp, &appH); if (pVnode->tsdb == NULL) { vError("vgId:%d, failed to open tsdb at %s(%s)", pVnode->vgId, temp, tstrerror(terrno)); - taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); + taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); return terrno; } @@ -248,7 +253,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } int32_t vnodeClose(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); if (ppVnode == NULL || *ppVnode == NULL) return 0; SVnodeObj *pVnode = *ppVnode; @@ -271,6 +276,7 @@ void vnodeRelease(void *pVnodeRaw) { return; } + tfree(pVnode->rootDir); // remove read queue dnodeFreeRqueue(pVnode->rqueue); pVnode->rqueue = NULL; @@ -288,17 +294,19 @@ void vnodeRelease(void *pVnodeRaw) { free(pVnode); int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1); - vTrace("vgId:%d, vnode is released, vnodes:%d", pVnode, vgId, count); + vTrace("vgId:%d, vnode is released, vnodes:%d", vgId, count); if (count <= 0) { - taosCleanUpIntHash(tsDnodeVnodesHash); + taosHashCleanup(tsDnodeVnodesHash); vnodeModuleInit = PTHREAD_ONCE_INIT; tsDnodeVnodesHash = NULL; } } void *vnodeGetVnode(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosGetIntHashData(tsDnodeVnodesHash, vgId); + if (tsDnodeVnodesHash == NULL) return NULL; + + SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_INVALID_VGROUP_ID; vPrint("vgId:%d, not exist", vgId); @@ -332,16 +340,8 @@ void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } -void vnodeBuildStatusMsg(void *param) { - SDMStatusMsg *pStatus = param; - taosVisitIntHashWithFp(tsDnodeVnodesHash, vnodeBuildVloadMsg, pStatus); -} - -static void vnodeBuildVloadMsg(char *pNode, void * param) { - SVnodeObj *pVnode = *(SVnodeObj **) pNode; +static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { if (pVnode->status == TAOS_VN_STATUS_DELETING) return; - - SDMStatusMsg *pStatus = param; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; @@ -355,8 +355,23 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { pLoad->replica = pVnode->syncCfg.replica; } +void vnodeBuildStatusMsg(void *param) { + SDMStatusMsg *pStatus = param; + SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); + + while (taosHashIterNext(pIter)) { + SVnodeObj **pVnode = taosHashIterGet(pIter); + if (pVnode == NULL) continue; + if (*pVnode == NULL) continue; + + vnodeBuildVloadMsg(*pVnode, pStatus); + } + + taosHashDestroyIter(pIter); +} + static void vnodeCleanUp(SVnodeObj *pVnode) { - taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId); + taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); if (pVnode->sync) { syncStop(pVnode->sync); @@ -366,26 +381,34 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { cqClose(pVnode->cq); pVnode->cq = NULL; - tsdbCloseRepo(pVnode->tsdb); + tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; walClose(pVnode->wal); pVnode->wal = NULL; - vnodeSaveVersion(pVnode); vnodeRelease(pVnode); } // TODO: this is a simple implement -static int vnodeWalCallback(void *arg) { +static int vnodeProcessTsdbStatus(void *arg, int status) { SVnodeObj *pVnode = arg; - return walRenew(pVnode->wal); + + if (status == TSDB_STATUS_COMMIT_START) { + pVnode->fversion = pVnode->version; + return walRenew(pVnode->wal); + } + + if (status == TSDB_STATUS_COMMIT_OVER) + return vnodeSaveVersion(pVnode); + + return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size) { - // SVnodeObj *pVnode = ahandle; - //tsdbGetFileInfo(pVnode->tsdb, name, index, size); - return 0; +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) { + SVnodeObj *pVnode = ahandle; + *fversion = pVnode->fversion; + return tsdbGetFileInfo(pVnode->tsdb, name, index, size); } static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { @@ -395,6 +418,7 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { static void vnodeNotifyRole(void *ahandle, int8_t role) { SVnodeObj *pVnode = ahandle; + vPrint("vgId:%d, sync role changed from %d to %d", pVnode->vgId, pVnode->role, role); pVnode->role = role; if (pVnode->role == TAOS_SYNC_ROLE_MASTER) @@ -403,11 +427,23 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { cqStop(pVnode->cq); } -static void vnodeNotifyFileSynced(void *ahandle) { +static void vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { SVnodeObj *pVnode = ahandle; - vTrace("pVnode:%p vgId:%d, data file is synced", pVnode, pVnode->vgId); + vTrace("vgId:%d, data file is synced, fversion:%" PRId64 "", pVnode->vgId, fversion); + pVnode->fversion = fversion; + pVnode->version = fversion; + vnodeSaveVersion(pVnode); + + char rootDir[128] = "\0"; + sprintf(rootDir, "%s/tsdb", pVnode->rootDir); // clsoe tsdb, then open tsdb + tsdbCloseRepo(pVnode->tsdb, 0); + STsdbAppH appH = {0}; + appH.appH = (void *)pVnode; + appH.notifyStatus = vnodeProcessTsdbStatus; + appH.cqH = pVnode->cq; + pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); } static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { @@ -676,14 +712,14 @@ static int32_t vnodeSaveVersion(SVnodeObj *pVnode) { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->version); + len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); fclose(fp); free(content); - vPrint("vgId:%d, save vnode version:%" PRId64 " successed", pVnode->vgId, pVnode->version); + vPrint("vgId:%d, save vnode version:%" PRId64 " succeed", pVnode->vgId, pVnode->fversion); return 0; } @@ -725,7 +761,7 @@ static bool vnodeReadVersion(SVnodeObj *pVnode) { ret = true; - vPrint("vgId:%d, read vnode version successed, version:%%" PRId64, pVnode->vgId, pVnode->version); + vPrint("vgId:%d, read vnode version succeed, version:%" PRId64, pVnode->vgId, pVnode->version); PARSE_OVER: free(content); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index d2dd1811cf..03407a5aa2 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -65,7 +65,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, void *pCont, int32_t cont pRet->len = sizeof(SQueryTableRsp); pRet->rsp = pRsp; - vTrace("vgId:%d QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo); } else { pQInfo = pCont; code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -83,7 +83,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont int32_t code = TSDB_CODE_SUCCESS; - vTrace("vgId:%d QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, retrieve msg is received", pVnode->vgId, pQInfo); pRet->code = qRetrieveQueryResultInfo(pQInfo); if (pRet->code != TSDB_CODE_SUCCESS) { @@ -104,6 +104,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, void *pCont, int32_t cont } } - vTrace("vgId:%d QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); + vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, pQInfo); return code; } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 3fc079ca13..635c466978 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -51,10 +51,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) return TSDB_CODE_MSG_NOT_PROCESSED; - if (pVnode->status != TAOS_VN_STATUS_READY && qtype == TAOS_QTYPE_RPC) - return TSDB_CODE_NOT_ACTIVE_VNODE; - - if (pHead->version == 0) { // from client + if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) return TSDB_CODE_NOT_ACTIVE_VNODE; @@ -64,12 +61,10 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // assign version pVnode->version++; pHead->version = pVnode->version; - } else { + } else { // from wal or forward // for data from WAL or forward, version may be smaller if (pHead->version <= pVnode->version) return 0; } - - // more status and role checking here pVnode->version = pHead->version; @@ -77,9 +72,13 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { code = walWrite(pVnode->wal, pHead); if (code < 0) return code; - int32_t syncCode = syncForwardToPeer(pVnode->sync, pHead, item); + // forward to peers if data is from RPC or CQ + int32_t syncCode = 0; + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_CQ) + syncCode = syncForwardToPeer(pVnode->sync, pHead, item); if (syncCode < 0) return syncCode; + // write data locally code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, item); if (code < 0) return code; @@ -92,17 +91,16 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR // save insert result into item vTrace("vgId:%d, submit msg is processed", pVnode->vgId); - code = tsdbInsertData(pVnode->tsdb, pCont); - + pRet->len = sizeof(SShellSubmitRspMsg); pRet->rsp = rpcMallocCont(pRet->len); SShellSubmitRspMsg *pRsp = pRet->rsp; - + code = tsdbInsertData(pVnode->tsdb, pCont, pRsp); + pRsp->numOfFailedBlocks = 0; //TODO + //pRet->len += pRsp->numOfFailedBlocks * sizeof(SShellSubmitRspBlock); //TODO pRsp->code = 0; pRsp->numOfRows = htonl(1); - pRsp->affectedRows = htonl(1); - pRsp->numOfFailedBlocks = 0; - + return code; } diff --git a/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md new file mode 100644 index 0000000000..7c42d47d1b --- /dev/null +++ b/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md @@ -0,0 +1,235 @@ +### Prepare development environment + +1. sudo apt install + build-essential cmake net-tools python-pip python-setuptools python3-pip + python3-setuptools valgrind psmisc curl + +2. git clone ; cd TDengine + +3. mkdir debug; cd debug; cmake ..; make ; sudo make install + +4. pip install src/connector/python/linux/python2 ; pip3 install + src/connector/python/linux/python3 + +### How to run Python test suite + +1. cd \/tests/pytest + +2. ./smoketest.sh \# for smoke test + +3. ./smoketest.sh -g \# for memory leak detection test with valgrind + +4. ./fulltest.sh \# for full test + +> Note1: TDengine daemon's configuration and data files are stored in +> \/sim directory. As a historical design, it's same place with +> TSIM script. So after the TSIM script ran with sudo privilege, the directory +> has been used by TSIM then the python script cannot write it by a normal +> user. You need to remove the directory completely first before running the +> Python test case. We should consider using two different locations to store +> for TSIM and Python script. + +> Note2: if you need to debug crash problem with a core dump, you need +> manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited" +> before the script line. Then you can look for the core file in +> \/tests/pytest after the program crash. + + +### How to add a new test case + +**1. TSIM test cases:** + +TSIM was the testing framework has been used internally. Now it still be used to run the test cases we develop in the past as a legacy system. We are turning to use Python to develop new test case and are abandoning TSIM gradually. + +**2. Python test cases:** + +**2.1 Please refer to \/tests/pytest/insert/basic.py to add a new +test case.** The new test case must implement 3 functions, where self.init() +and self.stop() simply copy the contents of insert/basic.py and the test +logic is implemented in self.run(). You can refer to the code in the util +directory for more information. + +**2.2 Edit smoketest.sh to add the path and filename of the new test case** + +Note: The Python test framework may continue to be improved in the future, +hopefully, to provide more functionality and ease of writing test cases. The +method of writing the test case above does not exclude that it will also be +affected. + +**2.3 What test.py does in detail:** + +test.py is the entry program for test case execution and monitoring. + +test.py has the following functions. + +\-f --file, Specifies the test case file name to be executed +-p --path, Specifies deployment path + +\-m --master, Specifies the master server IP for cluster deployment +-c--cluster, test cluster function +-s--stop, terminates all running nodes + +\-g--valgrind, load valgrind for memory leak detection test + +\-h--help, display help + +**2.4 What util/log.py does in detail:** + +log.py is quite simple, the main thing is that you can print the output in +different colors as needed. The success() should be called for successful +test case execution and the success() will print green text. The exit() will +print red text and exit the program, exit() should be called for test +failure. + +**util/log.py** + +... + +    def info(self, info): + +        printf("%s %s" % (datetime.datetime.now(), info)) + +  + +    def sleep(self, sec): + +        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + +        time.sleep(sec) + +  + +    def debug(self, err): + +        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def success(self, info): + +        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info)) + +  + +    def notice(self, err): + +        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def exit(self, err): + +        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +        sys.exit(1) + +  + +    def printNoPrefix(self, info): + +        printf("\\033[1;36m%s\\033[0m" % (info) + +... + +**2.5 What util/sql.py does in detail:** + +SQL.py is mainly used to execute SQL statements to manipulate the database, +and the code is extracted and commented as follows: + +**util/sql.py** + +\# prepare() is mainly used to set up the environment for testing table and +data, and to set up the database db for testing. do not call prepare() if you +need to test the database operation command. + +def prepare(self): + +tdLog.info("prepare database:db") + +self.cursor.execute('reset query cache') + +self.cursor.execute('drop database if exists db') + +self.cursor.execute('create database db') + +self.cursor.execute('use db') + +... + +\# query() is mainly used to execute select statements for normal syntax input + +def query(self, sql): + +... + +\# error() is mainly used to execute the select statement with the wrong syntax +input, the error will be caught as a reasonable behavior, if not caught it will +prove that the test failed + +def error() + +... + +\# checkRows() is used to check the number of returned lines after calling +query(select ...) after calling the query(select ...) to check the number of +rows of returned results. + +def checkRows(self, expectRows): + +... + +\# checkData() is used to check the returned result data after calling +query(select ...) after the query(select ...) is called, failure to meet +expectation is + +def checkData(self, row, col, data): + +... + +\# getData() returns the result data after calling query(select ...) to return +the resulting data after calling query(select ...) + +def getData(self, row, col): + +... + +\# execute() used to execute sql and return the number of affected rows + +def execute(self, sql): + +... + +\# executeTimes() Multiple executions of the same sql statement + +def executeTimes(self, sql, times): + +... + +\# CheckAffectedRows() Check if the number of affected rows is as expected + +def checkAffectedRows(self, expectAffectedRows): + +... + +> Note: Both Python2 and Python3 are currently supported by the Python test +> case. Since Python2 is no longer officially supported by January 1, 2020, it +> is recommended that subsequent test case development be guaranteed to run +> correctly on Python3. For Python2, please consider being compatible if +> appropriate without additional +> burden.   + +### CI submission adoption principle. + +- Every commit / PR compilation must pass. Currently, the warning is treated + as an error, so the warning must also be resolved. + +- Test cases that already exist must pass. + +- Because CI is very important to support build and automatically test + procedure, it is necessary to manually test the test case before adding it + and do as many iterations as possible to ensure that the test case provides + stable and reliable test results when added. + +> Note: In the future, according to the requirements and test development +> progress will add stress testing, performance testing, code style, +> and other features based on functional testing. diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index 0bf93f6f2d..f9acf2bb10 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -56,32 +56,46 @@ void run_test(TAOS* taos) { taos_query(taos, "drop database if exists test;"); usleep(100000); - taos_query(taos, "create database test tables 5;"); + //taos_query(taos, "create database test tables 5;"); + taos_query(taos, "create database test;"); usleep(100000); taos_query(taos, "use test;"); - usleep(100000); - taos_query(taos, "create table meters(ts timestamp, a int, b binary(20)) tags(loc binary(20), area int);"); - taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:00:00.000', 0, 'china');"); - taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:01:00.000', 0, 'china');"); - taos_query(taos, "insert into t0 using meters tags('beijing', 0) values('2020-01-01 00:02:00.000', 0, 'china');"); - taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:00:00.000', 0, 'china');"); - taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:01:00.000', 0, 'china');"); - taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:02:00.000', 0, 'china');"); - taos_query(taos, "insert into t1 using meters tags('shanghai', 0) values('2020-01-01 00:03:00.000', 0, 'china');"); - taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:00:00.000', 0, 'UK');"); - taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:00.000', 0, 'UK');"); - taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:01.000', 0, 'UK');"); - taos_query(taos, "insert into t2 using meters tags('london', 0) values('2020-01-01 00:01:02.000', 0, 'UK');"); - taos_query(taos, "insert into t3 using meters tags('tianjin', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t4 using meters tags('wuhan', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t5 using meters tags('jinan', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t6 using meters tags('haikou', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t7 using meters tags('nanjing', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t8 using meters tags('lanzhou', 0) values('2020-01-01 00:01:02.000', 0, 'china');"); - taos_query(taos, "insert into t9 using meters tags('tokyo', 0) values('2020-01-01 00:01:02.000', 0, 'japan');"); + usleep(100000); + taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + + taos_query(taos, "create table t0 using meters tags(0);"); + taos_query(taos, "create table t1 using meters tags(1);"); + taos_query(taos, "create table t2 using meters tags(2);"); + taos_query(taos, "create table t3 using meters tags(3);"); + taos_query(taos, "create table t4 using meters tags(4);"); + taos_query(taos, "create table t5 using meters tags(5);"); + taos_query(taos, "create table t6 using meters tags(6);"); + taos_query(taos, "create table t7 using meters tags(7);"); + taos_query(taos, "create table t8 using meters tags(8);"); + taos_query(taos, "create table t9 using meters tags(9);"); + + taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0);"); + taos_query(taos, "insert into t0 values('2020-01-01 00:01:00.000', 0);"); + taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.000', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:00:00.000', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:01:00.000', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:02:00.000', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.000', 0);"); + taos_query(taos, "insert into t2 values('2020-01-01 00:00:00.000', 0);"); + taos_query(taos, "insert into t2 values('2020-01-01 00:01:00.000', 0);"); + taos_query(taos, "insert into t2 values('2020-01-01 00:01:01.000', 0);"); + taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t3 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t4 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t5 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t6 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t7 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t8 values('2020-01-01 00:01:02.000', 0);"); + taos_query(taos, "insert into t9 values('2020-01-01 00:01:02.000', 0);"); // super tables subscription + usleep(1000000); TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); TAOS_RES* res = taos_consume(tsub); @@ -90,23 +104,23 @@ void run_test(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:03:00.000', 0, 'china');"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0, 'china');"); + taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0, 'UK');"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0, 'UK');"); + taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0, 'china');"); + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); // keep progress information and restart subscription taos_unsubscribe(tsub, 1); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0, 'china');"); + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); res = taos_consume(tsub); check_row_count(__LINE__, res, 24); @@ -133,7 +147,7 @@ void run_test(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0, 'china');"); + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); @@ -197,7 +211,7 @@ int main(int argc, char *argv[]) { // init TAOS taos_init(); - TAOS* taos = taos_connect(host, user, passwd, "test", 0); + TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); exit(1); @@ -209,6 +223,7 @@ int main(int argc, char *argv[]) { exit(0); } + taos_query(taos, "use test;"); TAOS_SUB* tsub = NULL; if (async) { // create an asynchronized subscription, the callback function will be called every 1s diff --git a/tests/pytest/dbmgmt/__init__.py b/tests/pytest/dbmgmt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/dbmgmt/createTableAndDropDnodes.py b/tests/pytest/dbmgmt/createTableAndDropDnodes.py new file mode 100644 index 0000000000..6e29c023ab --- /dev/null +++ b/tests/pytest/dbmgmt/createTableAndDropDnodes.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +import sys +import taos +import threading +import traceback +import random +import datetime +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + + def init(self): + tdLog.debug("start to execute %s" % __file__) + tdLog.info("prepare cluster") + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + + self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + tdSql.init(self.conn.cursor()) + tdSql.execute('reset query cache') + tdSql.execute('create dnode 192.168.0.2') + tdDnodes.deploy(2) + tdDnodes.start(2) + tdSql.execute('create dnode 192.168.0.3') + tdDnodes.deploy(3) + tdDnodes.start(3) + time.sleep(3) + + self.db = "db" + self.stb = "stb" + self.tbPrefix = "tb" + self.tbNum = 100000 + self.count = 0 + # self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + self.threadNum = 1 + # threadLock = threading.Lock() + # global counter for number of tables created by all threads + self.global_counter = 0 + + tdSql.init(self.conn.cursor()) + + def _createTable(self, threadId): + print("Thread%d : createTable" % (threadId)) + conn = taos.connect(config=tdDnodes.getSimCfgPath()) + cursor = conn.cursor() + i = 0 + try: + sql = "use %s" % (self.db) + cursor.execute(sql) + while i < self.tbNum: + if (i % self.threadNum == threadId): + cursor.execute( + "create table tb%d using %s tags(%d)" % + (i + 1, self.stb, i + 1)) + with threading.Lock(): + self.global_counter += 1 + i += 1 + except Exception as e: + tdLog.info( + "Failure when creating table tb%d, exception: %s" % + (i + 1, str(e))) + finally: + cursor.close() + conn.close() + + def _interfereDnodes(self, threadId, dnodeId): + conn = taos.connect(config=tdDnodes.getSimCfgPath()) + cursor = conn.cursor() + # interfere dnode while creating table + print("Thread%d to interfere dnode%d" % (threadId, dnodeId)) + while self.global_counter < self.tbNum * 0.05: + time.sleep(0.2) + cursor.execute("drop dnode 192.168.0.%d" % (dnodeId)) + while self.global_counter < self.tbNum * 0.15: + time.sleep(0.2) + cursor.execute("create dnode 192.168.0.%d" % (dnodeId)) + while self.global_counter < self.tbNum * 0.35: + time.sleep(0.2) + cursor.execute("drop dnode 192.168.0.%d" % (dnodeId)) + while self.global_counter < self.tbNum * 0.45: + time.sleep(0.2) + cursor.execute("create dnode 192.168.0.%d" % (dnodeId)) + while self.global_counter < self.tbNum * 0.65: + time.sleep(0.2) + cursor.execute("drop dnode 192.168.0.%d" % (dnodeId)) + while self.global_counter < self.tbNum * 0.85: + time.sleep(0.2) + cursor.execute("create dnode 192.168.0.%d" % (dnodeId)) + + def run(self): + tdLog.info("================= creating database with replica 2") + threadId = 0 + threads = [] + try: + tdSql.execute("drop database if exists %s" % (self.db)) + tdSql.execute( + "create database %s replica 2 cache 2048 ablocks 2.0 tblocks 10 tables 2000" % + (self.db)) + tdLog.sleep(3) + tdSql.execute("use %s" % (self.db)) + tdSql.execute( + "create table %s (ts timestamp, c1 bigint, stime timestamp) tags(tg1 bigint)" % + (self.stb)) + tdLog.info("Start to create tables") + while threadId < self.threadNum: + tdLog.info("Thread-%d starts to create tables" % (threadId)) + cThread = threading.Thread( + target=self._createTable, + name="thread-%d" % + (threadId), + args=( + threadId, + )) + cThread.start() + threads.append(cThread) + threadId += 1 + + except Exception as e: + tdLog.info("Failed to create tb%d, exception: %s" % (i, str(e))) + # tdDnodes.stopAll() + finally: + time.sleep(1) + + threading.Thread( + target=self._interfereDnodes, + name="thread-interfereDnode%d" % + (3), + args=( + 1, + 3, + )).start() + for t in range(len(threads)): + tdLog.info("Join threads") + # threads[t].start() + threads[t].join() + + tdSql.query("show stables") + tdSql.checkData(0, 4, self.tbNum) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addCluster(__file__, TDTestCase()) diff --git a/tests/pytest/dbmgmt/createTableAndKillDnodes.py b/tests/pytest/dbmgmt/createTableAndKillDnodes.py new file mode 100644 index 0000000000..7772ecdc68 --- /dev/null +++ b/tests/pytest/dbmgmt/createTableAndKillDnodes.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +import sys +import taos +import threading +import traceback +import random +import datetime +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + + def init(self): + tdLog.debug("start to execute %s" % __file__) + tdLog.info("prepare cluster") + tdDnodes.stopAll() + tdDnodes.deploy(1) + tdDnodes.start(1) + + self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + tdSql.init(self.conn.cursor()) + tdSql.execute('reset query cache') + tdSql.execute('create dnode 192.168.0.2') + tdDnodes.deploy(2) + tdDnodes.start(2) + tdSql.execute('create dnode 192.168.0.3') + tdDnodes.deploy(3) + tdDnodes.start(3) + time.sleep(3) + + self.db = "db" + self.stb = "stb" + self.tbPrefix = "tb" + self.tbNum = 100000 + self.count = 0 + # self.conn = taos.connect(config=tdDnodes.getSimCfgPath()) + self.threadNum = 1 + # threadLock = threading.Lock() + # global counter for number of tables created by all threads + self.global_counter = 0 + + tdSql.init(self.conn.cursor()) + + def _createTable(self, threadId): + print("Thread%d : createTable" % (threadId)) + conn = taos.connect(config=tdDnodes.getSimCfgPath()) + cursor = conn.cursor() + i = 0 + try: + sql = "use %s" % (self.db) + cursor.execute(sql) + while i < self.tbNum: + if (i % self.threadNum == threadId): + cursor.execute( + "create table tb%d using %s tags(%d)" % + (i + 1, self.stb, i + 1)) + with threading.Lock(): + self.global_counter += 1 + time.sleep(0.01) + i += 1 + except Exception as e: + tdLog.info( + "Failure when creating table tb%d, exception: %s" % + (i + 1, str(e))) + finally: + cursor.close() + conn.close() + + def _interfereDnodes(self, threadId, dnodeId): + # interfere dnode while creating table + print("Thread%d to interfere dnode%d" % (threadId, dnodeId)) + percent = 0.05 + loop = int(1 / (2 * percent)) + for t in range(1, loop): + while self.global_counter < self.tbNum * (t * percent): + time.sleep(0.2) + tdDnodes.forcestop(dnodeId) + while self.global_counter < self.tbNum * ((t + 1) * percent): + time.sleep(0.2) + tdDnodes.start(dnodeId) + + # while self.global_counter < self.tbNum * 0.05: + # time.sleep(0.2) + # tdDnodes.forcestop(dnodeId) + # while self.global_counter < self.tbNum * 0.10: + # time.sleep(0.2) + # tdDnodes.start(dnodeId) + # while self.global_counter < self.tbNum * 0.15: + # time.sleep(0.2) + # tdDnodes.forcestop(dnodeId) + # while self.global_counter < self.tbNum * 0.20: + # time.sleep(0.2) + # tdDnodes.start(dnodeId) + # while self.global_counter < self.tbNum * 0.25: + # time.sleep(0.2) + # tdDnodes.forcestop(dnodeId) + # while self.global_counter < self.tbNum * 0.30: + # time.sleep(0.2) + # tdDnodes.start(dnodeId) + # while self.global_counter < self.tbNum * 0.35: + # time.sleep(0.2) + # tdDnodes.forcestop(dnodeId) + # while self.global_counter < self.tbNum * 0.40: + # time.sleep(0.2) + # tdDnodes.start(dnodeId) + # while self.global_counter < self.tbNum * 0.45: + # time.sleep(0.2) + # tdDnodes.forcestop(dnodeId) + # while self.global_counter < self.tbNum * 0.50: + # time.sleep(0.2) + # tdDnodes.start(dnodeId) + + def run(self): + tdLog.info("================= creating database with replica 2") + threadId = 0 + threads = [] + try: + tdSql.execute("drop database if exists %s" % (self.db)) + tdSql.execute( + "create database %s replica 2 cache 1024 ablocks 2.0 tblocks 4 tables 1000" % + (self.db)) + tdLog.sleep(3) + tdSql.execute("use %s" % (self.db)) + tdSql.execute( + "create table %s (ts timestamp, c1 bigint, stime timestamp) tags(tg1 bigint)" % + (self.stb)) + tdLog.info("Start to create tables") + while threadId < self.threadNum: + tdLog.info("Thread-%d starts to create tables" % (threadId)) + cThread = threading.Thread( + target=self._createTable, + name="thread-%d" % + (threadId), + args=( + threadId, + )) + cThread.start() + threads.append(cThread) + threadId += 1 + + except Exception as e: + tdLog.info("Failed to create tb%d, exception: %s" % (i, str(e))) + # tdDnodes.stopAll() + finally: + time.sleep(1) + + threading.Thread( + target=self._interfereDnodes, + name="thread-interfereDnode%d" % + (3), + args=( + 1, + 3, + )).start() + for t in range(len(threads)): + tdLog.info("Join threads") + # threads[t].start() + threads[t].join() + + tdSql.query("show stables") + tdSql.checkData(0, 4, self.tbNum) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addCluster(__file__, TDTestCase()) diff --git a/tests/pytest/dbmgmt/database-name-boundary.py b/tests/pytest/dbmgmt/database-name-boundary.py new file mode 100644 index 0000000000..ff6dce22ae --- /dev/null +++ b/tests/pytest/dbmgmt/database-name-boundary.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime +import string +import random +import subprocess + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + + chars = string.ascii_uppercase + string.ascii_lowercase + + getDbNameLen = "grep -w '#define TSDB_DB_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" + dbNameMaxLen = int(subprocess.check_output(getDbNameLen, shell=True)) + tdLog.info("DB name max length is %d" % dbNameMaxLen) + + tdLog.info("=============== step1") + db_name = ''.join(random.choices(chars, k=(dbNameMaxLen + 1))) + tdLog.info('db_name length %d' % len(db_name)) + tdLog.info('create database %s' % db_name) + tdSql.error('create database %s' % db_name) + + tdLog.info("=============== step2") + db_name = ''.join(random.choices(chars, k=dbNameMaxLen)) + tdLog.info('db_name length %d' % len(db_name)) + tdLog.info('create database %s' % db_name) + tdSql.execute('create database %s' % db_name) + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0, 0, db_name.lower()) + + tdLog.info("=============== step3") + db_name = ''.join(random.choices(chars, k=(dbNameMaxLen - 1))) + tdLog.info('db_name length %d' % len(db_name)) + tdLog.info('create database %s' % db_name) + tdSql.execute('create database %s' % db_name) + + tdSql.query('show databases') + tdSql.checkRows(2) + tdSql.checkData(0, 0, db_name.lower()) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/dbmgmt/dropDB_memory_test.py b/tests/pytest/dbmgmt/dropDB_memory_test.py new file mode 100644 index 0000000000..b029945be2 --- /dev/null +++ b/tests/pytest/dbmgmt/dropDB_memory_test.py @@ -0,0 +1,67 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + tbNum = 10000 + insertRows = 1 + db = "db" + loop = 2 + tdSql.execute("drop database if exists %s" % (db)) + tdSql.execute("reset query cache") + tdLog.sleep(1) + for k in range(1, loop + 1): + tdLog.info("===========Loop%d starts============" % (k)) + tdSql.execute( + "create database %s cache 163840 ablocks 40 maxtables 5000 wal 0" % + (db)) + tdSql.execute("use %s" % (db)) + tdSql.execute( + "create table stb (ts timestamp, c1 int) tags(t1 bigint, t2 double)") + for j in range(1, tbNum): + tdSql.execute( + "create table tb%d using stb tags(%d, %d)" % + (j, j, j)) + + for j in range(1, tbNum): + for i in range(0, insertRows): + tdSql.execute( + "insert into tb%d values (now + %dm, %d)" % + (j, i, i)) + tdSql.query("select * from tb%d" % (j)) + tdSql.checkRows(insertRows) + tdLog.info("insert %d rows into tb%d" % (insertRows, j)) + # tdSql.sleep(3) + tdSql.execute("drop database %s" % (db)) + tdLog.sleep(2) + tdLog.info("===========Loop%d completed!=============" % (k)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +#tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 6aec67f9ce..8634a523f3 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -10,10 +10,18 @@ python3 ./test.py $1 -f insert/tinyint.py python3 ./test.py $1 -f insert/date.py python3 ./test.py $1 -f insert/binary.py python3 ./test.py $1 -f insert/nchar.py +python3 ./test.py $1 -f insert/nchar-boundary.py +python3 ./test.py $1 -f insert/nchar-unicode.py +python3 ./test.py $1 -f insert/multi.py python3 ./test.py $1 -f table/column_name.py python3 ./test.py $1 -f table/column_num.py python3 ./test.py $1 -f table/db_table.py +python3 ./test.py $1 -f table/tablename-boundary.py + +python3 ./test.py $1 -f tag_lite/create-tags-boundary.py + +python3 ./test.py $1 -f dbmgmt/database-name-boundary.py python3 ./test.py $1 -f import_merge/importBlock1HO.py python3 ./test.py $1 -f import_merge/importBlock1HPO.py @@ -84,4 +92,7 @@ python3 ./test.py $1 -f import_merge/importTRestart.py # user python3 ./test.py $1 -f user/user_create.py -python3 ./test.py $1 -f user/pass_len.py \ No newline at end of file +python3 ./test.py $1 -f user/pass_len.py + +# table +#python3 ./test.py $1 -f table/del_stable.py diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py index 0fe6ab71d5..f6d65a5c53 100644 --- a/tests/pytest/import_merge/importDataHO.py +++ b/tests/pytest/import_merge/importDataHO.py @@ -27,7 +27,7 @@ class TDTestCase: def run(self): self.ntables = 1 self.startTime = 1520000010000 - self.maxwrows = 200 + self.maxrows = 200 self.rowsPerTable = 20 tdDnodes.stop(1) diff --git a/tests/pytest/insert/binary-boundary.py b/tests/pytest/insert/binary-boundary.py new file mode 100644 index 0000000000..583217a732 --- /dev/null +++ b/tests/pytest/insert/binary-boundary.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed binary(4089))') + tdSql.error('create table tb (ts timestamp, speed binary(4089))') + tdLog.info('create table tb (ts timestamp, speed binary(4088))') + tdSql.error('create table tb (ts timestamp, speed binary(4088))') + tdLog.info('create table tb (ts timestamp, speed binary(4084))') + tdSql.execute('create table tb (ts timestamp, speed binary(4084))') + tdLog.info("insert into tb values (now, ) -x step1") + tdSql.error("insert into tb values (now, )") + + with open("../../README.md", "r") as inputFile: + data = inputFile.read(4084).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + + tdLog.info("insert %d length data: %s" % (len(data), data)) + + tdLog.info("insert into tb values (now+2a, data)") + tdSql.execute("insert into tb values (now+2a, '%s')" % data) + tdLog.info('select speed from tb order by ts desc') + tdSql.query('select speed from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('==> $data01') + tdLog.info("tdSql.checkData(0, 1, '%s')" % data) + tdSql.checkData(0, 1, data) + + tdLog.info( + 'create table tb2 (ts timestamp, speed binary(2040), temp binary(2044))') + tdSql.execute( + 'create table tb2 (ts timestamp, speed binary(2040), temp binary(2044))') + speed = inputFile.read(2044).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + temp = inputFile.read(2040).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + tdLog.info("insert into tb values (now+3a, speed, temp)") + tdSql.error( + "insert into tb values (now+3a, '%s', '%s')" % + (speed, temp)) + + speed = inputFile.read(2040).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + temp = inputFile.read(2044).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + tdLog.info("insert into tb values (now+4a, speed, temp)") + tdSql.error( + "insert into tb values (now+4a, '%s', '%s')" % + (speed, temp)) + + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('==> $data11') + tdLog.info("tdSql.checkData(1, 1, '%s')" % speed) + tdSql.checkData(1, 1, speed) + + tdLog.info('==> $data12') + tdLog.info("tdSql.checkData(1, 2, '%s')" % temp) + tdSql.checkData(1, 1, temp) + + inputFile.close() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/bool.py b/tests/pytest/insert/bool.py index 062563f4ab..c175afd8b5 100644 --- a/tests/pytest/insert/bool.py +++ b/tests/pytest/insert/bool.py @@ -58,6 +58,14 @@ class TDTestCase: tdSql.query('select * from tb order by ts desc') tdLog.info('tdSql.checkRow(6)') tdSql.checkRows(6) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, true)") + tdSql.execute("insert into tb values (now+5m, true)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) +# convert end # convert end def stop(self): diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py index 30d7e223f1..414833877e 100644 --- a/tests/pytest/insert/float.py +++ b/tests/pytest/insert/float.py @@ -43,7 +43,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = 'select * from tb order by ts desc' tdLog.info(cmd) @@ -82,7 +82,7 @@ class TDTestCase: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = "insert into tb values (now+4a, 0)" tdLog.info(cmd) @@ -103,7 +103,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = "insert into tb values (now+5a, 2)" tdLog.info(cmd) @@ -124,7 +124,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = "insert into tb values (now+6a, 2)" tdLog.info(cmd) diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py index d007cc56ea..350426a5bd 100644 --- a/tests/pytest/insert/int.py +++ b/tests/pytest/insert/int.py @@ -47,7 +47,7 @@ class TDTestCase: "This test failed: INT data overflow error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("INT data overflow error catched") + tdLog.info("INT data overflow error catched") cmd = 'insert into tb values (now+1m, NULL)' tdLog.info(cmd) @@ -76,7 +76,7 @@ class TDTestCase: "This test failed: INT data overflow error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("INT data overflow error catched") + tdLog.info("INT data overflow error catched") cmd = 'insert into tb values (now+3m, NULL)' tdLog.info(cmd) @@ -96,7 +96,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = 'insert into tb values (now+4m, 0)' tdLog.info(cmd) @@ -116,7 +116,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = 'insert into tb values (now+5m, 2)' tdLog.info(cmd) @@ -135,7 +135,7 @@ class TDTestCase: "This test failed: insert wrong data error _not_ catched") except Exception as e: tdLog.info(repr(e)) - tdLog.notice("insert wrong data error catched") + tdLog.info("insert wrong data error catched") cmd = 'insert into tb values (now+6m, 2)' tdLog.info(cmd) diff --git a/tests/pytest/insert/multi.py b/tests/pytest/insert/multi.py new file mode 100644 index 0000000000..c14d7dc2e0 --- /dev/null +++ b/tests/pytest/insert/multi.py @@ -0,0 +1,57 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + + tdLog.info("create table") + + tdSql.execute( + "create table if not exists st(ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + "CREATE TABLE if not exists dev_001 using st tags('dev_01')") + + print("==============step2") + tdLog.info("multiple inserts by insert") + tdSql.execute( + "insert INTO dev_001 VALUES ('2020-05-13 10:00:00.000', 1),('2020-05-13 10:00:00.001', 1)") + tdSql.checkAffectedRows(2) + + print("==============step3") + tdLog.info("multiple inserts by import") + tdSql.execute( + "import INTO dev_001 VALUES ('2020-05-13 10:00:00.000', 1),('2020-05-13 10:00:00.001', 1)") + tdSql.checkAffectedRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/nchar-boundary.py b/tests/pytest/insert/nchar-boundary.py new file mode 100644 index 0000000000..255cc5b79a --- /dev/null +++ b/tests/pytest/insert/nchar-boundary.py @@ -0,0 +1,65 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.error('create table tb (ts timestamp, col nchar(1022))') + tdSql.execute('create table tb (ts timestamp, col nchar(1021))') + tdSql.execute("insert into tb values (now, 'taosdata')") + tdSql.query("select * from tb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 'taosdata') + + with open("../../README.md", "r") as inputFile: + data = inputFile.read(1021).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + + tdLog.info("insert %d length data: %s" % (len(data), data)) + + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(2) + tdSql.checkData(1, 1, data) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/nchar-unicode.py b/tests/pytest/insert/nchar-unicode.py new file mode 100644 index 0000000000..12eef379d3 --- /dev/null +++ b/tests/pytest/insert/nchar-unicode.py @@ -0,0 +1,660 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.error('create table tb (ts timestamp, col nchar(1022))') + tdSql.execute('create table tb (ts timestamp, col nchar(1021))') + tdSql.execute("insert into tb values (now, 'taosdata')") + tdSql.query("select * from tb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 'taosdata') + + with open("../../README.md", "r") as inputFile: + data = inputFile.read(1021).replace( + "\n", + " ").replace( + "\\", + " ").replace( + "\'", + " ").replace( + "\"", + " ").replace( + "[", + " ").replace( + "]", + " ").replace( + "!", + " ") + + tdLog.info("insert %d length data: %s" % (len(data), data)) + + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(2) + tdSql.checkData(1, 1, data) + + # https://www.ltg.ed.ac.uk/~richard/unicode-sample.html + # Basic Latin + data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' + tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(3) + tdSql.checkData(2, 1, data) + + # Latin-1 Supplement + data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ' + tdLog.info( + "insert Latin-1 Supplement %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(4) + tdSql.checkData(3, 1, data) + + # Latin Extended-A + data = 'Ā ā Ă ă Ą ą Ć ć Ĉ ĉ Ċ ċ Č č Ď ď Đ đ Ē ē Ĕ ĕ Ė ė Ę ę Ě ě Ĝ ĝ Ğ ğ Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĩ ĩ Ī ī Ĭ ĭ Į į İ ı IJ ij Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ ŀ Ł ł Ń ń Ņ ņ Ň ň ʼn Ŋ ŋ Ō ō Ŏ ŏ Ő ő Œ œ Ŕ ŕ Ŗ ŗ Ř ř Ś ś Ŝ ŝ Ş ş Š š Ţ ţ Ť ť Ŧ ŧ Ũ ũ Ū ū Ŭ ŭ Ů ů Ű ű Ų ų Ŵ ŵ Ŷ ŷ Ÿ Ź ź Ż ż Ž ž ſ' + tdLog.info( + "insert Latin Extended-A %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(5) + tdSql.checkData(4, 1, data) + + # Latin Extended-B + data = 'ƀ Ɓ Ƃ ƃ Ƅ ƅ Ɔ Ƈ ƈ Ɖ Ɗ Ƌ ƌ ƍ Ǝ Ə Ɛ Ƒ ƒ Ɠ Ɣ ƕ Ɩ Ɨ Ƙ ƙ ƚ ƛ Ɯ Ɲ ƞ Ɵ Ơ ơ Ƣ ƣ Ƥ ƥ Ʀ Ƨ ƨ Ʃ ƪ ƫ Ƭ ƭ Ʈ Ư ư Ʊ Ʋ Ƴ ƴ Ƶ ƶ Ʒ Ƹ ƹ ƺ ƻ Ƽ ƽ ƾ ƿ ǀ ǁ ǂ ǃ DŽ Dž dž LJ Lj lj NJ Nj nj Ǎ ǎ Ǐ ǐ Ǒ ǒ Ǔ ǔ Ǖ ǖ Ǘ ǘ Ǚ ǚ Ǜ ǜ ǝ Ǟ ǟ Ǡ ǡ Ǣ ǣ Ǥ ǥ Ǧ ǧ Ǩ ǩ Ǫ ǫ Ǭ ǭ Ǯ ǯ ǰ DZ Dz dz Ǵ ǵ Ǻ ǻ Ǽ ǽ Ǿ ǿ Ȁ ȁ Ȃ ȃ ...' + tdLog.info( + "insert Latin Extended-B %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(6) + tdSql.checkData(5, 1, data) + + # IPA Extensions + data = 'ɐ ɑ ɒ ɓ ɔ ɕ ɖ ɗ ɘ ə ɚ ɛ ɜ ɝ ɞ ɟ ɠ ɡ ɢ ɣ ɤ ɥ ɦ ɧ ɨ ɩ ɪ ɫ ɬ ɭ ɮ ɯ ɰ ɱ ɲ ɳ ɴ ɵ ɶ ɷ ɸ ɹ ɺ ɻ ɼ ɽ ɾ ɿ ʀ ʁ ʂ ʃ ʄ ʅ ʆ ʇ ʈ ʉ ʊ ʋ ʌ ʍ ʎ ʏ ʐ ʑ ʒ ʓ ʔ ʕ ʖ ʗ ʘ ʙ ʚ ʛ ʜ ʝ ʞ ʟ ʠ ʡ ʢ ʣ ʤ ʥ ʦ ʧ ʨ' + tdLog.info( + "insert IPA Extensions %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(7) + tdSql.checkData(6, 1, data) + + # Spacing Modifier Letters + data = 'ʰ ʱ ʲ ʳ ʴ ʵ ʶ ʷ ʸ ʹ ʺ ʻ ʼ ʽ ʾ ʿ ˀ ˁ ˂ ˃ ˄ ˅ ˆ ˇ ˈ ˉ ˊ ˋ ˌ ˍ ˎ ˏ ː ˑ ˒ ˓ ˔ ˕ ˖ ˗ ˘ ˙ ˚ ˛ ˜ ˝ ˞ ˠ ˡ ˢ ˣ ˤ ˥ ˦ ˧ ˨ ˩' + tdLog.info( + "insert Spacing Modifier Letters %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(8) + tdSql.checkData(7, 1, data) + + # Combining Diacritical Marks + data = '̀ ́ ̂ ̃ ̄ ̅ ̆ ̇ ̈ ̉ ̊ ̋ ̌ ̍ ̎ ̏ ̐ ̑ ̒ ̓ ̔ ̕ ̖ ̗ ̘ ̙ ̚ ̛ ̜ ̝ ̞ ̟ ̠ ̡ ̢ ̣ ̤ ̥ ̦ ̧ ̨ ̩ ̪ ̫ ̬ ̭ ̮ ̯ ̰ ̱ ̲ ̳ ̴ ̵ ̶ ̷ ̸ ̹ ̺ ̻ ̼ ̽ ̾ ̿ ̀ ́ ͂ ̓ ̈́ ͅ ͠ ͡' + tdLog.info( + "insert Combining Diacritical Marks %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(9) + tdSql.checkData(8, 1, data) + + # Greek + data = 'ʹ ͵ ͺ ; ΄ ΅ Ά · Έ Ή Ί Ό Ύ Ώ ΐ Α Β Γ Δ Ε Ζ Η Θ Ι Κ Λ Μ Ν Ξ Ο Π Ρ Σ Τ Υ Φ Χ Ψ Ω Ϊ Ϋ ά έ ή ί ΰ α β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ ς σ τ υ φ χ ψ ω ϊ ϋ ό ύ ώ ϐ ϑ ϒ ϓ ϔ ϕ ϖ Ϛ Ϝ Ϟ Ϡ Ϣ ϣ Ϥ ϥ Ϧ ϧ Ϩ ϩ Ϫ ϫ Ϭ ϭ Ϯ ϯ ϰ ϱ ϲ ϳ' + tdLog.info("insert Greek %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(10) + tdSql.checkData(9, 1, data) + + # Cyrillic + data = 'Ё Ђ Ѓ Є Ѕ І Ї Ј Љ Њ Ћ Ќ Ў Џ А Б В Г Д Е Ж З И Й К Л М Н О П Р С Т У Ф Х Ц Ч Ш Щ Ъ Ы Ь Э Ю Я а б в г д е ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я ё ђ ѓ є ѕ і ї ј љ њ ћ ќ ў џ Ѡ ѡ Ѣ ѣ Ѥ ѥ Ѧ ѧ Ѩ ѩ Ѫ ѫ Ѭ ѭ Ѯ ѯ Ѱ ѱ Ѳ ѳ Ѵ ѵ Ѷ ѷ Ѹ ѹ Ѻ ѻ Ѽ ѽ Ѿ ѿ Ҁ ҁ ҂ ҃ ...' + tdLog.info("insert Cyrillic %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(11) + tdSql.checkData(10, 1, data) + + # Armenian + data = 'Ա Բ Գ Դ Ե Զ Է Ը Թ Ժ Ի Լ Խ Ծ Կ Հ Ձ Ղ Ճ Մ Յ Ն Շ Ո Չ Պ Ջ Ռ Ս Վ Տ Ր Ց Ւ Փ Ք Օ Ֆ ՙ ՚ ՛ ՜ ՝ ՞ ՟ ա բ գ դ ե զ է ը թ ժ ի լ խ ծ կ հ ձ ղ ճ մ յ ն շ ո չ պ ջ ռ ս վ տ ր ց ւ փ ք օ ֆ և ։' + tdLog.info("insert Armenian %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(12) + tdSql.checkData(11, 1, data) + + # Hebrew + data = ' ֒ ֓ ֔ ֕ ֖ ֗ ֘ ֙ ֚ ֛ ֜ ֝ ֞ ֟ ֠ ֡ ֣ ֤ ֥ ֦ ֧ ֨ ֩ ֪ ֫ ֬ ֭ ֮ ֯ ְ ֱ ֲ ֳ ִ ֵ ֶ ַ ָ ֹ ֻ ּ ֽ ־ ֿ ׀ ׁ ׂ ׃ ׄ א ב ג ד ה ו ז ח ט י ך כ ל ם מ ן נ ס ע ף פ ץ צ ק ר ש ת װ ױ ײ ׳ ״' + tdLog.info("insert Hebrew %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(13) + tdSql.checkData(12, 1, data) + + # Arabic + data = '، ؛ ؟ ء آ أ ؤ إ ئ ا ب ة ت ث ج ح خ د ذ ر ز س ش ص ض ط ظ ع غ ـ ف ق ك ل م ن ه و ى ي ً ٌ ٍ َ ُ ِ ّ ْ ٠ ١ ٢ ٣ ٤ ٥ ٦ ٧ ٨ ٩ ٪ ٫ ٬ ٭ ٰ ٱ ٲ ٳ ٴ ٵ ٶ ٷ ٸ ٹ ٺ ٻ ټ ٽ پ ٿ ڀ ځ ڂ ڃ ڄ څ چ ڇ ڈ ډ ڊ ڋ ڌ ڍ ڎ ڏ ڐ ڑ ڒ ړ ڔ ڕ ږ ڗ ژ ڙ ښ ڛ ڜ ڝ ڞ ڟ ڠ ڡ ڢ ڣ ڤ ڥ ڦ ڧ ڨ ک ڪ ګ ڬ ڭ ڮ گ ڰ ڱ ...' + tdLog.info( + "FAILED: insert Arabic %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(14) + tdSql.checkData(13, 1, data) + + # Devanagari + data = 'ँ ं ः अ आ इ ई उ ऊ ऋ ऌ ऍ ऎ ए ऐ ऑ ऒ ओ औ क ख ग घ ङ च छ ज झ ञ ट ठ ड ढ ण त थ द ध न ऩ प फ ब भ म य र ऱ ल ळ ऴ व श ष स ह ़ ऽ ा ि ी ु ू ृ ॄ ॅ ॆ े ै ॉ ॊ ो ौ ् ॐ ॑ ॒ ॓ ॔ क़ ख़ ग़ ज़ ड़ ढ़ फ़ य़ ॠ ॡ ॢ ॣ । ॥ ० १ २ ३ ४ ५ ६ ७ ८ ९ ॰' + tdLog.info("insert Devanagari %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(15) + tdSql.checkData(14, 1, data) + + # Bengali + data = 'ঁ ং ঃ অ আ ই ঈ উ ঊ ঋ ঌ এ ঐ ও ঔ ক খ গ ঘ ঙ চ ছ জ ঝ ঞ ট ঠ ড ঢ ণ ত থ দ ধ ন প ফ ব ভ ম য র ল শ ষ স হ ় া ি ী ু ূ ৃ ৄ ে ৈ ো ৌ ্ ৗ ড় ঢ় য় ৠ ৡ ৢ ৣ ০ ১ ২ ৩ ৪ ৫ ৬ ৭ ৮ ৯ ৰ ৱ ৲ ৳ ৴ ৵ ৶ ৷ ৸ ৹ ৺' + tdLog.info("insert Bengali %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(16) + tdSql.checkData(15, 1, data) + + # Gurmukhi + data = 'ਂ ਅ ਆ ਇ ਈ ਉ ਊ ਏ ਐ ਓ ਔ ਕ ਖ ਗ ਘ ਙ ਚ ਛ ਜ ਝ ਞ ਟ ਠ ਡ ਢ ਣ ਤ ਥ ਦ ਧ ਨ ਪ ਫ ਬ ਭ ਮ ਯ ਰ ਲ ਲ਼ ਵ ਸ਼ ਸ ਹ ਼ ਾ ਿ ੀ ੁ ੂ ੇ ੈ ੋ ੌ ੍ ਖ਼ ਗ਼ ਜ਼ ੜ ਫ਼ ੦ ੧ ੨ ੩ ੪ ੫ ੬ ੭ ੮ ੯ ੰ ੱ ੲ ੳ ੴ' + tdLog.info("insert Gurmukhi %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(17) + tdSql.checkData(16, 1, data) + + # Gujarati + data = 'ઁ ં ઃ અ આ ઇ ઈ ઉ ઊ ઋ ઍ એ ઐ ઑ ઓ ઔ ક ખ ગ ઘ ઙ ચ છ જ ઝ ઞ ટ ઠ ડ ઢ ણ ત થ દ ધ ન પ ફ બ ભ મ ય ર લ ળ વ શ ષ સ હ ઼ ઽ ા િ ી ુ ૂ ૃ ૄ ૅ ે ૈ ૉ ો ૌ ્ ૐ ૠ ૦ ૧ ૨ ૩ ૪ ૫ ૬ ૭ ૮ ૯' + tdLog.info("insert Gujarati %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(18) + tdSql.checkData(17, 1, data) + + # Oriya + data = 'ଁ ଂ ଃ ଅ ଆ ଇ ଈ ଉ ଊ ଋ ଌ ଏ ଐ ଓ ଔ କ ଖ ଗ ଘ ଙ ଚ ଛ ଜ ଝ ଞ ଟ ଠ ଡ ଢ ଣ ତ ଥ ଦ ଧ ନ ପ ଫ ବ ଭ ମ ଯ ର ଲ ଳ ଶ ଷ ସ ହ ଼ ଽ ା ି ୀ ୁ ୂ ୃ େ ୈ ୋ ୌ ୍ ୖ ୗ ଡ଼ ଢ଼ ୟ ୠ ୡ ୦ ୧ ୨ ୩ ୪ ୫ ୬ ୭ ୮ ୯ ୰' + tdLog.info("insert Oriya %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(19) + tdSql.checkData(18, 1, data) + + # Tamil + data = 'ஂ ஃ அ ஆ இ ஈ உ ஊ எ ஏ ஐ ஒ ஓ ஔ க ங ச ஜ ஞ ட ண த ந ன ப ம ய ர ற ல ள ழ வ ஷ ஸ ஹ ா ி ீ ு ூ ெ ே ை ொ ோ ௌ ் ௗ ௧ ௨ ௩ ௪ ௫ ௬ ௭ ௮ ௯ ௰ ௱ ௲' + tdLog.info("insert Tamil %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(20) + tdSql.checkData(19, 1, data) + + # Telugu + data = 'ఁ ం ః అ ఆ ఇ ఈ ఉ ఊ ఋ ఌ ఎ ఏ ఐ ఒ ఓ ఔ క ఖ గ ఘ ఙ చ ఛ జ ఝ ఞ ట ఠ డ ఢ ణ త థ ద ధ న ప ఫ బ భ మ య ర ఱ ల ళ వ శ ష స హ ా ి ీ ు ూ ృ ౄ ె ే ై ొ ో ౌ ్ ౕ ౖ ౠ ౡ ౦ ౧ ౨ ౩ ౪ ౫ ౬ ౭ ౮ ౯' + tdLog.info("insert Telugu %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(21) + tdSql.checkData(20, 1, data) + + # Kannada + data = 'ಂ ಃ ಅ ಆ ಇ ಈ ಉ ಊ ಋ ಌ ಎ ಏ ಐ ಒ ಓ ಔ ಕ ಖ ಗ ಘ ಙ ಚ ಛ ಜ ಝ ಞ ಟ ಠ ಡ ಢ ಣ ತ ಥ ದ ಧ ನ ಪ ಫ ಬ ಭ ಮ ಯ ರ ಱ ಲ ಳ ವ ಶ ಷ ಸ ಹ ಾ ಿ ೀ ು ೂ ೃ ೄ ೆ ೇ ೈ ೊ ೋ ೌ ್ ೕ ೖ ೞ ೠ ೡ ೦ ೧ ೨ ೩ ೪ ೫ ೬ ೭ ೮ ೯' + tdLog.info("insert Kannada %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(22) + tdSql.checkData(21, 1, data) + + # Malayalam + data = 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ ഐ ഒ ഓ ഔ ക ഖ ഗ ഘ ങ ച ഛ ജ ഝ ഞ ട ഠ ഡ ഢ ണ ത ഥ ദ ധ ന പ ഫ ബ ഭ മ യ ര റ ല ള ഴ വ ശ ഷ സ ഹ ാ ി ീ ു ൂ ൃ െ േ ൈ ൊ ോ ൌ ് ൗ ൠ ൡ ൦ ൧ ൨ ൩ ൪ ൫ ൬ ൭ ൮ ൯' + tdLog.info("insert Malayalam %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(23) + tdSql.checkData(22, 1, data) + + # Thai + data = 'ก ข ฃ ค ฅ ฆ ง จ ฉ ช ซ ฌ ญ ฎ ฏ ฐ ฑ ฒ ณ ด ต ถ ท ธ น บ ป ผ ฝ พ ฟ ภ ม ย ร ฤ ล ฦ ว ศ ษ ส ห ฬ อ ฮ ฯ ะ ั า ำ ิ ี ึ ื ุ ู ฺ ฿ เ แ โ ใ ไ ๅ ๆ ็ ่ ้ ๊ ๋ ์ ํ ๎ ๏ ๐ ๑ ๒ ๓ ๔ ๕ ๖ ๗ ๘ ๙ ๚ ๛' + tdLog.info("insert Thai %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(24) + tdSql.checkData(23, 1, data) + + # Thai + data = 'ก ข ฃ ค ฅ ฆ ง จ ฉ ช ซ ฌ ญ ฎ ฏ ฐ ฑ ฒ ณ ด ต ถ ท ธ น บ ป ผ ฝ พ ฟ ภ ม ย ร ฤ ล ฦ ว ศ ษ ส ห ฬ อ ฮ ฯ ะ ั า ำ ิ ี ึ ื ุ ู ฺ ฿ เ แ โ ใ ไ ๅ ๆ ็ ่ ้ ๊ ๋ ์ ํ ๎ ๏ ๐ ๑ ๒ ๓ ๔ ๕ ๖ ๗ ๘ ๙ ๚ ๛' + tdLog.info("insert Thai %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(25) + tdSql.checkData(24, 1, data) + + # Lao + data = 'ກ ຂ ຄ ງ ຈ ຊ ຍ ດ ຕ ຖ ທ ນ ບ ປ ຜ ຝ ພ ຟ ມ ຢ ຣ ລ ວ ສ ຫ ອ ຮ ຯ ະ ັ າ ຳ ິ ີ ຶ ື ຸ ູ ົ ຼ ຽ ເ ແ ໂ ໃ ໄ ໆ ່ ້ ໊ ໋ ໌ ໍ ໐ ໑ ໒ ໓ ໔ ໕ ໖ ໗ ໘ ໙ ໜ ໝ' + tdLog.info("insert Lao %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(26) + tdSql.checkData(25, 1, data) + + # Tibetan + data = 'ༀ ༁ ༂ ༃ ༄ ༅ ༆ ༇ ༈ ༉ ༊ ་ ༌ ། ༎ ༏ ༐ ༑ ༒ ༓ ༔ ༕ ༖ ༗ ༘ ༙ ༚ ༛ ༜ ༝ ༞ ༟ ༠ ༡ ༢ ༣ ༤ ༥ ༦ ༧ ༨ ༩ ༪ ༫ ༬ ༭ ༮ ༯ ༰ ༱ ༲ ༳ ༴ ༵ ༶ ༷ ༸ ༹ ༺ ༻ ༼ ༽ ༾ ༿ ཀ ཁ ག གྷ ང ཅ ཆ ཇ ཉ ཊ ཋ ཌ ཌྷ ཎ ཏ ཐ ད དྷ ན པ ཕ བ བྷ མ ཙ ཚ ཛ ཛྷ ཝ ཞ ཟ འ ཡ ར ལ ཤ ཥ ས ཧ ཨ ཀྵ ཱ ི ཱི ུ ཱུ ྲྀ ཷ ླྀ ཹ ེ ཻ ོ ཽ ཾ ཿ ྀ ཱྀ ྂ ྃ ྄ ྅ ྆ ྇ ...' + tdLog.info("insert Tibetan %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(27) + tdSql.checkData(26, 1, data) + + # Georgian + data = 'Ⴀ Ⴁ Ⴂ Ⴃ Ⴄ Ⴅ Ⴆ Ⴇ Ⴈ Ⴉ Ⴊ Ⴋ Ⴌ Ⴍ Ⴎ Ⴏ Ⴐ Ⴑ Ⴒ Ⴓ Ⴔ Ⴕ Ⴖ Ⴗ Ⴘ Ⴙ Ⴚ Ⴛ Ⴜ Ⴝ Ⴞ Ⴟ Ⴠ Ⴡ Ⴢ Ⴣ Ⴤ Ⴥ ა ბ გ დ ე ვ ზ თ ი კ ლ მ ნ ო პ ჟ რ ს ტ უ ფ ქ ღ ყ შ ჩ ც ძ წ ჭ ხ ჯ ჰ ჱ ჲ ჳ ჴ ჵ ჶ ჻' + tdLog.info("insert Georgian %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(28) + tdSql.checkData(27, 1, data) + + # Hangul Jamo + data = 'ᄀ ᄁ ᄂ ᄃ ᄄ ᄅ ᄆ ᄇ ᄈ ᄉ ᄊ ᄋ ᄌ ᄍ ᄎ ᄏ ᄐ ᄑ ᄒ ᄓ ᄔ ᄕ ᄖ ᄗ ᄘ ᄙ ᄚ ᄛ ᄜ ᄝ ᄞ ᄟ ᄠ ᄡ ᄢ ᄣ ᄤ ᄥ ᄦ ᄧ ᄨ ᄩ ᄪ ᄫ ᄬ ᄭ ᄮ ᄯ ᄰ ᄱ ᄲ ᄳ ᄴ ᄵ ᄶ ᄷ ᄸ ᄹ ᄺ ᄻ ᄼ ᄽ ᄾ ᄿ ᅀ ᅁ ᅂ ᅃ ᅄ ᅅ ᅆ ᅇ ᅈ ᅉ ᅊ ᅋ ᅌ ᅍ ᅎ ᅏ ᅐ ᅑ ᅒ ᅓ ᅔ ᅕ ᅖ ᅗ ᅘ ᅙ ᅟ ᅠ ᅡ ᅢ ᅣ ᅤ ᅥ ᅦ ᅧ ᅨ ᅩ ᅪ ᅫ ᅬ ᅭ ᅮ ᅯ ᅰ ᅱ ᅲ ᅳ ᅴ ᅵ ᅶ ᅷ ᅸ ᅹ ᅺ ᅻ ᅼ ᅽ ᅾ ᅿ ᆀ ᆁ ᆂ ᆃ ᆄ ...' + tdLog.info("insert Hangul Jamo %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(29) + tdSql.checkData(28, 1, data) + + # Latin Extended Additional + data = 'Ḁ ḁ Ḃ ḃ Ḅ ḅ Ḇ ḇ Ḉ ḉ Ḋ ḋ Ḍ ḍ Ḏ ḏ Ḑ ḑ Ḓ ḓ Ḕ ḕ Ḗ ḗ Ḙ ḙ Ḛ ḛ Ḝ ḝ Ḟ ḟ Ḡ ḡ Ḣ ḣ Ḥ ḥ Ḧ ḧ Ḩ ḩ Ḫ ḫ Ḭ ḭ Ḯ ḯ Ḱ ḱ Ḳ ḳ Ḵ ḵ Ḷ ḷ Ḹ ḹ Ḻ ḻ Ḽ ḽ Ḿ ḿ Ṁ ṁ Ṃ ṃ Ṅ ṅ Ṇ ṇ Ṉ ṉ Ṋ ṋ Ṍ ṍ Ṏ ṏ Ṑ ṑ Ṓ ṓ Ṕ ṕ Ṗ ṗ Ṙ ṙ Ṛ ṛ Ṝ ṝ Ṟ ṟ Ṡ ṡ Ṣ ṣ Ṥ ṥ Ṧ ṧ Ṩ ṩ Ṫ ṫ Ṭ ṭ Ṯ ṯ Ṱ ṱ Ṳ ṳ Ṵ ṵ Ṷ ṷ Ṹ ṹ Ṻ ṻ Ṽ ṽ Ṿ ṿ ...' + tdLog.info( + "insert Latin Extended Additional %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(30) + tdSql.checkData(29, 1, data) + + # Geek Extended + data = 'ἀ ἁ ἂ ἃ ἄ ἅ ἆ ἇ Ἀ Ἁ Ἂ Ἃ Ἄ Ἅ Ἆ Ἇ ἐ ἑ ἒ ἓ ἔ ἕ Ἐ Ἑ Ἒ Ἓ Ἔ Ἕ ἠ ἡ ἢ ἣ ἤ ἥ ἦ ἧ Ἠ Ἡ Ἢ Ἣ Ἤ Ἥ Ἦ Ἧ ἰ ἱ ἲ ἳ ἴ ἵ ἶ ἷ Ἰ Ἱ Ἲ Ἳ Ἴ Ἵ Ἶ Ἷ ὀ ὁ ὂ ὃ ὄ ὅ Ὀ Ὁ Ὂ Ὃ Ὄ Ὅ ὐ ὑ ὒ ὓ ὔ ὕ ὖ ὗ Ὑ Ὓ Ὕ Ὗ ὠ ὡ ὢ ὣ ὤ ὥ ὦ ὧ Ὠ Ὡ Ὢ Ὣ Ὤ Ὥ Ὦ Ὧ ὰ ά ὲ έ ὴ ή ὶ ί ὸ ό ὺ ύ ὼ ώ ᾀ ᾁ ᾂ ᾃ ᾄ ᾅ ᾆ ᾇ ᾈ ᾉ ᾊ ᾋ ᾌ ᾍ ...' + tdLog.info( + "insert Geek Extended %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(31) + tdSql.checkData(30, 1, data) + + # General Punctuation + data = '                      ‐ ‑ ‒ – — ― ‖ ‗ ‘ ’ ‚ ‛ “ ” „ ‟ † ‡ • ‣ ․ ‥ … ‧ 
 
 ' + tdLog.info( + "insert General Punctuation %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(32) + tdSql.checkData(31, 1, data) + + # Superscripts and Subscripts + data = '⁰ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁺ ⁻ ⁼ ⁽ ⁾ ⁿ ₀ ₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₊ ₋ ₌ ₍ ₎' + tdLog.info( + "insert Superscripts and Subscripts %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(33) + tdSql.checkData(32, 1, data) + + # Currency Symbols + data = '₠ ₡ ₢ ₣ ₤ ₥ ₦ ₧ ₨ ₩ ₪ ₫' + tdLog.info( + "insert Currency Symbols %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(34) + tdSql.checkData(33, 1, data) + + # Combining Marks for Symbols + data = '⃐ ⃑ ⃒ ⃓ ⃔ ⃕ ⃖ ⃗ ⃘ ⃙ ⃚ ⃛ ⃜ ⃝ ⃞ ⃟ ⃠ ⃡' + tdLog.info( + "insert Combining Marks for Symbols %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(35) + tdSql.checkData(34, 1, data) + + # Letterlike Symbols + data = '℀ ℁ ℂ ℃ ℄ ℅ ℆ ℇ ℈ ℉ ℊ ℋ ℌ ℍ ℎ ℏ ℐ ℑ ℒ ℓ ℔ ℕ № ℗ ℘ ℙ ℚ ℛ ℜ ℝ ℞ ℟ ℠ ℡ ™ ℣ ℤ ℥ Ω ℧ ℨ ℩ K Å ℬ ℭ ℮ ℯ ℰ ℱ Ⅎ ℳ ℴ ℵ ℶ ℷ ℸ' + tdLog.info( + "insert Letterlike Symbols %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(36) + tdSql.checkData(35, 1, data) + + # Number Forms + data = '⅓ ⅔ ⅕ ⅖ ⅗ ⅘ ⅙ ⅚ ⅛ ⅜ ⅝ ⅞ ⅟ Ⅰ Ⅱ Ⅲ Ⅳ Ⅴ Ⅵ Ⅶ Ⅷ Ⅸ Ⅹ Ⅺ Ⅻ Ⅼ Ⅽ Ⅾ Ⅿ ⅰ ⅱ ⅲ ⅳ ⅴ ⅵ ⅶ ⅷ ⅸ ⅹ ⅺ ⅻ ⅼ ⅽ ⅾ ⅿ ↀ ↁ ↂ' + tdLog.info( + "insert Number Forms %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(37) + tdSql.checkData(36, 1, data) + + # Arrows + data = '← ↑ → ↓ ↔ ↕ ↖ ↗ ↘ ↙ ↚ ↛ ↜ ↝ ↞ ↟ ↠ ↡ ↢ ↣ ↤ ↥ ↦ ↧ ↨ ↩ ↪ ↫ ↬ ↭ ↮ ↯ ↰ ↱ ↲ ↳ ↴ ↵ ↶ ↷ ↸ ↹ ↺ ↻ ↼ ↽ ↾ ↿ ⇀ ⇁ ⇂ ⇃ ⇄ ⇅ ⇆ ⇇ ⇈ ⇉ ⇊ ⇋ ⇌ ⇍ ⇎ ⇏ ⇐ ⇑ ⇒ ⇓ ⇔ ⇕ ⇖ ⇗ ⇘ ⇙ ⇚ ⇛ ⇜ ⇝ ⇞ ⇟ ⇠ ⇡ ⇢ ⇣ ⇤ ⇥ ⇦ ⇧ ⇨ ⇩ ⇪' + tdLog.info("insert Arrows %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(38) + tdSql.checkData(37, 1, data) + + # Mathematical Operators + data = '∀ ∁ ∂ ∃ ∄ ∅ ∆ ∇ ∈ ∉ ∊ ∋ ∌ ∍ ∎ ∏ ∐ ∑ − ∓ ∔ ∕ ∖ ∗ ∘ ∙ √ ∛ ∜ ∝ ∞ ∟ ∠ ∡ ∢ ∣ ∤ ∥ ∦ ∧ ∨ ∩ ∪ ∫ ∬ ∭ ∮ ∯ ∰ ∱ ∲ ∳ ∴ ∵ ∶ ∷ ∸ ∹ ∺ ∻ ∼ ∽ ∾ ∿ ≀ ≁ ≂ ≃ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≏ ≐ ≑ ≒ ≓ ≔ ≕ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≠ ≡ ≢ ≣ ≤ ≥ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ...' + tdLog.info( + "insert Mathematical Operators %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(39) + tdSql.checkData(38, 1, data) + + # Miscellaneous Technical + data = '⌀ ⌂ ⌃ ⌄ ⌅ ⌆ ⌇ ⌈ ⌉ ⌊ ⌋ ⌌ ⌍ ⌎ ⌏ ⌐ ⌑ ⌒ ⌓ ⌔ ⌕ ⌖ ⌗ ⌘ ⌙ ⌚ ⌛ ⌜ ⌝ ⌞ ⌟ ⌠ ⌡ ⌢ ⌣ ⌤ ⌥ ⌦ ⌧ ⌨ 〈 〉 ⌫ ⌬ ⌭ ⌮ ⌯ ⌰ ⌱ ⌲ ⌳ ⌴ ⌵ ⌶ ⌷ ⌸ ⌹ ⌺ ⌻ ⌼ ⌽ ⌾ ⌿ ⍀ ⍁ ⍂ ⍃ ⍄ ⍅ ⍆ ⍇ ⍈ ⍉ ⍊ ⍋ ⍌ ⍍ ⍎ ⍏ ⍐ ⍑ ⍒ ⍓ ⍔ ⍕ ⍖ ⍗ ⍘ ⍙ ⍚ ⍛ ⍜ ⍝ ⍞ ⍟ ⍠ ⍡ ⍢ ⍣ ⍤ ⍥ ⍦ ⍧ ⍨ ⍩ ⍪ ⍫ ⍬ ⍭ ⍮ ⍯ ⍰ ⍱ ⍲ ⍳ ⍴ ⍵ ⍶ ⍷ ⍸ ⍹ ⍺' + tdLog.info( + "insert Miscellaneous Technical %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(40) + tdSql.checkData(39, 1, data) + + # Control Pictures + data = '␀ ␁ ␂ ␃ ␄ ␅ ␆ ␇ ␈ ␉ ␊ ␋ ␌ ␍ ␎ ␏ ␐ ␑ ␒ ␓ ␔ ␕ ␖ ␗ ␘ ␙ ␚ ␛ ␜ ␝ ␞ ␟ ␠ ␡ ␢ ␣ ␤' + tdLog.info( + "insert Control Pictures %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(41) + tdSql.checkData(40, 1, data) + + # Optical Character Recognition + data = '⑀ ⑁ ⑂ ⑃ ⑄ ⑅ ⑆ ⑇ ⑈ ⑉ ⑊' + tdLog.info( + "insert Optical Character Recognition %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(42) + tdSql.checkData(41, 1, data) + + # Enclosed Alphanumerics + data = '① ② ③ ④ ⑤ ⑥ ⑦ ⑧ ⑨ ⑩ ⑪ ⑫ ⑬ ⑭ ⑮ ⑯ ⑰ ⑱ ⑲ ⑳ ⑴ ⑵ ⑶ ⑷ ⑸ ⑹ ⑺ ⑻ ⑼ ⑽ ⑾ ⑿ ⒀ ⒁ ⒂ ⒃ ⒄ ⒅ ⒆ ⒇ ⒈ ⒉ ⒊ ⒋ ⒌ ⒍ ⒎ ⒏ ⒐ ⒑ ⒒ ⒓ ⒔ ⒕ ⒖ ⒗ ⒘ ⒙ ⒚ ⒛ ⒜ ⒝ ⒞ ⒟ ⒠ ⒡ ⒢ ⒣ ⒤ ⒥ ⒦ ⒧ ⒨ ⒩ ⒪ ⒫ ⒬ ⒭ ⒮ ⒯ ⒰ ⒱ ⒲ ⒳ ⒴ ⒵ Ⓐ Ⓑ Ⓒ Ⓓ Ⓔ Ⓕ Ⓖ Ⓗ Ⓘ Ⓙ Ⓚ Ⓛ Ⓜ Ⓝ Ⓞ Ⓟ Ⓠ Ⓡ Ⓢ Ⓣ Ⓤ Ⓥ Ⓦ Ⓧ Ⓨ Ⓩ ⓐ ⓑ ⓒ ⓓ ⓔ ⓕ ⓖ ⓗ ⓘ ⓙ ⓚ ⓛ ⓜ ⓝ ⓞ ⓟ ...' + tdLog.info( + "insert Enclosed Alphanumerics %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(43) + tdSql.checkData(42, 1, data) + + # Box Drawing + data = '─ ━ │ ┃ ┄ ┅ ┆ ┇ ┈ ┉ ┊ ┋ ┌ ┍ ┎ ┏ ┐ ┑ ┒ ┓ └ ┕ ┖ ┗ ┘ ┙ ┚ ┛ ├ ┝ ┞ ┟ ┠ ┡ ┢ ┣ ┤ ┥ ┦ ┧ ┨ ┩ ┪ ┫ ┬ ┭ ┮ ┯ ┰ ┱ ┲ ┳ ┴ ┵ ┶ ┷ ┸ ┹ ┺ ┻ ┼ ┽ ┾ ┿ ╀ ╁ ╂ ╃ ╄ ╅ ╆ ╇ ╈ ╉ ╊ ╋ ╌ ╍ ╎ ╏ ═ ║ ╒ ╓ ╔ ╕ ╖ ╗ ╘ ╙ ╚ ╛ ╜ ╝ ╞ ╟ ╠ ╡ ╢ ╣ ╤ ╥ ╦ ╧ ╨ ╩ ╪ ╫ ╬ ╭ ╮ ╯ ╰ ╱ ╲ ╳ ╴ ╵ ╶ ╷ ╸ ╹ ╺ ╻ ╼ ╽ ╾ ╿' + tdLog.info("insert Box Drawing %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(44) + tdSql.checkData(43, 1, data) + + # Block Elements + data = '▀ ▁ ▂ ▃ ▄ ▅ ▆ ▇ █ ▉ ▊ ▋ ▌ ▍ ▎ ▏ ▐ ░ ▒ ▓ ▔ ▕' + tdLog.info( + "insert Block Elements %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(45) + tdSql.checkData(44, 1, data) + + # Geometric Shapes + data = '■ □ ▢ ▣ ▤ ▥ ▦ ▧ ▨ ▩ ▪ ▫ ▬ ▭ ▮ ▯ ▰ ▱ ▲ △ ▴ ▵ ▶ ▷ ▸ ▹ ► ▻ ▼ ▽ ▾ ▿ ◀ ◁ ◂ ◃ ◄ ◅ ◆ ◇ ◈ ◉ ◊ ○ ◌ ◍ ◎ ● ◐ ◑ ◒ ◓ ◔ ◕ ◖ ◗ ◘ ◙ ◚ ◛ ◜ ◝ ◞ ◟ ◠ ◡ ◢ ◣ ◤ ◥ ◦ ◧ ◨ ◩ ◪ ◫ ◬ ◭ ◮ ◯' + tdLog.info( + "insert Geometric Shapes %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(46) + tdSql.checkData(45, 1, data) + + # Miscellaneous Symbols + data = '☀ ☁ ☂ ☃ ☄ ★ ☆ ☇ ☈ ☉ ☊ ☋ ☌ ☍ ☎ ☏ ☐ ☑ ☒ ☓ ☚ ☛ ☜ ☝ ☞ ☟ ☠ ☡ ☢ ☣ ☤ ☥ ☦ ☧ ☨ ☩ ☪ ☫ ☬ ☭ ☮ ☯ ☰ ☱ ☲ ☳ ☴ ☵ ☶ ☷ ☸ ☹ ☺ ☻ ☼ ☽ ☾ ☿ ♀ ♁ ♂ ♃ ♄ ♅ ♆ ♇ ♈ ♉ ♊ ♋ ♌ ♍ ♎ ♏ ♐ ♑ ♒ ♓ ♔ ♕ ♖ ♗ ♘ ♙ ♚ ♛ ♜ ♝ ♞ ♟ ♠ ♡ ♢ ♣ ♤ ♥ ♦ ♧ ♨ ♩ ♪ ♫ ♬ ♭ ♮ ♯' + tdLog.info( + "insert Miscellaneous Symbols %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(47) + tdSql.checkData(46, 1, data) + + # Dingbats + data = '✁ ✂ ✃ ✄ ✆ ✇ ✈ ✉ ✌ ✍ ✎ ✏ ✐ ✑ ✒ ✓ ✔ ✕ ✖ ✗ ✘ ✙ ✚ ✛ ✜ ✝ ✞ ✟ ✠ ✡ ✢ ✣ ✤ ✥ ✦ ✧ ✩ ✪ ✫ ✬ ✭ ✮ ✯ ✰ ✱ ✲ ✳ ✴ ✵ ✶ ✷ ✸ ✹ ✺ ✻ ✼ ✽ ✾ ✿ ❀ ❁ ❂ ❃ ❄ ❅ ❆ ❇ ❈ ❉ ❊ ❋ ❍ ❏ ❐ ❑ ❒ ❖ ❘ ❙ ❚ ❛ ❜ ❝ ❞ ❡ ❢ ❣ ❤ ❥ ❦ ❧ ❶ ❷ ❸ ❹ ❺ ❻ ❼ ❽ ❾ ❿ ➀ ➁ ➂ ➃ ➄ ➅ ➆ ➇ ➈ ➉ ➊ ➋ ➌ ➍ ➎ ➏ ➐ ➑ ➒ ➓ ➔ ➘ ➙ ➚ ➛ ➜ ➝ ...' + tdLog.info("insert Dingbats %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(48) + tdSql.checkData(47, 1, data) + + # CJK Symbols and Punctuation + data = '、 。 〃 〄 々 〆 〇 〈 〉 《 》 「 」 『 』 【 】 〒 〓 〔 〕 〖 〗 〘 〙 〚 〛 〜 〝 〞 〟 〠 〡 〢 〣 〤 〥 〦 〧 〨 〩 〪 〫 〬 〭 〮 〯 〰 〱 〲 〳 〴 〵 〶 〷 〿' + tdLog.info( + "insert CJK Symbols and Punctuation %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(49) + tdSql.checkData(48, 1, data) + + # Hiragana + data = 'ぁ あ ぃ い ぅ う ぇ え ぉ お か が き ぎ く ぐ け げ こ ご さ ざ し じ す ず せ ぜ そ ぞ た だ ち ぢ っ つ づ て で と ど な に ぬ ね の は ば ぱ ひ び ぴ ふ ぶ ぷ へ べ ぺ ほ ぼ ぽ ま み む め も ゃ や ゅ ゆ ょ よ ら り る れ ろ ゎ わ ゐ ゑ を ん ゔ ゙ ゚ ゛ ゜ ゝ ゞ' + tdLog.info("insert Hiragana %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(50) + tdSql.checkData(49, 1, data) + + # Katakana + data = 'ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ ゲ コ ゴ サ ザ シ ジ ス ズ セ ゼ ソ ゾ タ ダ チ ヂ ッ ツ ヅ テ デ ト ド ナ ニ ヌ ネ ノ ハ バ パ ヒ ビ ピ フ ブ プ ヘ ベ ペ ホ ボ ポ マ ミ ム メ モ ャ ヤ ュ ユ ョ ヨ ラ リ ル レ ロ ヮ ワ ヰ ヱ ヲ ン ヴ ヵ ヶ ヷ ヸ ヹ ヺ ・ ー ヽ ヾ' + tdLog.info("insert Katakana %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(51) + tdSql.checkData(50, 1, data) + + # Bopomofo + data = 'ㄅ ㄆ ㄇ ㄈ ㄉ ㄊ ㄋ ㄌ ㄍ ㄎ ㄏ ㄐ ㄑ ㄒ ㄓ ㄔ ㄕ ㄖ ㄗ ㄘ ㄙ ㄚ ㄛ ㄜ ㄝ ㄞ ㄟ ㄠ ㄡ ㄢ ㄣ ㄤ ㄥ ㄦ ㄧ ㄨ ㄩ ㄪ ㄫ ㄬ' + tdLog.info("insert Bopomofo %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(52) + tdSql.checkData(51, 1, data) + + # Hangul Compatibility Jamo + data = 'ㄱ ㄲ ㄳ ㄴ ㄵ ㄶ ㄷ ㄸ ㄹ ㄺ ㄻ ㄼ ㄽ ㄾ ㄿ ㅀ ㅁ ㅂ ㅃ ㅄ ㅅ ㅆ ㅇ ㅈ ㅉ ㅊ ㅋ ㅌ ㅍ ㅎ ㅏ ㅐ ㅑ ㅒ ㅓ ㅔ ㅕ ㅖ ㅗ ㅘ ㅙ ㅚ ㅛ ㅜ ㅝ ㅞ ㅟ ㅠ ㅡ ㅢ ㅣ ㅤ ㅥ ㅦ ㅧ ㅨ ㅩ ㅪ ㅫ ㅬ ㅭ ㅮ ㅯ ㅰ ㅱ ㅲ ㅳ ㅴ ㅵ ㅶ ㅷ ㅸ ㅹ ㅺ ㅻ ㅼ ㅽ ㅾ ㅿ ㆀ ㆁ ㆂ ㆃ ㆄ ㆅ ㆆ ㆇ ㆈ ㆉ ㆊ ㆋ ㆌ ㆍ ㆎ' + tdLog.info( + "insert Hangul Compatibility Jamo %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(53) + tdSql.checkData(52, 1, data) + + # Kanbun + data = '㆐ ㆑ ㆒ ㆓ ㆔ ㆕ ㆖ ㆗ ㆘ ㆙ ㆚ ㆛ ㆜ ㆝ ㆞ ㆟' + tdLog.info("insert Kanbun %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(54) + tdSql.checkData(53, 1, data) + + # Enclosed CJK Letters and Months + data = '㈀ ㈁ ㈂ ㈃ ㈄ ㈅ ㈆ ㈇ ㈈ ㈉ ㈊ ㈋ ㈌ ㈍ ㈎ ㈏ ㈐ ㈑ ㈒ ㈓ ㈔ ㈕ ㈖ ㈗ ㈘ ㈙ ㈚ ㈛ ㈜ ㈠ ㈡ ㈢ ㈣ ㈤ ㈥ ㈦ ㈧ ㈨ ㈩ ㈪ ㈫ ㈬ ㈭ ㈮ ㈯ ㈰ ㈱ ㈲ ㈳ ㈴ ㈵ ㈶ ㈷ ㈸ ㈹ ㈺ ㈻ ㈼ ㈽ ㈾ ㈿ ㉀ ㉁ ㉂ ㉃ ㉠ ㉡ ㉢ ㉣ ㉤ ㉥ ㉦ ㉧ ㉨ ㉩ ㉪ ㉫ ㉬ ㉭ ㉮ ㉯ ㉰ ㉱ ㉲ ㉳ ㉴ ㉵ ㉶ ㉷ ㉸ ㉹ ㉺ ㉻ ㉿ ㊀ ㊁ ㊂ ㊃ ㊄ ㊅ ㊆ ㊇ ㊈ ㊉ ㊊ ㊋ ㊌ ㊍ ㊎ ㊏ ㊐ ㊑ ㊒ ㊓ ㊔ ㊕ ㊖ ㊗ ㊘ ㊙ ㊚ ㊛ ㊜ ㊝ ㊞ ㊟ ㊠ ㊡ ...' + tdLog.info( + "insert Enclosed CJK Letters and Months %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(55) + tdSql.checkData(54, 1, data) + + # CJK Compatibility + data = '㌀ ㌁ ㌂ ㌃ ㌄ ㌅ ㌆ ㌇ ㌈ ㌉ ㌊ ㌋ ㌌ ㌍ ㌎ ㌏ ㌐ ㌑ ㌒ ㌓ ㌔ ㌕ ㌖ ㌗ ㌘ ㌙ ㌚ ㌛ ㌜ ㌝ ㌞ ㌟ ㌠ ㌡ ㌢ ㌣ ㌤ ㌥ ㌦ ㌧ ㌨ ㌩ ㌪ ㌫ ㌬ ㌭ ㌮ ㌯ ㌰ ㌱ ㌲ ㌳ ㌴ ㌵ ㌶ ㌷ ㌸ ㌹ ㌺ ㌻ ㌼ ㌽ ㌾ ㌿ ㍀ ㍁ ㍂ ㍃ ㍄ ㍅ ㍆ ㍇ ㍈ ㍉ ㍊ ㍋ ㍌ ㍍ ㍎ ㍏ ㍐ ㍑ ㍒ ㍓ ㍔ ㍕ ㍖ ㍗ ㍘ ㍙ ㍚ ㍛ ㍜ ㍝ ㍞ ㍟ ㍠ ㍡ ㍢ ㍣ ㍤ ㍥ ㍦ ㍧ ㍨ ㍩ ㍪ ㍫ ㍬ ㍭ ㍮ ㍯ ㍰ ㍱ ㍲ ㍳ ㍴ ㍵ ㍶ ㍻ ㍼ ㍽ ㍾ ㍿ ㎀ ㎁ ㎂ ㎃ ...' + tdLog.info( + "insert CJK Compatibility %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(56) + tdSql.checkData(55, 1, data) + + # CJK Unified Ideographs + data = '一 丁 丂 七 丄 丅 丆 万 丈 三 上 下 丌 不 与 丏 丐 丑 丒 专 且 丕 世 丗 丘 丙 业 丛 东 丝 丞 丟 丠 両 丢 丣 两 严 並 丧 丨 丩 个 丫 丬 中 丮 丯 丰 丱 串 丳 临 丵 丶 丷 丸 丹 为 主 丼 丽 举 丿 乀 乁 乂 乃 乄 久 乆 乇 么 义 乊 之 乌 乍 乎 乏 乐 乑 乒 乓 乔 乕 乖 乗 乘 乙 乚 乛 乜 九 乞 也 习 乡 乢 乣 乤 乥 书 乧 乨 乩 乪 乫 乬 乭 乮 乯 买 乱 乲 乳 乴 乵 乶 乷 乸 乹 乺 乻 乼 乽 乾 乿 ...' + tdLog.info( + "insert CJK Unified Ideographs %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(57) + tdSql.checkData(56, 1, data) + + # Hangul Syllables + data = '一 丁 丂 七 丄 丅 丆 万 丈 三 上 下 丌 不 与 丏 丐 丑 丒 专 且 丕 世 丗 丘 丙 业 丛 东 丝 丞 丟 丠 両 丢 丣 两 严 並 丧 丨 丩 个 丫 丬 中 丮 丯 丰 丱 串 丳 临 丵 丶 丷 丸 丹 为 主 丼 丽 举 丿 乀 乁 乂 乃 乄 久 乆 乇 么 义 乊 之 乌 乍 乎 乏 乐 乑 乒 乓 乔 乕 乖 乗 乘 乙 乚 乛 乜 九 乞 也 习 乡 乢 乣 乤 乥 书 乧 乨 乩 乪 乫 乬 乭 乮 乯 买 乱 乲 乳 乴 乵 乶 乷 乸 乹 乺 乻 乼 乽 乾 乿 ...' + tdLog.info( + "insert Hangul Syllables %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(58) + tdSql.checkData(57, 1, data) + + # Private Use + data = '                                                                                                                                ...' + tdLog.info("insert Private Use %d length data: %s" % (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(59) + tdSql.checkData(58, 1, data) + + # CJK Compatibility Ideographs + data = '豈 更 車 賈 滑 串 句 龜 龜 契 金 喇 奈 懶 癩 羅 蘿 螺 裸 邏 樂 洛 烙 珞 落 酪 駱 亂 卵 欄 爛 蘭 鸞 嵐 濫 藍 襤 拉 臘 蠟 廊 朗 浪 狼 郎 來 冷 勞 擄 櫓 爐 盧 老 蘆 虜 路 露 魯 鷺 碌 祿 綠 菉 錄 鹿 論 壟 弄 籠 聾 牢 磊 賂 雷 壘 屢 樓 淚 漏 累 縷 陋 勒 肋 凜 凌 稜 綾 菱 陵 讀 拏 樂 諾 丹 寧 怒 率 異 北 磻 便 復 不 泌 數 索 參 塞 省 葉 說 殺 辰 沈 拾 若 掠 略 亮 兩 凉 梁 糧 良 諒 量 勵 ...' + tdLog.info( + "insert CJK Compatibility Ideographs %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(60) + tdSql.checkData(59, 1, data) + + # Alphabetic Presentation Forms + data = 'ff fi fl ffi ffl ſt st ﬓ ﬔ ﬕ ﬖ ﬗ ﬞ ײַ ﬠ ﬡ ﬢ ﬣ ﬤ ﬥ ﬦ ﬧ ﬨ ﬩ שׁ שׂ שּׁ שּׂ אַ אָ אּ בּ גּ דּ הּ וּ זּ טּ יּ ךּ כּ לּ מּ נּ סּ ףּ פּ צּ קּ רּ שּ תּ וֹ בֿ כֿ פֿ ﭏ' + tdLog.info( + "insert Alphabetic Presentation Forms %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(61) + tdSql.checkData(60, 1, data) + + # Arabic Presentation Forms-A + data = 'ﭐ ﭑ ﭒ ﭓ ﭔ ﭕ ﭖ ﭗ ﭘ ﭙ ﭚ ﭛ ﭜ ﭝ ﭞ ﭟ ﭠ ﭡ ﭢ ﭣ ﭤ ﭥ ﭦ ﭧ ﭨ ﭩ ﭪ ﭫ ﭬ ﭭ ﭮ ﭯ ﭰ ﭱ ﭲ ﭳ ﭴ ﭵ ﭶ ﭷ ﭸ ﭹ ﭺ ﭻ ﭼ ﭽ ﭾ ﭿ ﮀ ﮁ ﮂ ﮃ ﮄ ﮅ ﮆ ﮇ ﮈ ﮉ ﮊ ﮋ ﮌ ﮍ ﮎ ﮏ ﮐ ﮑ ﮒ ﮓ ﮔ ﮕ ﮖ ﮗ ﮘ ﮙ ﮚ ﮛ ﮜ ﮝ ﮞ ﮟ ﮠ ﮡ ﮢ ﮣ ﮤ ﮥ ﮦ ﮧ ﮨ ﮩ ﮪ ﮫ ﮬ ﮭ ﮮ ﮯ ﮰ ﮱ ﯓ ﯔ ﯕ ﯖ ﯗ ﯘ ﯙ ﯚ ﯛ ﯜ ﯝ ﯞ ﯟ ﯠ ﯡ ﯢ ﯣ ﯤ ﯥ ﯦ ﯧ ﯨ ﯩ ﯪ ﯫ ﯬ ﯭ ﯮ ﯯ ﯰ ...' + tdLog.info( + "insert Arabic Presentation Forms-A %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(62) + tdSql.checkData(61, 1, data) + + # Combining Half Marks + data = '︠ ︡ ︢ ︣' + tdLog.info( + "insert Combining Half Marks %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(63) + tdSql.checkData(62, 1, data) + + # CJK Compatibility Forms + data = '︰ ︱ ︲ ︳ ︴ ︵ ︶ ︷ ︸ ︹ ︺ ︻ ︼ ︽ ︾ ︿ ﹀ ﹁ ﹂ ﹃ ﹄ ﹉ ﹊ ﹋ ﹌ ﹍ ﹎ ﹏' + tdLog.info( + "insert CJK Compatibility Forms %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(64) + tdSql.checkData(63, 1, data) + + # Small Form Variants + data = '﹐ ﹑ ﹒ ﹔ ﹕ ﹖ ﹗ ﹘ ﹙ ﹚ ﹛ ﹜ ﹝ ﹞ ﹟ ﹠ ﹡ ﹢ ﹣ ﹤ ﹥ ﹦ ﹨ ﹩ ﹪ ﹫' + tdLog.info( + "insert Small Form Variants %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(65) + tdSql.checkData(64, 1, data) + + # Arabic Presentation Forms-B + data = 'ﹰ ﹱ ﹲ ﹴ ﹶ ﹷ ﹸ ﹹ ﹺ ﹻ ﹼ ﹽ ﹾ ﹿ ﺀ ﺁ ﺂ ﺃ ﺄ ﺅ ﺆ ﺇ ﺈ ﺉ ﺊ ﺋ ﺌ ﺍ ﺎ ﺏ ﺐ ﺑ ﺒ ﺓ ﺔ ﺕ ﺖ ﺗ ﺘ ﺙ ﺚ ﺛ ﺜ ﺝ ﺞ ﺟ ﺠ ﺡ ﺢ ﺣ ﺤ ﺥ ﺦ ﺧ ﺨ ﺩ ﺪ ﺫ ﺬ ﺭ ﺮ ﺯ ﺰ ﺱ ﺲ ﺳ ﺴ ﺵ ﺶ ﺷ ﺸ ﺹ ﺺ ﺻ ﺼ ﺽ ﺾ ﺿ ﻀ ﻁ ﻂ ﻃ ﻄ ﻅ ﻆ ﻇ ﻈ ﻉ ﻊ ﻋ ﻌ ﻍ ﻎ ﻏ ﻐ ﻑ ﻒ ﻓ ﻔ ﻕ ﻖ ﻗ ﻘ ﻙ ﻚ ﻛ ﻜ ﻝ ﻞ ﻟ ﻠ ﻡ ﻢ ﻣ ﻤ ﻥ ﻦ ﻧ ﻨ ﻩ ﻪ ﻫ ﻬ ﻭ ﻮ ﻯ ﻰ ﻱ ...' + tdLog.info( + "insert Arabic Presentation Forms-B %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(66) + tdSql.checkData(65, 1, data) + + # Halfwidth and Fullwidth Forms + data = '！ ＂ ＃ ＄ ％ ＆ ＇ （ ） ＊ ＋ ， － ． ／ ０ １ ２ ３ ４ ５ ６ ７ ８ ９ ： ； ＜ ＝ ＞ ？ ＠ Ａ Ｂ Ｃ Ｄ Ｅ Ｆ Ｇ Ｈ Ｉ Ｊ Ｋ Ｌ Ｍ Ｎ Ｏ Ｐ Ｑ Ｒ Ｓ Ｔ Ｕ Ｖ Ｗ Ｘ Ｙ Ｚ ［ ＼ ］ ＾ ＿ ｀ ａ ｂ ｃ ｄ ｅ ｆ ｇ ｈ ｉ ｊ ｋ ｌ ｍ ｎ ｏ ｐ ｑ ｒ ｓ ｔ ｕ ｖ ｗ ｘ ｙ ｚ ｛ ｜ ｝ ～ ｡ ｢ ｣ ､ ･ ｦ ｧ ｨ ｩ ｪ ｫ ｬ ｭ ｮ ｯ ｰ ｱ ｲ ｳ ｴ ｵ ｶ ｷ ｸ ｹ ｺ ｻ ｼ ｽ ｾ ｿ ﾀ ﾁ ﾂ ...' + tdLog.info( + "insert Halfwidth and Fullwidth Forms %d length data: %s" % + (len(data), data)) + tdSql.execute("insert into tb values (now, '%s')" % data) + tdSql.query("select * from tb") + tdSql.checkRows(67) + tdSql.checkData(66, 1, data) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py new file mode 100644 index 0000000000..68b8baab64 --- /dev/null +++ b/tests/pytest/query/query.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + + print("==============step2") + + tdSql.execute( + """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_002 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index 853ebe1d76..6314b3f83b 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -21,6 +21,8 @@ python3 ./test.py $1 -f insert/date.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f insert/nchar.py python3 ./test.py $1 -s && sleep 1 +python3 ./test.py $1 -f insert/multi.py +python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f table/column_name.py python3 ./test.py $1 -s && sleep 1 @@ -29,8 +31,6 @@ python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f table/db_table.py python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importCacheFileT.py -python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importDataLastSub.py python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f import_merge/importHead.py diff --git a/tests/pytest/stable/__init__.py b/tests/pytest/stable/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py new file mode 100644 index 0000000000..9f9e7c6e06 --- /dev/null +++ b/tests/pytest/stable/insert.py @@ -0,0 +1,56 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + + print("==============step2") + + tdSql.execute( + """INSERT INTO dev_001(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_002 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.st where dev='dev_01'") + tdSql.checkRows(2) + + tdSql.query("select * from db.st where dev='dev_02'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/__init__.py b/tests/pytest/table/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/table/alter_column.py b/tests/pytest/table/alter_column.py new file mode 100644 index 0000000000..15bc9f7aa7 --- /dev/null +++ b/tests/pytest/table/alter_column.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + + tdLog.info("create database and table") + tdSql.execute("create database db_test") + tdSql.execute( + "create table if not exists db_test.st(ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + "CREATE TABLE if not exists db_test.dev_001 using db_test.st tags('dev_01')") + + print("==============step2") + tdLog.info("alter table add column") + tdSql.execute( + "ALTER TABLE db_test.st add COLUMN tag_version nchar(20)") + tdSql.query("describe db_test.st") + tdSql.checkRows(4) + + print("==============step3") + tdLog.info("alter table drop column") + tdSql.execute("ALTER TABLE db_test.st drop COLUMN tag_version") + tdSql.query("describe db_test.st") + tdSql.checkRows(3) + + print("==============step4") + tdLog.info("drop table") + tdSql.execute("drop table db_test.st") + tdSql.execute( + "create table if not exists db_test.st(ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + "CREATE TABLE if not exists db_test.dev_001 using db_test.st tags('dev_01')") + tdSql.execute( + "INSERT INTO db_test.dev_001 VALUES ('2020-05-13 10:00:00.000', 1)") + tdSql.query("select * from db_test.dev_001") + tdSql.checkRows(1) + + print("==============step5") + tdLog.info("alter table add column") + tdSql.execute( + "ALTER TABLE db_test.st add COLUMN tag_version nchar(20)") + tdSql.query("describe db_test.st") + tdSql.checkRows(4) + + tdSql.execute( + "INSERT INTO db_test.dev_001 VALUES ('2020-05-13 10:00:00.010', 1, '1.2.1')") + tdSql.query("select * from db_test.st") + tdSql.checkRows(2) + + print("==============step6") + tdLog.info("alter table drop column") + tdSql.execute("ALTER TABLE db_test.st drop COLUMN tag_version") + tdSql.query("describe db_test.st") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/boundary.py b/tests/pytest/table/boundary.py new file mode 100644 index 0000000000..faa222231b --- /dev/null +++ b/tests/pytest/table/boundary.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +import random +import string +import subprocess +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init( self, conn ): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + + def getLimitFromSourceCode( self, name ): + cmd = "grep -w '#define %s' ../../src/inc/taosdef.h|awk '{print $3}'" % name + return int(subprocess.check_output(cmd, shell=True)) + + + def generateString( self, length ): + chars = string.ascii_uppercase + string.ascii_lowercase + v = "" + for i in range( length ): + v += random.choice( chars ) + return v + + + def checkTagBoundaries( self ): + tdLog.debug( "checking tag boundaries" ) + tdSql.prepare() + + maxTags = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS' ) + totalTagsLen = self.getLimitFromSourceCode( 'TSDB_MAX_TAGS_LEN' ) + tdLog.notice( "max tags is %d" % maxTags ) + tdLog.notice( "max total tag length is %d" % totalTagsLen ) + + # for binary tags, 2 bytes are used for length + tagLen = (totalTagsLen - maxTags * 2) // maxTags + firstTagLen = totalTagsLen - 2 * maxTags - tagLen * (maxTags - 1) + + sql = "create table cars(ts timestamp, f int) tags(t0 binary(%d)" % firstTagLen + for i in range( 1, maxTags ): + sql += ", t%d binary(%d)" % (i, tagLen) + sql += ");" + + tdLog.debug( "creating super table: " + sql ) + tdSql.execute( sql ) + tdSql.query( 'show stables' ) + tdSql.checkRows( 1 ) + + for i in range( 10 ): + sql = "create table car%d using cars tags('%d'" % (i, i) + sql += ", '0'" * (maxTags - 1) + ");" + tdLog.debug( "creating table: " + sql ) + tdSql.execute( sql ) + + sql = "insert into car%d values(now, 0);" % i + tdLog.debug( "inserting data: " + sql ) + tdSql.execute( sql ) + + tdSql.query( 'show tables' ) + tdLog.info( 'tdSql.checkRow(10)' ) + tdSql.checkRows( 10 ) + + tdSql.query( 'select * from cars;' ) + tdSql.checkRows( 10 ) + + + def checkColumnBoundaries( self ): + tdLog.debug( "checking column boundaries" ) + tdSql.prepare() + + # one column is for timestamp + maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1 + + sql = "create table cars (ts timestamp" + for i in range( maxCols ): + sql += ", c%d int" % i + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + sql = "insert into cars values (now" + for i in range( maxCols ): + sql += ", %d" % i + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'select * from cars' ) + tdSql.checkRows( 1 ) + + + def checkTableNameBoundaries( self ): + tdLog.debug( "checking table name boundaries" ) + tdSql.prepare() + + maxTableNameLen = self.getLimitFromSourceCode( 'TSDB_TABLE_NAME_LEN' ) + tdLog.notice( "table name max length is %d" % maxTableNameLen ) + + name = self.generateString( maxTableNameLen - 1) + tdLog.info( "table name is '%s'" % name ) + + tdSql.execute( "create table %s (ts timestamp, value int)" % name ) + tdSql.execute( "insert into %s values(now, 0)" % name ) + + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + tdSql.query( 'select * from %s' % name ) + tdSql.checkRows( 1 ) + + + def checkRowBoundaries( self ): + tdLog.debug( "checking row boundaries" ) + tdSql.prepare() + + # 8 bytes for timestamp + maxRowSize = 65536 - 8 + maxCols = self.getLimitFromSourceCode( 'TSDB_MAX_COLUMNS' ) - 1 + + # for binary cols, 2 bytes are used for length + colLen = (maxRowSize - maxCols * 2) // maxCols + firstColLen = maxRowSize - 2 * maxCols - colLen * (maxCols - 1) + + sql = "create table cars (ts timestamp, c0 binary(%d)" % firstColLen + for i in range( 1, maxCols ): + sql += ", c%d binary(%d)" % (i, colLen) + sql += ");" + tdSql.execute( sql ) + tdSql.query( 'show tables' ) + tdSql.checkRows( 1 ) + + col = self.generateString( firstColLen ) + sql = "insert into cars values (now, '%s'" % col + col = self.generateString( colLen ) + for i in range( 1, maxCols ): + sql += ", '%s'" % col + sql += ");" + tdLog.info( sql ); + tdSql.execute( sql ) + tdSql.query( "select * from cars" ) + tdSql.checkRows( 1 ) + + + def run(self): + self.checkTagBoundaries() + self.checkColumnBoundaries() + self.checkTableNameBoundaries() + self.checkRowBoundaries() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/column_name.py b/tests/pytest/table/column_name.py index bb1f587a65..aa958fd60c 100644 --- a/tests/pytest/table/column_name.py +++ b/tests/pytest/table/column_name.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- import sys +import string +import random +import subprocess from util.log import * from util.cases import * from util.sql import * @@ -14,34 +17,9 @@ class TDTestCase: def run(self): tdSql.prepare() - # TSIM: system sh/stop_dnodes.sh - # TSIM: - # TSIM: system sh/ip.sh -i 1 -s up - # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 - # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 - # TSIM: system sh/exec.sh -n dnode1 -s start - # TSIM: - # TSIM: sleep 3000 - # TSIM: sql connect - # TSIM: - # TSIM: $i = 0 - # TSIM: $dbPrefix = lm_cm_db - # TSIM: $tbPrefix = lm_cm_tb - # TSIM: $db = $dbPrefix . $i - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: print =============== step1 tdLog.info('=============== step1') - # TSIM: sql create database $db - # TSIM: sql use $db - # TSIM: - # TSIM: sql drop table dd -x step0 tdLog.info('drop table dd -x step0') tdSql.error('drop table dd') - # TSIM: return -1 - # TSIM: step0: - # TSIM: - # TSIM: sql create table $tb(ts timestamp, int) -x step1 tdLog.info('create table tb(ts timestamp, int) -x step1') tdSql.error('create table tb(ts timestamp, int)') # TSIM: return -1 @@ -112,37 +90,24 @@ class TDTestCase: tdLog.info('=============== step4') # TSIM: sql create table $tb (ts timestamp, # a0123456789012345678901234567890123456789 int) + getMaxColNum = "grep -w '#define TSDB_COL_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" + boundary = int(subprocess.check_output(getMaxColNum, shell=True)) + tdLog.info("get max column name length is %d" % boundary) + chars = string.ascii_uppercase + string.ascii_lowercase + +# col_name = ''.join(random.choices(chars, k=boundary+1)) +# tdLog.info( +# 'create table tb (ts timestamp, %s int), col_name length is %d' % (col_name, len(col_name))) +# tdSql.error( +# 'create table tb (ts timestamp, %s int)' % col_name) + + col_name = ''.join(random.choices(chars, k=boundary)) tdLog.info( - 'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)') + 'create table tb (ts timestamp, %s int), col_name length is %d' % + (col_name, len(col_name))) tdSql.execute( - 'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)') - # TSIM: sql drop table $tb - tdLog.info('drop table tb') - tdSql.execute('drop table tb') - # TSIM: - # TSIM: sql show tables - tdLog.info('show tables') - tdSql.query('show tables') - # TSIM: if $rows != 0 then - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step5 - tdLog.info('=============== step5') - # TSIM: sql create table $tb (ts timestamp, a0123456789 int) - tdLog.info('create table tb (ts timestamp, a0123456789 int)') - tdSql.execute('create table tb (ts timestamp, a0123456789 int)') - # TSIM: sql show tables - tdLog.info('show tables') - tdSql.query('show tables') - # TSIM: if $rows != 1 then - tdLog.info('tdSql.checkRow(1)') - tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: + 'create table tb (ts timestamp, %s int)' % col_name) + # TSIM: sql insert into $tb values (now , 1) tdLog.info("insert into tb values (now , 1)") tdSql.execute("insert into tb values (now , 1)") @@ -152,24 +117,6 @@ class TDTestCase: # TSIM: if $rows != 1 then tdLog.info('tdSql.checkRow(1)') tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: sql drop database $db - tdLog.info('drop database db') - tdSql.execute('drop database db') - # TSIM: sql show databases - tdLog.info('show databases') - tdSql.query('show databases') - # TSIM: if $rows != 0 then - tdLog.info('tdSql.checkRow(0)') - tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: - # TSIM: - # TSIM: # convert end def stop(self): diff --git a/tests/pytest/table/column_num.py b/tests/pytest/table/column_num.py index 87299827cc..877f0409dc 100644 --- a/tests/pytest/table/column_num.py +++ b/tests/pytest/table/column_num.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import sys +import subprocess from util.log import * from util.cases import * from util.sql import * @@ -14,127 +15,71 @@ class TDTestCase: def run(self): tdSql.prepare() - # TSIM: system sh/stop_dnodes.sh - # TSIM: - # TSIM: system sh/ip.sh -i 1 -s up - # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 - # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 - # TSIM: system sh/exec.sh -n dnode1 -s start - # TSIM: - # TSIM: sleep 3000 - # TSIM: sql connect - # TSIM: - # TSIM: $i = 0 - # TSIM: $dbPrefix = lm_cn_db - # TSIM: $tbPrefix = lm_cn_tb - # TSIM: $db = $dbPrefix . $i - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: print =============== step1 tdLog.info('=============== step1') - # TSIM: sql create database $db - # TSIM: sql use $db - # TSIM: - # TSIM: sql create table $tb() -x step1 tdLog.info('create table tb() -x step1') tdSql.error('create table tb()') - # TSIM: return -1 - # TSIM: step1: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step2 tdLog.info('=============== step2') # TSIM: sql create table $tb (ts timestamp) -x step2 tdLog.info('create table tb (ts timestamp) -x step2') tdSql.error('create table tb (ts timestamp) ') - # TSIM: return -1 - # TSIM: step2: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step3 tdLog.info('=============== step3') - # TSIM: sql create table $tb (ts int) -x step3 tdLog.info('create table tb (ts int) -x step3') tdSql.error('create table tb (ts int) ') - # TSIM: return -1 - # TSIM: step3: - # TSIM: - # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') - # TSIM: if $rows != 0 then tdLog.info('tdSql.checkRow(0)') tdSql.checkRows(0) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step4 tdLog.info('=============== step4') - # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4 - # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 - # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, - # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 - # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, - # b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 - # int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, - # b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 - # int, b28 int,b29 int,b30 int,b31 int,b32 int) tdLog.info('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)') tdSql.execute('create table tb (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int, b6 int, b7 int, b8 int, b9 int, b10 int, b11 int, b12 int, b13 int, b14 int, b15 int, b16 int, b17 int, b18 int, b19 int, b20 int, b21 int, b22 int, b23 int, b24 int, b25 int, b26 int, b27 int, b28 int,b29 int,b30 int,b31 int,b32 int)') # TSIM: # TSIM: sql show tables tdLog.info('show tables') tdSql.query('show tables') - # TSIM: if $rows != 1 then tdLog.info('tdSql.checkRow(1)') tdSql.checkRows(1) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: print =============== step5 + tdLog.info('=============== step5') - # TSIM: $i = 1 - # TSIM: $tb = $tbPrefix . $i - # TSIM: - # TSIM: sql create table $tb (ts timestamp, a1 int, a2 int, a3 int, a4 - # int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 - # int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, - # a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 - # int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, - # b4 int, b5 int) - tdLog.info('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)') - tdSql.execute('create table tb1 (ts timestamp, a1 int, a2 int, a3 int, a4 int, a5 int, a6 int, a7 int, a8 int, a9 int, a10 int, a11 int, a12 int, a13 int, a14 int, a15 int, a16 int, a17 int, a18 int, a19 int, a20 int, a21 int, a22 int, a23 int, a24 int, a25 int, a26 int, a27 int, a28 int,a29 int,a30 int,a31 int,a32 int, b1 int, b2 int, b3 int, b4 int, b5 int)') - # TSIM: - # TSIM: sql show tables + + getMaxColumnNum = "grep -w '#define TSDB_MAX_COLUMNS' ../../src/inc/taosdef.h|awk '{print $3}'" + boundary = int(subprocess.check_output(getMaxColumnNum, shell=True)) + tdLog.info("get max column number is %d" % boundary) + + columnSeq = "ts timestamp" + for x in range(0, boundary): + columnSeq = columnSeq + ", col%d int" % x + + tdLog.info("create table tb1 (%s)" % columnSeq) + tdSql.error('create table tb1 (%s)' % columnSeq) + + columnSeq = "ts timestamp" + for x in range(0, boundary - 1): + columnSeq = columnSeq + ", col%d int" % x + + tdLog.info("create table tb1 (%s)" % columnSeq) + tdSql.execute('create table tb1 (%s)' % columnSeq) + tdLog.info('show tables') tdSql.query('show tables') # TSIM: if $rows != 2 then tdLog.info('tdSql.checkRow(2)') tdSql.checkRows(2) - # TSIM: return -1 - # TSIM: endi - # TSIM: - # TSIM: sql insert into $tb values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 - # , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 - # , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37) - tdLog.info("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)") - tdSql.execute("insert into tb1 values (now, 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 ,26 , 27 ,28 ,29,30,31, 32, 33, 34, 35, 36, 37)") + + data = "now" + for x in range(0, boundary - 1): + data = data + ", %d" % x + tdLog.info("insert into tb1 values (%s)" % data) + tdSql.execute("insert into tb1 values (%s)" % data) # TSIM: sql select * from $tb tdLog.info('select * from tb1') tdSql.query('select * from tb1') diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py new file mode 100644 index 0000000000..b456b444f4 --- /dev/null +++ b/tests/pytest/table/create.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + print("prepare data") + tdSql.execute("create table db.st (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.tb using st tags(1)") + tdSql.execute("insert into db.tb values(now, 1)") + + print("==============step2") + print("create table as select") + try: + tdSql.execute("create table db.test as select * from db.st") + except Exception as e: + tdLog.exit(e) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/del_stable.py b/tests/pytest/table/del_stable.py new file mode 100644 index 0000000000..3932f32536 --- /dev/null +++ b/tests/pytest/table/del_stable.py @@ -0,0 +1,58 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table db.st (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.tb using st tags(1)") + tdSql.execute("insert into db.tb values(now, 1)") + + print("==============step2") + try: + tdSql.execute("drop table db.st") + except Exception as e: + tdLog.exit(e) + + try: + tdSql.execute("select * from db.st") + except Exception as e: + if e.args[0] != 'invalid table name': + tdLog.exit(e) + + try: + tdSql.execute("select * from db.tb") + except Exception as e: + if e.args[0] != 'invalid table name': + tdLog.exit(e) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/tablename-boundary.py b/tests/pytest/table/tablename-boundary.py new file mode 100644 index 0000000000..335073065c --- /dev/null +++ b/tests/pytest/table/tablename-boundary.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import sys +import string +import random +import subprocess +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + getTableNameLen = "grep -w '#define TSDB_TABLE_NAME_LEN' ../../src/inc/taosdef.h|awk '{print $3}'" + tableNameMaxLen = int( + subprocess.check_output( + getTableNameLen, shell=True)) + tdLog.info("table name max length is %d" % tableNameMaxLen) + chars = string.ascii_uppercase + string.ascii_lowercase + tb_name = ''.join(random.choices(chars, k=tableNameMaxLen)) + tdLog.info('tb_name length %d' % len(tb_name)) + tdLog.info('create table %s (ts timestamp, value int)' % tb_name) + tdSql.error( + 'create table %s (ts timestamp, speed binary(4089))' % + tb_name) + + tb_name = ''.join(random.choices(chars, k=191)) + tdLog.info('tb_name length %d' % len(tb_name)) + tdLog.info('create table %s (ts timestamp, value int)' % tb_name) + tdSql.execute( + 'create table %s (ts timestamp, speed binary(4089))' % + tb_name) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/create-tags-boundary.py b/tests/pytest/tag_lite/create-tags-boundary.py new file mode 100644 index 0000000000..e80f458f0c --- /dev/null +++ b/tests/pytest/tag_lite/create-tags-boundary.py @@ -0,0 +1,66 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import subprocess +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + getMaxTagNum = "grep -w TSDB_MAX_TAGS ../../src/inc/taosdef.h|awk '{print $3}'" + boundary = int(subprocess.check_output(getMaxTagNum, shell=True)) + tdLog.info("get max tags number is %d" % boundary) + for x in range(0, boundary): + stb_name = "stb%d" % x + + tagSeq = "tag0 int" + for y in range(1, x + 1): + tagSeq = tagSeq + ", tag%d int" % y + + tdLog.info( + "create table %s (ts timestamp, value int) tags (%s)" % + (stb_name, tagSeq)) + tdSql.execute( + "create table %s (ts timestamp, value int) tags (%s)" % + (stb_name, tagSeq)) + + tdSql.query("show stables") + tdSql.checkRows(boundary) + + stb_name = "stb%d" % (boundary + 1) + tagSeq = tagSeq + ", tag%d int" % (boundary) + tdLog.info( + "create table %s (ts timestamp, value int) tags (%s)" % + (stb_name, tagSeq)) + tdSql.error( + "create table %s (ts timestamp, value int) tags (%s)" % + (stb_name, tagSeq)) + tdSql.query("show stables") + tdSql.checkRows(boundary) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/datatype-without-alter.py b/tests/pytest/tag_lite/datatype-without-alter.py new file mode 100644 index 0000000000..1a8d05d648 --- /dev/null +++ b/tests/pytest/tag_lite/datatype-without-alter.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + self.ntables = 10 + self.rowsPerTable = 10 + self.startTime = 1520000010000 + + tdDnodes.stop(1) + tdDnodes.deploy(1) + tdDnodes.start(1) + + tdLog.info("================= step0") + tdSql.execute('reset query cache') + tdLog.info("drop database db if exits") + tdSql.execute('drop database if exists db') + tdLog.info("================= step1") + tdSql.execute('create database db maxtables 4') + tdLog.sleep(5) + tdSql.execute('use db') + + tdLog.info("================= step1") + tdLog.info("create 1 super table") + tdSql.execute('create table stb (ts timestamp, i int) \ + tags (tin int, tfl float, tbg bigint, tdo double, tbi binary(10), tbl bool)') + + tdLog.info("================= step2") + tdLog.info("create %d tables" % self.ntables) + for tid in range(1, self.ntables + 1): + tdSql.execute( + 'create table tb%d using stb tags(%d,%f,%ld,%f,\'%s\',%d)' % + (tid, + tid % + 3, + 1.2 * + tid, + self.startTime + + tid, + 1.22 * + tid, + 't' + + str(tid), + tid % + 2)) + tdLog.sleep(5) + + tdLog.info("================= step3") + tdLog.info( + "insert %d data in to each %d tables" % + (self.rowsPerTable, self.ntables)) + for rid in range(1, self.rowsPerTable + 1): + sqlcmd = ['insert into'] + for tid in range(1, self.ntables + 1): + sqlcmd.append( + 'tb%d values(%ld,%d)' % + (tid, self.startTime + rid, rid)) + tdSql.execute(" ".join(sqlcmd)) + tdSql.query('select count(*) from stb') + tdSql.checkData(0, 0, self.rowsPerTable * self.ntables) + + tdLog.info("================= step6") + tdLog.info("group and filter by tag1 int") + tdSql.query('select max(i) from stb where tbl=0 group by tin') + tdSql.checkRows(3) + tdSql.execute('reset query cache') + tdSql.query('select max(i) from stb where tbl=true group by tin') + tdSql.checkData(2, 0, self.rowsPerTable) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a4129be34c..9d76b0a70e 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -70,8 +70,6 @@ if __name__ == "__main__": toBeKilled = "valgrind.bin" killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP " % toBeKilled -# os.system(killCmd) -# time.sleep(1) psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) @@ -81,7 +79,8 @@ if __name__ == "__main__": time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) - tdLog.exit('stop All dnodes') + tdLog.info('stop All dnodes') + sys.exit(0) tdDnodes.init(deployPath) tdDnodes.setTestCluster(testCluster) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 9a4f0fc98a..f65b0dfde3 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -57,7 +57,7 @@ class TDCases: runNum += 1 continue - tdLog.notice("total %d Linux test case(s) executed" % (runNum)) + tdLog.info("total %d Linux test case(s) executed" % (runNum)) def runOneLinux(self, conn, fileName): testModule = self.__dynamicLoadModule(fileName) @@ -71,13 +71,11 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) - tdLog.exit("%s failed: %s" % (__file__, fileName)) + tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 continue - tdLog.success("total %d Linux test case(s) executed" % (runNum)) - def runAllWindows(self, conn): # TODO: load all Windows cases here runNum = 0 diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 23adab2c47..727016adb3 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -38,9 +38,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path,) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = "%s/pysim/psim/log" % (self.path,) + self.cfgDir = "%s/pysim/psim/cfg" % (self.path) + self.cfgPath = "%s/pysim/psim/cfg/taos.cfg" % (self.path) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -100,10 +100,11 @@ class TDDnode: self.valgrind = value def deploy(self): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (self.path, self.index) + self.logDir = "%s/pysim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/pysim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/pysim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/pysim/dnode%d/cfg/taos.cfg" % ( + self.path, self.index) cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -177,21 +178,42 @@ class TDDnode: (self.index, self.cfgPath)) def start(self): - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - binPath = os.path.realpath(binPath) - binPath += "/build/bin/" + selfPath = os.path.dirname(os.path.realpath(__file__)) + binPath = "" + + if ("TDinternal" in selfPath): + projPath = selfPath + "/../../../../" + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("community" not in rootRealPath): + binPath = os.path.join(root, "taosd") + break + else: + projPath = selfPath + "/../../../" + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + binPath = os.path.join(root, "taosd") + break + + if (binPath == ""): + tdLog.exit("taosd not found!s") + else: + tdLog.info("taosd found in %s" % rootRealPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) if self.valgrind == 0: - cmd = "nohup %staosd -c %s > /dev/null 2>&1 & " % ( + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - cmd = "nohup %s %staosd -c %s 2>&1 & " % ( + cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) print(cmd) @@ -201,8 +223,8 @@ class TDDnode: self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - tdLog.debug("wait 4 seconds for the dnode:%d to start." % (self.index)) - time.sleep(4) + tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) + time.sleep(5) def stop(self): if self.valgrind == 0: @@ -211,22 +233,17 @@ class TDDnode: toBeKilled = "valgrind.bin" if self.running != 0: - killCmd = "ps -ef|grep -w %s| grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -INT" % ( - toBeKilled, self.cfgDir) - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -INT %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - tdLog.debug( - "wait 2 seconds for the dnode:%d to stop." % - (self.index)) - time.sleep(2) def forcestop(self): if self.valgrind == 0: @@ -235,21 +252,17 @@ class TDDnode: toBeKilled = "valgrind.bin" if self.running != 0: - killCmd = "ps -ef|grep -w %s| grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -KILL" % ( - toBeKilled, self.cfgDir) psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -KILL %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + self.running = 0 tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - tdLog.debug( - "wait 2 seconds for the dnode:%d to stop." % - (self.index)) - time.sleep(2) def startIP(self): cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) @@ -268,11 +281,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = "%s/pysim/psim/dnode%d" % (self.path, index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = "%s/pysim/psim" % (self.path) return dnodesRootDir @@ -291,21 +304,21 @@ class TDDnodes: self.dnodes.append(TDDnode(10)) def init(self, path): - killCmd = "ps -ef|grep -w taosd | grep -v grep | awk '{print $2}' | xargs kill -KILL" psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -KILL %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - killCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs kill -KILL" psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -KILL %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") binPath = os.path.dirname(os.path.realpath(__file__)) binPath = binPath + "/../../../debug/" @@ -386,38 +399,38 @@ class TDDnodes: tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) def stopAll(self): - tdLog.debug("stop all dnodes") + tdLog.info("stop all dnodes") for i in range(len(self.dnodes)): self.dnodes[i].stop() psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") if processID: cmd = "sudo systemctl stop taosd" os.system(cmd) # if os.system(cmd) != 0 : # tdLog.exit(cmd) - killCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}' | xargs kill -KILL" psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -KILL %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - killCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs kill -KILL" psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): + killCmd = "kill -KILL %s" % processID os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") # if os.system(cmd) != 0 : # tdLog.exit(cmd) def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) + dnodesRootDir = "%s/pysim" % (self.path) return dnodesRootDir def getSimCfgPath(self): diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 0e7e186206..75f3bd9044 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -15,6 +15,7 @@ import sys import os import time import datetime +import inspect from util.log import * @@ -44,7 +45,12 @@ class TDSql: except BaseException: expectErrNotOccured = False if expectErrNotOccured: - tdLog.exit("failed: sql:%.40s, expect error not occured" % (sql)) + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + tdLog.exit( + "%s failed: sql:%.40s, expect error not occured" % + (callerFilename, sql)) else: tdLog.info("sql:%.40s, expect error occured" % (sql)) @@ -62,33 +68,39 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ tdLog.exit( - "failed: sql:%.40s, queryRows:%d != expect:%d" % - (self.sql, self.queryRows, expectRows)) + "%s failed: sql:%.40s, queryRows:%d != expect:%d" % + (callerFilename, self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) def checkData(self, row, col, data): + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + if row < 0: tdLog.exit( - "failed: sql:%.40s, row:%d is smaller than zero" % - (self.sql, row)) + "%s failed: sql:%.40s, row:%d is smaller than zero" % + (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "failed: sql:%.40s, col:%d is smaller than zero" % - (self.sql, col)) + "%s failed: sql:%.40s, col:%d is smaller than zero" % + (callerFilename, self.sql, col)) if row >= self.queryRows: tdLog.exit( - "failed: sql:%.40s, row:%d is larger than queryRows:%d" % - (self.sql, row, self.queryRows)) + "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + (callerFilename, self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "failed: sql:%.40s, col:%d is larger than queryRows:%d" % - (self.sql, col, self.queryCols)) + "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + (callerFilename, self.sql, col, self.queryCols)) if self.queryResult[row][col] != data: - tdLog.exit( - "failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % - (self.sql, row, col, self.queryResult[row][col], data)) + tdLog.exit("%s failed: sql:%.40s row:%d col:%d data:%s != expect:%s" % ( + callerFilename, self.sql, row, col, self.queryResult[row][col], data)) if data is None: tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % @@ -104,22 +116,26 @@ class TDSql: (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + if row < 0: tdLog.exit( - "failed: sql:%.40s, row:%d is smaller than zero" % - (self.sql, row)) + "%s failed: sql:%.40s, row:%d is smaller than zero" % + (callerFilename, self.sql, row)) if col < 0: tdLog.exit( - "failed: sql:%.40s, col:%d is smaller than zero" % - (self.sql, col)) + "%s failed: sql:%.40s, col:%d is smaller than zero" % + (callerFilename, self.sql, col)) if row >= self.queryRows: tdLog.exit( - "failed: sql:%.40s, row:%d is larger than queryRows:%d" % - (self.sql, row, self.queryRows)) + "%s failed: sql:%.40s, row:%d is larger than queryRows:%d" % + (callerFilename, self.sql, row, self.queryRows)) if col >= self.queryCols: tdLog.exit( - "failed: sql:%.40s, col:%d is larger than queryRows:%d" % - (self.sql, col, self.queryCols)) + "%s failed: sql:%.40s, col:%d is larger than queryRows:%d" % + (callerFilename, self.sql, col, self.queryCols)) return self.queryResult[row][col] def executeTimes(self, sql, times): @@ -137,8 +153,12 @@ class TDSql: def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: - tdLog.exit("failed: sql:%.40s, affectedRows:%d != expect:%d" % - (self.sql, self.affectedRows, expectAffectedRows)) + frame = inspect.stack()[1] + callerModule = inspect.getmodule(frame[0]) + callerFilename = callerModule.__file__ + + tdLog.exit("%s failed: sql:%.40s, affectedRows:%d != expect:%d" % ( + callerFilename, self.sql, self.affectedRows, expectAffectedRows)) tdLog.info("sql:%.40s, affectedRows:%d == expect:%d" % (self.sql, self.affectedRows, expectAffectedRows)) diff --git a/tests/pytest/valgrind-test.sh b/tests/pytest/valgrind-test.sh index 853ebe1d76..0b5dfc0fa3 100755 --- a/tests/pytest/valgrind-test.sh +++ b/tests/pytest/valgrind-test.sh @@ -1,45 +1,35 @@ #!/bin/bash -python3 ./test.py $1 -f insert/basic.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/int.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/float.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/bigint.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/bool.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/double.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/smallint.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/tinyint.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/binary.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/date.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f insert/nchar.py -python3 ./test.py $1 -s && sleep 1 +python3 ./test.py -g -f insert/basic.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/int.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/float.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/bigint.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/bool.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/double.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/smallint.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/tinyint.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/binary.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/date.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/nchar.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f insert/multi.py +python3 ./test.py -g -s && sleep 1 -python3 ./test.py $1 -f table/column_name.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f table/column_num.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f table/db_table.py -python3 ./test.py $1 -s && sleep 1 +python3 ./test.py -g -f table/column_name.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f table/column_num.py +python3 ./test.py -g -s && sleep 1 +python3 ./test.py -g -f table/db_table.py +python3 ./test.py -g -s && sleep 1 -python3 ./test.py $1 -f import_merge/importCacheFileT.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importDataLastSub.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importHead.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importLastT.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importSpan.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTail.py -python3 ./test.py $1 -s && sleep 1 -python3 ./test.py $1 -f import_merge/importTRestart.py -python3 ./test.py $1 -s && sleep 1 +python3 ./test.py -g -f import_merge/importDataLastSub.py +python3 ./test.py -g -s && sleep 1 diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index a99edfd93d..420b08c1e1 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -9,7 +9,6 @@ run general/import/basic.sim run general/import/commit.sim run general/insert/query_file_memory.sim run general/parser/binary_escapeCharacter.sim -run general/parser/columnValue_bigint.sim run general/parser/select_from_cache_disk.sim run general/table/autocreate.sim run general/table/column_name.sim @@ -18,3 +17,5 @@ run general/table/vgroup.sim run general/user/basic1.sim run general/user/pass_alter.sim run general/vector/single.sim +#run general/connection/connection.sim +run general/user/authority.sim diff --git a/tests/script/general/cache/new_metrics.sim b/tests/script/general/cache/new_metrics.sim index 64170ec67e..0e304d9acb 100644 --- a/tests/script/general/cache/new_metrics.sim +++ b/tests/script/general/cache/new_metrics.sim @@ -27,7 +27,7 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool) $i = 0 while $i < 5 $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 0 ) + sql create table $tb using $mt tags( $i ) $x = 0 while $x < $rowNum $val = $x * 60000 diff --git a/tests/script/general/column/commit.sim b/tests/script/general/column/commit.sim index 7bc62619f3..035d28d3e5 100644 --- a/tests/script/general/column/commit.sim +++ b/tests/script/general/column/commit.sim @@ -26,21 +26,16 @@ endi print =============== step2 -sql insert into d3.t1 values (now -300d,0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ) -sql insert into d3.t1 values (now-200d,1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ) -sql insert into d3.t1 values (now-150d,2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ) -sql insert into d3.t1 values (now-100d,3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 ) -sql insert into d3.t1 values (now-50d,4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 ) -sql insert into d3.t1 values (now-20d,5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 ) - -sql insert into d3.t1 values (now-10d,6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 ) - -sql insert into d3.t1 values (now-1d,7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 ) - -sql insert into d3.t1 values (now,8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 ) - -sql insert into d3.t1 values (now+1d,9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 ) - +sql insert into d3.t1 values (now -300d,0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ); +sql insert into d3.t1 values (now-200d,1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 ); +sql insert into d3.t1 values (now-150d,2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 , 2 ); +sql insert into d3.t1 values (now-100d,3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 , 3 ); +sql insert into d3.t1 values (now-50d,4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 , 4 ); +sql insert into d3.t1 values (now-20d,5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 , 5 ); +sql insert into d3.t1 values (now-10d,6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 , 6 ); +sql insert into d3.t1 values (now-1d,7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 , 7 ); +sql insert into d3.t1 values (now,8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 , 8 ); +sql insert into d3.t1 values (now+1d,9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 , 9 ); print =============== step3 diff --git a/tests/script/general/compute/diff2.sim b/tests/script/general/compute/diff2.sim index 53b508ca6f..0be907c16a 100644 --- a/tests/script/general/compute/diff2.sim +++ b/tests/script/general/compute/diff2.sim @@ -81,7 +81,7 @@ sql_error select diff(c8) from $tb sql_error select diff(c9) from $tb sql_error select diff(ts) from $tb sql_error select diff(c1), diff(c2) from $tb -sql_error select 2+diff(c1) from $tb +#sql_error select 2+diff(c1) from $tb sql_error select diff(c1+2) from $tb sql_error select diff(c1) from $tb where ts > 0 and ts < now + 100m interval(10m) sql_error select diff(c1) from $mt diff --git a/tests/script/general/compute/leastsquare.sim b/tests/script/general/compute/leastsquare.sim index bbadd849cb..5a28e74bff 100644 --- a/tests/script/general/compute/leastsquare.sim +++ b/tests/script/general/compute/leastsquare.sim @@ -48,41 +48,41 @@ $tb = $tbPrefix . $i sql select leastsquares(tbcol, 1, 1) from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step3 sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step4 sql select leastsquares(tbcol, 1, 1) as b from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step5 sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step6 sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print ===> $rows diff --git a/tests/script/general/connection/connection.sim b/tests/script/general/connection/connection.sim new file mode 100644 index 0000000000..abebbacbd9 --- /dev/null +++ b/tests/script/general/connection/connection.sim @@ -0,0 +1,21 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +print ============= step1 +sql close +print close1 +sql connect + +print ============= step2 +sql close +sql connect + +print ============= step3 +sql close +sql connect + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/db/delete.sim b/tests/script/general/db/delete.sim new file mode 100644 index 0000000000..f95676088b --- /dev/null +++ b/tests/script/general/db/delete.sim @@ -0,0 +1,65 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 10 + +print ========= start dnodes +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +print ======== step1 +sql create database db blocks 2 maxtables 1000 +sql create table db.mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$tbPrefix = db.t +$i = 0 +while $i < 2000 + $tb = $tbPrefix . $i + sql create table $tb using db.mt tags( $i ) + $i = $i + 1 +endw + +sql show db.vgroups +if $rows != 2 then + return -1 +endi + +return +print ======== step2 +sleep 1000 +sql drop database db +sql show databases +if $rows != 0 then + return -1 +endi + +sleep 1000 +sql show dnodes +print dnode1 openVnodes $data2_1 +if $data2_1 != 0 then + return -1 +endi + +print ======= step3 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 1000 +system sh/exec.sh -n dnode1 -s start + +$x = 0 +step3: + $x = $x + 1 + sleep 2000 + if $x == 30 then + return -1 + endi + +sql show mnodes +print dnode1 role $data2_1 +if $data2_1 != master then + goto step3 +endi + +sleep 1000 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/db/delete_reuse2.sim b/tests/script/general/db/delete_reuse2.sim index d8b2763042..d2fcaad838 100644 --- a/tests/script/general/db/delete_reuse2.sim +++ b/tests/script/general/db/delete_reuse2.sim @@ -30,6 +30,7 @@ sleep 2000 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect +sql reset query cache print ======== step1 sql create database d1 replica 1 diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim index 684910884b..54d8aa77c3 100644 --- a/tests/script/general/db/vnodes.sim +++ b/tests/script/general/db/vnodes.sim @@ -33,6 +33,8 @@ endw print ========== step2 sql select * from mt +print $rows +print $totalRows if $rows != $totalRows then return -1 endi diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index a5cc7b3237..a08c07a841 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -157,25 +157,25 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select count(i) from d1.mt group by a"},{"refId":"B","alias":"","sql":"select sum(i) from d1.mt group by b"}]' 127.0.0.1:6020/grafana/query print 16-> $system_content -if $system_content != @[{"refId":"A","target":"1","datapoints":[[5,"-"]]},{"refId":"A","target":"2","datapoints":[[4,"-"]]},{"refId":"A","target":"3","datapoints":[[3,"-"]]},{"refId":"B","target":"a","datapoints":[[5,"-"]]},{"refId":"B","target":"b","datapoints":[[8,"-"]]},{"refId":"B","target":"c","datapoints":[[9,"-"]]}]@ then +if $system_content != @[{"refId":"A","target":"{a:1,}","datapoints":[[5,"-"]]},{"refId":"A","target":"{a:2,}","datapoints":[[4,"-"]]},{"refId":"A","target":"{a:3,}","datapoints":[[3,"-"]]},{"refId":"B","target":"{b:a}","datapoints":[[5,"-"]]},{"refId":"B","target":"{b:b}","datapoints":[[8,"-"]]},{"refId":"B","target":"{b:c}","datapoints":[[9,"-"]]}]@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"count","sql":"select count(i) from d1.mt group by a"},{"refId":"B","alias":"sum-","sql":"select sum(i) from d1.mt group by b"}]' 127.0.0.1:6020/grafana/query print 17-> $system_content -if $system_content != @[{"refId":"A","target":"count1","datapoints":[[5,"-"]]},{"refId":"A","target":"count2","datapoints":[[4,"-"]]},{"refId":"A","target":"count3","datapoints":[[3,"-"]]},{"refId":"B","target":"sum-a","datapoints":[[5,"-"]]},{"refId":"B","target":"sum-b","datapoints":[[8,"-"]]},{"refId":"B","target":"sum-c","datapoints":[[9,"-"]]}]@ then +if $system_content != @[{"refId":"A","target":"count{a:1,}","datapoints":[[5,"-"]]},{"refId":"A","target":"count{a:2,}","datapoints":[[4,"-"]]},{"refId":"A","target":"count{a:3,}","datapoints":[[3,"-"]]},{"refId":"B","target":"sum-{b:a}","datapoints":[[5,"-"]]},{"refId":"B","target":"sum-{b:b}","datapoints":[[8,"-"]]},{"refId":"B","target":"sum-{b:c}","datapoints":[[9,"-"]]}]@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"count","sql":"select count(i) from d1.mt interval(1m) group by a "}]' 127.0.0.1:6020/grafana/query print 18-> $system_content -if $system_content != @[{"refId":"A","target":"count1","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000],[1,1514208540000]]},{"refId":"A","target":"count2","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000]]},{"refId":"A","target":"count3","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000]]}]@ then +if $system_content != @[{"refId":"A","target":"count{a:1,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000],[1,1514208540000]]},{"refId":"A","target":"count{a:2,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000],[1,1514208480000]]},{"refId":"A","target":"count{a:3,}","datapoints":[[1,1514208300000],[1,1514208360000],[1,1514208420000]]}]@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"refId":"A","alias":"","sql":"select sum(v2), count(v1) from d1.m1"},{"refId":"B","alias":"","sql":"select count(v2), sum(v2) from d1.m1"}]' 127.0.0.1:6020/grafana/query print 19-> $system_content -if $system_content != @[{"refId":"A","target":"3","datapoints":[[15.299999714,"-"]]},{"refId":"B","target":"15.299999714","datapoints":[[3,"-"]]}]@ then +if $system_content != @[{"refId":"A","target":"{count(v1):3}","datapoints":[[15.299999714,"-"]]},{"refId":"B","target":"{sum(v2):15.299999714}","datapoints":[[3,"-"]]}]@ then return -1 endi diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim index b2b7291623..1bd1419496 100644 --- a/tests/script/general/import/replica1.sim +++ b/tests/script/general/import/replica1.sim @@ -111,7 +111,8 @@ sql import into tb values(1520000034001, 34001) sql import into tb values(1520000050001, 50001) sql select * from tb; print $rows -if $rows != 19 then +if $rows != 19 then + print expect 19, actual: $rows return -1 endi diff --git a/tests/script/general/insert/insert_drop.sim b/tests/script/general/insert/insert_drop.sim index 5655576e20..c688a35557 100644 --- a/tests/script/general/insert/insert_drop.sim +++ b/tests/script/general/insert/insert_drop.sim @@ -43,7 +43,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -69,7 +69,7 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 4000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index f97581c5f5..e57b1f43d2 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -238,3 +238,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/alter_stable.sim b/tests/script/general/parser/alter_stable.sim index 6715c44b96..b64c919042 100644 --- a/tests/script/general/parser/alter_stable.sim +++ b/tests/script/general/parser/alter_stable.sim @@ -39,4 +39,4 @@ sql alter table tb1 set tag len = 379 # test end sql drop database $db - +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/auto_create_tb.sim b/tests/script/general/parser/auto_create_tb.sim index 54e028a200..ad65ea723f 100644 --- a/tests/script/general/parser/auto_create_tb.sim +++ b/tests/script/general/parser/auto_create_tb.sim @@ -1,6 +1,4 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 @@ -151,16 +149,17 @@ sql drop table tb1 #### auto create multiple tables sql insert into tb1 using $stb tags(1) values ( $ts0 , 1, 1, 1, 1, 'bin1', 1, 1, 1, '涛思数据1') tb2 using $stb tags(2) values ( $ts0 , 2, 2, 2, 2, 'bin2', 2, 2, 2, '涛思数据2') tb3 using $stb tags(3) values ( $ts0 , 3, 3, 3, 3, 'bin3', 3, 3, 3, '涛思数据3') sql show tables +print $rows $data00 $data10 $data20 if $rows != 3 then return -1 endi -if $data00 != tb1 then +if $data00 != tb3 then return -1 endi if $data10 != tb2 then return -1 endi -if $data20 != tb3 then +if $data20 != tb1 then return -1 endi @@ -301,6 +300,9 @@ sql_error create table txu using tu tags(0) values(now, 1); #[TBASE-675] sql insert into tu values(1565971200000, 1) (1565971200000,2) (1565971200001, 3)(1565971200001, 4) +sql select * from tu if $rows != 2 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/auto_create_tb_drop_tb.sim b/tests/script/general/parser/auto_create_tb_drop_tb.sim index aa14d2fdfb..f739f42a6e 100644 --- a/tests/script/general/parser/auto_create_tb_drop_tb.sim +++ b/tests/script/general/parser/auto_create_tb_drop_tb.sim @@ -1,8 +1,7 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c commitTime -v 30 +system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect @@ -22,7 +21,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db maxrows 200 cache 2048 maxTables 4 +sql create database $db maxrows 200 cache 2 maxTables 4 print ====== create tables sql use $db @@ -81,3 +80,5 @@ while $x < 100 $x = $x + 1 print loop $x endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index 5f7d98a75a..c3c867795a 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -95,9 +95,4 @@ if $data41 != @udp005@ then endi - - - - - - +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim index 0233b2ee04..0e97acb453 100644 --- a/tests/script/general/parser/bug.sim +++ b/tests/script/general/parser/bug.sim @@ -42,4 +42,3 @@ sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:6020/restful/sql system sh/exec.sh -n dnode1 -s stop -x SIGINT - diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 9ea8492907..132ecf4342 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -90,3 +90,5 @@ endi #### illegal operations sql_error select max(c2*2) from $tb sql_error select max(c1-c2) from $tb + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/columnValue.sim b/tests/script/general/parser/columnValue.sim index aa5c5f72c8..e905f61215 100644 --- a/tests/script/general/parser/columnValue.sim +++ b/tests/script/general/parser/columnValue.sim @@ -22,15 +22,5 @@ run general/parser/columnValue_bigint.sim run general/parser/columnValue_float.sim run general/parser/columnValue_double.sim - - - - - - - - - - - +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index 2b69b1ab8c..f1c980f3db 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -107,3 +107,5 @@ while $loop <= $loops endi $loop = $loop + 1 endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 8048770198..817d712aa6 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -188,3 +188,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/create_mt.sim b/tests/script/general/parser/create_mt.sim index 76fb486f1e..5b951b7b35 100644 --- a/tests/script/general/parser/create_mt.sim +++ b/tests/script/general/parser/create_mt.sim @@ -258,3 +258,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/create_tb.sim b/tests/script/general/parser/create_tb.sim index 7df7d8a4da..7bb329396c 100644 --- a/tests/script/general/parser/create_tb.sim +++ b/tests/script/general/parser/create_tb.sim @@ -188,3 +188,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/dbtbnameValidate.sim b/tests/script/general/parser/dbtbnameValidate.sim index 5dea4a09c5..cf72f3ae61 100644 --- a/tests/script/general/parser/dbtbnameValidate.sim +++ b/tests/script/general/parser/dbtbnameValidate.sim @@ -127,5 +127,4 @@ if $rows != 7 then return -1 endi - - +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index 459bc7611e..4dfee222cc 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -849,3 +849,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 113f4b68a4..402b79d6b5 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -430,3 +430,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index d2586ce04a..48288daeca 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -83,3 +83,4 @@ sleep 3000 run general/parser/first_last_query.sim +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 5bf41772ed..9788e4d484 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -406,4 +406,6 @@ endi if $data97 != @group_tb0@ then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/import_commit1.sim b/tests/script/general/parser/import_commit1.sim index fde0698c33..a929c1846b 100644 --- a/tests/script/general/parser/import_commit1.sim +++ b/tests/script/general/parser/import_commit1.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c commitTime -v 30 +system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect @@ -53,3 +53,4 @@ if $data00 != $res then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/import_commit2.sim b/tests/script/general/parser/import_commit2.sim index 2c49aa7eca..c93b3168f1 100644 --- a/tests/script/general/parser/import_commit2.sim +++ b/tests/script/general/parser/import_commit2.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c commitTime -v 30 +system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect @@ -52,3 +52,4 @@ if $data00 != $res then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/import_commit3.sim b/tests/script/general/parser/import_commit3.sim index 7ed5045b9a..916bf6d05e 100644 --- a/tests/script/general/parser/import_commit3.sim +++ b/tests/script/general/parser/import_commit3.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c commitTime -v 30 +system sh/cfg.sh -n dnode1 -c ctime -v 30 system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect @@ -59,3 +59,4 @@ if $data00 != $res then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/import_file.sim b/tests/script/general/parser/import_file.sim index cd63612a0e..6b4dd07c79 100644 --- a/tests/script/general/parser/import_file.sim +++ b/tests/script/general/parser/import_file.sim @@ -34,4 +34,6 @@ endi #system rm -f $inFileName # invalid shell -system rm -f ~/data.csv \ No newline at end of file +system rm -f ~/data.csv + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/insert_tb.sim b/tests/script/general/parser/insert_tb.sim index 4f2c5fcc78..4ba455c244 100644 --- a/tests/script/general/parser/insert_tb.sim +++ b/tests/script/general/parser/insert_tb.sim @@ -233,3 +233,4 @@ endi #endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index c7c633d422..4033d1c735 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -65,3 +65,5 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed run general/parser/interp_test.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 6addcb1ac1..2524fbe631 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -444,3 +444,5 @@ sql insert into um1 using m2 tags(1) values(1000001, 10)(2000000, 20); sql insert into um2 using m2 tags(9) values(1000001, 10)(2000000, 20); sql_error select count(*) from m1,m2 where m1.a=m2.a and m1.ts=m2.ts; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 86488ea844..3e32064f3b 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -127,4 +127,6 @@ sql select join_mt0.ts, join_mt1.t1 from join_mt0, join_mt1 where join_mt0.ts=jo sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 -sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1 \ No newline at end of file +sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index 5eb3b144a6..29f535942d 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -68,3 +68,5 @@ sql connect sleep 3000 run general/parser/lastrow_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index 8f00912a59..e3e952bdd3 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -69,3 +69,5 @@ sleep 3000 run general/parser/limit_tb.sim run general/parser/limit_stb.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/limit1.sim b/tests/script/general/parser/limit1.sim index 040b6303de..48fb6aaede 100644 --- a/tests/script/general/parser/limit1.sim +++ b/tests/script/general/parser/limit1.sim @@ -67,3 +67,5 @@ print ================== server restart completed run general/parser/limit1_tb.sim run general/parser/limit1_stb.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/limit1_tblocks100.sim b/tests/script/general/parser/limit1_tblocks100.sim index a776459640..af4d3eda85 100644 --- a/tests/script/general/parser/limit1_tblocks100.sim +++ b/tests/script/general/parser/limit1_tblocks100.sim @@ -67,3 +67,5 @@ print ================== server restart completed run general/parser/limit1_tb.sim run general/parser/limit1_stb.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/limit2.sim b/tests/script/general/parser/limit2.sim index cc9334db36..61f817644a 100644 --- a/tests/script/general/parser/limit2.sim +++ b/tests/script/general/parser/limit2.sim @@ -74,3 +74,5 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed run general/parser/limit2_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index 77838774df..569decaa14 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -142,3 +142,5 @@ endi if $data03 != 319 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 2a6bc83b41..2c86748f21 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -310,3 +310,5 @@ endi # if $rows != 0 then # return -1 # endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim index e4f6c5f849..d395b4c8ba 100644 --- a/tests/script/general/parser/null_char.sim +++ b/tests/script/general/parser/null_char.sim @@ -488,3 +488,4 @@ sql_error alter table st51 set tag tag_tinyint = abc379 #sql drop database $db +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index 4b10694190..52be50c032 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -375,4 +375,4 @@ sql_error select 'abc'; #=============================tbase-1205 sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); - +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/repeatAlter.sim b/tests/script/general/parser/repeatAlter.sim index 5f6a8e6a2a..3695206f90 100644 --- a/tests/script/general/parser/repeatAlter.sim +++ b/tests/script/general/parser/repeatAlter.sim @@ -5,3 +5,5 @@ while $i <= $loops run general/parser/alter.sim $i = $i + 1 endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/repeatStream.sim b/tests/script/general/parser/repeatStream.sim index bd631e36a1..616679e78b 100644 --- a/tests/script/general/parser/repeatStream.sim +++ b/tests/script/general/parser/repeatStream.sim @@ -5,3 +5,5 @@ while $i <= $repeats run general/parser/stream.sim $i = $i + 1 endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/selectResNum.sim b/tests/script/general/parser/selectResNum.sim index 0a4eafc8a0..a0adc46d5f 100644 --- a/tests/script/general/parser/selectResNum.sim +++ b/tests/script/general/parser/selectResNum.sim @@ -182,3 +182,5 @@ while $loop <= $loops endw $loop = $loop + 1 endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/select_across_vnodes.sim b/tests/script/general/parser/select_across_vnodes.sim index 883d4d59cd..65e8cbb7b1 100644 --- a/tests/script/general/parser/select_across_vnodes.sim +++ b/tests/script/general/parser/select_across_vnodes.sim @@ -80,3 +80,5 @@ sql show databases if $rows != 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/select_from_cache_disk.sim b/tests/script/general/parser/select_from_cache_disk.sim index 04cc68c310..9ffdf817d7 100644 --- a/tests/script/general/parser/select_from_cache_disk.sim +++ b/tests/script/general/parser/select_from_cache_disk.sim @@ -68,3 +68,5 @@ endi if $data12 != 1 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 181d7f312a..329787b69c 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -838,3 +838,5 @@ sql_error select first(c1), count(*), t2, t1, tbname from select_tags_mt0 group #sql select first(ts), tbname from select_tags_mt0 group by tbname; #sql select count(c1) from select_tags_mt0 where c1=99 group by tbname; #sql select count(*),tbname from select_tags_mt0 group by tbname + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim index de29c28ba3..a684fd5406 100644 --- a/tests/script/general/parser/set_tag_vals.sim +++ b/tests/script/general/parser/set_tag_vals.sim @@ -225,3 +225,5 @@ endi if $data04 != NULL then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/single_row_in_tb.sim b/tests/script/general/parser/single_row_in_tb.sim index 5ee65e3b10..52bdad2065 100644 --- a/tests/script/general/parser/single_row_in_tb.sim +++ b/tests/script/general/parser/single_row_in_tb.sim @@ -36,3 +36,5 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed run general/parser/single_row_in_tb_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/slimit.sim b/tests/script/general/parser/slimit.sim index 017e8d4d2e..161463a7c1 100644 --- a/tests/script/general/parser/slimit.sim +++ b/tests/script/general/parser/slimit.sim @@ -105,3 +105,5 @@ sql connect sleep 3000 run general/parser/slimit_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/slimit1.sim b/tests/script/general/parser/slimit1.sim index 3dbe9f0a06..b987a6852b 100644 --- a/tests/script/general/parser/slimit1.sim +++ b/tests/script/general/parser/slimit1.sim @@ -64,3 +64,5 @@ sql connect sleep 3000 run general/parser/slimit1_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/slimit_alter_tags.sim b/tests/script/general/parser/slimit_alter_tags.sim index 5c62482802..b8397f950c 100644 --- a/tests/script/general/parser/slimit_alter_tags.sim +++ b/tests/script/general/parser/slimit_alter_tags.sim @@ -255,3 +255,5 @@ endi #if $rows != 0 then # return -1 #endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/stream.sim b/tests/script/general/parser/stream.sim index c4be6efa1c..8e5dad1595 100644 --- a/tests/script/general/parser/stream.sim +++ b/tests/script/general/parser/stream.sim @@ -220,3 +220,5 @@ sql create database $db sql use $db sql create table stb (ts timestamp, c1 int) tags(t1 int) sql create table tb1 using stb tags(1) + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim index b7bf7a8900..5170ddb036 100644 --- a/tests/script/general/parser/stream_on_sys.sim +++ b/tests/script/general/parser/stream_on_sys.sim @@ -58,3 +58,5 @@ sql select * from iostrm if $rows <= 0 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/tags_dynamically_specifiy.sim b/tests/script/general/parser/tags_dynamically_specifiy.sim index af514d1a07..6e1766a91d 100644 --- a/tests/script/general/parser/tags_dynamically_specifiy.sim +++ b/tests/script/general/parser/tags_dynamically_specifiy.sim @@ -55,4 +55,6 @@ endi #if $data06 != 100.90000 then # print "expect: 100.90000, act: $data06" # return -1 -#endi \ No newline at end of file +#endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/tbnameIn.sim b/tests/script/general/parser/tbnameIn.sim index 548560e945..16302888d0 100644 --- a/tests/script/general/parser/tbnameIn.sim +++ b/tests/script/general/parser/tbnameIn.sim @@ -73,3 +73,5 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed run general/parser/tbnameIn_query.sim + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index a2a08097dd..0b7d705688 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -38,8 +38,16 @@ sleep 2000 run general/parser/lastrow.sim sleep 2000 run general/parser/nchar.sim + sleep 2000 run general/parser/null_char.sim +sleep 2000 +run general/parser/single_row_in_tb.sim +sleep 2000 +run general/parser/select_from_cache_disk.sim +sleep 2000 +run general/parser/limit.sim + sleep 2000 run general/parser/fill.sim sleep 2000 @@ -49,8 +57,6 @@ run general/parser/tags_dynamically_specifiy.sim sleep 2000 run general/parser/interp.sim sleep 2000 -run general/parser/limit.sim -sleep 2000 run general/parser/limit1.sim sleep 2000 run general/parser/limit1_tblocks100.sim @@ -63,12 +69,8 @@ run general/parser/selectResNum.sim sleep 2000 run general/parser/select_across_vnodes.sim sleep 2000 -run general/parser/select_from_cache_disk.sim -sleep 2000 run general/parser/set_tag_vals.sim sleep 2000 -run general/parser/single_row_in_tb.sim -sleep 2000 run general/parser/slimit.sim sleep 2000 run general/parser/slimit1.sim @@ -96,3 +98,8 @@ sleep 2000 run general/parser/select_with_tags.sim sleep 2000 run general/parser/groupby.sim + +sleep 2000 +run general/parser/binary_escapeCharacter.sim +sleep 2000 +#run general/parser/bug.sim \ No newline at end of file diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index fb3a0a69e7..de2312f2b9 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -272,3 +272,5 @@ sql select * from tb_where_NULL where c2 <> "nUll" if $rows != 2 then return -1 endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/dnode3.sim b/tests/script/general/stable/dnode3.sim new file mode 100644 index 0000000000..38d05a924b --- /dev/null +++ b/tests/script/general/stable/dnode3.sim @@ -0,0 +1,223 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode3 -c walLevel -v 0 +system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +createDnode: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi +sql show dnodes; +if $data4_2 == offline then + goto createDnode +endi +if $data4_3 == offline then + goto createDnode +endi +if $data4_4 == offline then + goto createDnode +endi + +print ======================== dnode1 start + +$dbPrefix = r3v3_db +$tbPrefix = r3v3_tb +$mtPrefix = r3v3_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql create database $db maxTables 4 +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $val = $x * 60000 + $ms = 1519833600000 + $val + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql show vgroups +print vgroups ==> $rows +if $rows != 3 then + return -1 +endi + +sleep 100 + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(*) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +sql select count(tbcol) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step3 +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi + +print =============== step4 +sql select count(tbcol) as b from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step5 +sql select count(tbcol) as b from $tb interval(1m) +print ===> $data01 +if $data01 != 1 then + return -1 +endi + +sql select count(tbcol) as b from $tb interval(1d) +print ===> $data01 +if $data01 != $rowNum then + return -1 +endi + +print =============== step6 +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data01 +if $data01 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi + +print =============== step7 +print select count(*) from $mt +sql select count(*) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +print =============== step8 +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi + +print =============== step9 +sql select count(tbcol) as b from $mt interval(1m) +print ===> $data01 +if $data01 != 10 then + return -1 +endi + +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data01 +if $data01 != 200 then + return -1 +endi + +print =============== step10 +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi + +print =============== step11 +sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol +print ===> $data01 +if $data01 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/testSuite.sim b/tests/script/general/stable/testSuite.sim index a3696c8706..e786ac9ca4 100644 --- a/tests/script/general/stable/testSuite.sim +++ b/tests/script/general/stable/testSuite.sim @@ -1,4 +1,5 @@ run general/stable/disk.sim +run general/stable/dnode3.sim run general/stable/metrics.sim run general/stable/values.sim run general/stable/vnode3.sim diff --git a/tests/script/general/tag/create.sim b/tests/script/general/tag/create.sim index c0cdbe87dc..3e16810f91 100644 --- a/tests/script/general/tag/create.sim +++ b/tests/script/general/tag/create.sim @@ -134,7 +134,8 @@ if $data01 != 1 then return -1 endi sql select * from $mt where tgcol = 0 -if $rows != 0 then +if $rows != 0 then + print expect 0, actual: $rows return -1 endi @@ -184,7 +185,8 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool, tgcol2 bool) sql create table $tb using $mt tags( 1, 2 ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = 2 -if $rows != 1 then +if $rows != 1 then + print expect 1, actual: $rows return -1 endi if $data01 != 1 then diff --git a/tests/script/general/user/authority.sim b/tests/script/general/user/authority.sim new file mode 100644 index 0000000000..7ae8c6f1a4 --- /dev/null +++ b/tests/script/general/user/authority.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ============= step1 +sql create user read pass 'taosdata' +sql create user write pass 'taosdata' +sql show users +if $rows != 5 then + return -1 +endi + +print ============= step2 +sql close +sql connect read +sleep 2000 + +sql create database dread +sql show databases +if $rows != 1 then + return -1 +endi + +print ============= step3 +sql close +sql connect write +sleep 2000 + +sql create database dwrite +sql show databases +if $rows != 2 then + return -1 +endi + +print ============ step4 +sql close +sql connect +sleep 2000 + +sql show databases +if $row != 2 then + return -1 +endi + +print ============ step5 +sql close +sql connect read +sleep 2000 +sql drop database dread +sql drop database dwrite + + +sql close +sql connect +sql show databases +if $rows != 0 then + return -1 +endi + +sql close +sql connect +sleep 2000 + +system sh/exec_up.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 62bae1c2a5..272183b26d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -3,21 +3,23 @@ cd ../../debug; make cd ../../../debug; cmake .. cd ../../../debug; make -#./test.sh -f general/alter/cached_schema_after_alter.sim -#./test.sh -f general/alter/count.sim -#./test.sh -f general/alter/import.sim -#./test.sh -f general/alter/insert1.sim -#./test.sh -f general/alter/insert2.sim -#./test.sh -f general/alter/metrics.sim -#./test.sh -f general/alter/table.sim +#unsupport ./test.sh -f general/alter/cached_schema_after_alter.sim +#unsupport ./test.sh -f general/alter/count.sim +#unsupport ./test.sh -f general/alter/import.sim +#unsupport ./test.sh -f general/alter/insert1.sim +#unsupport ./test.sh -f general/alter/insert2.sim +#unsupport ./test.sh -f general/alter/metrics.sim +#unsupport ./test.sh -f general/alter/table.sim ./test.sh -f general/cache/new_metrics.sim ./test.sh -f general/cache/restart_metrics.sim ./test.sh -f general/cache/restart_table.sim -#hongze ./test.sh -f general/column/commit.sim -#hongze ./test.sh -f general/column/metrics.sim -#hongze ./test.sh -f general/column/table.sim +./test.sh -f general/connection/connection.sim + +./test.sh -f general/column/commit.sim +./test.sh -f general/column/metrics.sim +./test.sh -f general/column/table.sim ./test.sh -f general/compress/commitlog.sim ./test.sh -f general/compress/compress.sim @@ -28,11 +30,11 @@ cd ../../../debug; make ./test.sh -f general/compute/bottom.sim ./test.sh -f general/compute/count.sim ./test.sh -f general/compute/diff.sim -# liao./test.sh -f general/compute/diff2.sim +./test.sh -f general/compute/diff2.sim ./test.sh -f general/compute/first.sim -# liao./test.sh -f general/compute/interval.sim -# liao./test.sh -f general/compute/last.sim -# liao./test.sh -f general/compute/leastsquare.sim +# liao ./test.sh -f general/compute/interval.sim +./test.sh -f general/compute/last.sim +./test.sh -f general/compute/leastsquare.sim ./test.sh -f general/compute/max.sim ./test.sh -f general/compute/min.sim ./test.sh -f general/compute/null.sim @@ -47,6 +49,7 @@ cd ../../../debug; make ./test.sh -f general/db/basic3.sim ./test.sh -f general/db/basic4.sim ./test.sh -f general/db/basic5.sim +./test.sh -f general/db/delete.sim ./test.sh -f general/db/delete_reuse1.sim ./test.sh -f general/db/delete_reuse2.sim ./test.sh -f general/db/delete_reusevnode.sim @@ -54,38 +57,38 @@ cd ../../../debug; make ./test.sh -f general/db/delete_writing1.sim ./test.sh -f general/db/delete_writing2.sim ./test.sh -f general/db/len.sim -#./test.sh -u -f general/db/vnodes.sim +#liao ./test.sh -f general/db/vnodes.sim ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim ./test.sh -f general/field/2.sim -#./test.sh -f general/field/3.sim -#./test.sh -f general/field/4.sim -#./test.sh -f general/field/5.sim -#./test.sh -f general/field/6.sim +./test.sh -f general/field/3.sim +./test.sh -f general/field/4.sim +./test.sh -f general/field/5.sim +./test.sh -f general/field/6.sim ./test.sh -f general/field/bigint.sim -# liao./test.sh -f general/field/binary.sim +./test.sh -f general/field/binary.sim ./test.sh -f general/field/bool.sim ./test.sh -f general/field/single.sim ./test.sh -f general/field/smallint.sim ./test.sh -f general/field/tinyint.sim -# jeff ./test.sh -f general/http/restful.sim +./test.sh -f general/http/restful.sim ./test.sh -f general/http/restful_insert.sim ./test.sh -f general/http/restful_limit.sim -# jeff ./test.sh -f general/http/restful_full.sim +./test.sh -f general/http/restful_full.sim ./test.sh -f general/http/prepare.sim ./test.sh -f general/http/telegraf.sim ./test.sh -f general/http/grafana_bug.sim -# jeff ./test.sh -f general/http/grafana.sim +./test.sh -f general/http/grafana.sim ./test.sh -f general/import/basic.sim ./test.sh -f general/import/commit.sim ./test.sh -f general/import/large.sim -#hongze ./test.sh -f general/import/replica1.sim +#liao ./test.sh -f general/import/replica1.sim ./test.sh -f general/insert/basic.sim -#hongze ./test.sh -f general/insert/insert_drop.sim +#liao ./test.sh -f general/insert/insert_drop.sim ./test.sh -f general/insert/query_block1_memory.sim ./test.sh -f general/insert/query_block2_memory.sim ./test.sh -f general/insert/query_block1_file.sim @@ -94,74 +97,68 @@ cd ../../../debug; make ./test.sh -f general/insert/query_multi_file.sim ./test.sh -f general/insert/tcp.sim -# ./test.sh -f general/parser/alter.sim -# ./test.sh -f general/parser/alter1.sim -# ./test.sh -f general/parser/alter_stable.sim -# ./test.sh -f general/parser/auto_create_tb.sim -# ./test.sh -f general/parser/auto_create_tb_drop_tb.sim -./test.sh -f general/parser/binary_escapeCharacter.sim -#./test.sh -f general/parser/bug.sim +#unsupport ./test.sh -f general/parser/alter.sim +#unsupport ./test.sh -f general/parser/alter1.sim +#unsupport ./test.sh -f general/parser/alter_stable.sim +./test.sh -f general/parser/auto_create_tb.sim +./test.sh -f general/parser/auto_create_tb_drop_tb.sim ./test.sh -f general/parser/col_arithmetic_operation.sim -./test.sh -f general/parser/columnValue_bigint.sim -./test.sh -f general/parser/columnValue_bool.sim -./test.sh -f general/parser/columnValue_double.sim -./test.sh -f general/parser/columnValue_float.sim -./test.sh -f general/parser/columnValue_int.sim -# ./test.sh -f general/parser/col_arithmetic_operation.sim ./test.sh -f general/parser/columnValue.sim ./test.sh -f general/parser/commit.sim # ./test.sh -f general/parser/create_db.sim # ./test.sh -f general/parser/create_mt.sim # ./test.sh -f general/parser/create_tb.sim # ./test.sh -f general/parser/dbtbnameValidate.sim -# ./test.sh -f general/parser/fill.sim -# ./test.sh -f general/parser/fill_stb.sim -# ./test.sh -f general/parser/first_last.sim ./test.sh -f general/parser/import_commit1.sim ./test.sh -f general/parser/import_commit2.sim ./test.sh -f general/parser/import_commit3.sim -# ./test.sh -f general/parser/import_file.sim # ./test.sh -f general/parser/insert_tb.sim +# ./test.sh -f general/parser/first_last.sim +# ./test.sh -f general/parser/import_file.sim +# ./test.sh -f general/parser/lastrow.sim +# ./test.sh -f general/parser/nchar.sim +# ./test.sh -f general/parser/null_char.sim +# ./test.sh -f general/parser/single_row_in_tb.sim +./test.sh -f general/parser/select_from_cache_disk.sim +# ./test.sh -f general/parser/limit.sim +# ./test.sh -f general/parser/fill.sim +# ./test.sh -f general/parser/fill_stb.sim # ./test.sh -f general/parser/tags_dynamically_specifiy.sim # ./test.sh -f general/parser/interp.sim -# ./test.sh -f general/parser/lastrow.sim -# ./test.sh -f general/parser/limit.sim # ./test.sh -f general/parser/limit1.sim # ./test.sh -f general/parser/limit1_tblocks100.sim # ./test.sh -f general/parser/limit2.sim # ./test.sh -f general/parser/mixed_blocks.sim -# ./test.sh -f general/parser/nchar.sim -# ./test.sh -f general/parser/null_char.sim # ./test.sh -f general/parser/selectResNum.sim # ./test.sh -f general/parser/select_across_vnodes.sim -./test.sh -f general/parser/select_from_cache_disk.sim # ./test.sh -f general/parser/set_tag_vals.sim -# ./test.sh -f general/parser/single_row_in_tb.sim # ./test.sh -f general/parser/slimit.sim ./test.sh -f general/parser/slimit1.sim -./test.sh -f general/parser/slimit1_query.sim -# ./test.sh -f general/parser/slimit_alter_tags.sim -# ./test.sh -f general/parser/stream_on_sys.sim -# ./test.sh -f general/parser/stream.sim +#unsupport ./test.sh -f general/parser/slimit_alter_tags.sim +#unsupport ./test.sh -f general/parser/stream_on_sys.sim +#unsupport ./test.sh -f general/parser/stream.sim # ./test.sh -f general/parser/tbnameIn.sim # ./test.sh -f general/parser/where.sim -# #./test.sh -f general/parser/repeatAlter.sim -# #./test.sh -f general/parser/repeatStream.sim +# ./test.sh -f general/parser/repeatAlter.sim +#unsupport ./test.sh -f general/parser/repeatStream.sim # ./test.sh -f general/parser/join.sim # ./test.sh -f general/parser/join_multivnode.sim # ./test.sh -f general/parser/projection_limit_offset.sim # ./test.sh -f general/parser/select_with_tags.sim # ./test.sh -f general/parser/groupby.sim +./test.sh -f general/parser/binary_escapeCharacter.sim +#./test.sh -f general/parser/bug.sim -#./test.sh -f general/stable/disk.sim -#./test.sh -f general/stable/metrics.sim -#./test.sh -f general/stable/values.sim +./test.sh -f general/stable/disk.sim +#liao ./test.sh -f general/stable/dnode3.sim +./test.sh -f general/stable/metrics.sim +./test.sh -f general/stable/values.sim ./test.sh -f general/stable/vnode3.sim ./test.sh -f general/table/autocreate.sim ./test.sh -f general/table/basic1.sim ./test.sh -f general/table/basic2.sim -#hongze ./test.sh -f general/table/basic3.sim +./test.sh -f general/table/basic3.sim ./test.sh -f general/table/bigint.sim ./test.sh -f general/table/binary.sim ./test.sh -f general/table/bool.sim @@ -186,32 +183,33 @@ cd ../../../debug; make ./test.sh -f general/table/tinyint.sim ./test.sh -f general/table/vgroup.sim -#./test.sh -f general/tag/3.sim -#./test.sh -f general/tag/4.sim -#./test.sh -f general/tag/5.sim -#./test.sh -f general/tag/6.sim -#./test.sh -f general/tag/add.sim +./test.sh -f general/tag/3.sim +./test.sh -f general/tag/4.sim +./test.sh -f general/tag/5.sim +./test.sh -f general/tag/6.sim +#unsupport ./test.sh -f general/tag/add.sim ./test.sh -f general/tag/bigint.sim -#./test.sh -f general/tag/binary_binary.sim -#./test.sh -f general/tag/binary.sim -#./test.sh -f general/tag/bool_binary.sim -#./test.sh -f general/tag/bool_int.sim +./test.sh -f general/tag/binary_binary.sim +./test.sh -f general/tag/binary.sim +./test.sh -f general/tag/bool_binary.sim +./test.sh -f general/tag/bool_int.sim ./test.sh -f general/tag/bool.sim -#./test.sh -f general/tag/change.sim -#liao ./test.sh -f general/tag/column.sim -#./test.sh -f general/tag/commit.sim -#liao ./test.sh -f general/tag/create.sim -#./test.sh -f general/tag/delete.sim -#./test.sh -f general/tag/double.sim +#unsupport ./test.sh -f general/tag/change.sim +./test.sh -f general/tag/column.sim +#unsupport ./test.sh -f general/tag/commit.sim +./test.sh -f general/tag/create.sim +#unsupport ./test.sh -f general/tag/delete.sim +./test.sh -f general/tag/double.sim ./test.sh -f general/tag/filter.sim -#./test.sh -f general/tag/float.sim -#./test.sh -f general/tag/int_binary.sim -#./test.sh -f general/tag/int_float.sim +./test.sh -f general/tag/float.sim +./test.sh -f general/tag/int_binary.sim +./test.sh -f general/tag/int_float.sim ./test.sh -f general/tag/int.sim -#./test.sh -f general/tag/set.sim +#unsupport ./test.sh -f general/tag/set.sim ./test.sh -f general/tag/smallint.sim ./test.sh -f general/tag/tinyint.sim +./test.sh -f general/user/authority.sim ./test.sh -f general/user/basic1.sim ./test.sh -f general/user/monitor.sim ./test.sh -f general/user/pass_alter.sim @@ -224,7 +222,7 @@ cd ../../../debug; make ./test.sh -f general/vector/metrics_query.sim ./test.sh -f general/vector/metrics_tag.sim ./test.sh -f general/vector/metrics_time.sim -#liao ./test.sh -f general/vector/multi.sim +./test.sh -f general/vector/multi.sim ./test.sh -f general/vector/single.sim ./test.sh -f general/vector/table_field.sim ./test.sh -f general/vector/table_mix.sim @@ -243,52 +241,52 @@ cd ../../../debug; make ./test.sh -u -f unique/account/user_create.sim ./test.sh -u -f unique/account/user_len.sim -#./test.sh -u -f unique/big/balance.sim -#slguan ./test.sh -u -f unique/big/maxvnodes.sim +#liao wait ./test.sh -u -f unique/big/balance.sim +#liao wait ./test.sh -u -f unique/big/maxvnodes.sim ./test.sh -u -f unique/big/tcp.sim -##./test.sh -u -f unique/cluster/balance1.sim -##./test.sh -u -f unique/cluster/balance2.sim -##./test.sh -u -f unique/cluster/balance3.sim -#./test.sh -u -f unique/cluster/cache.sim +./test.sh -u -f unique/cluster/balance1.sim +./test.sh -u -f unique/cluster/balance2.sim +./test.sh -u -f unique/cluster/balance3.sim +./test.sh -u -f unique/cluster/cache.sim ./test.sh -u -f unique/column/replica3.sim -#./test.sh -u -f unique/db/commit.sim -#./test.sh -u -f unique/db/delete.sim -#./test.sh -u -f unique/db/delete_part.sim -##./test.sh -u -f unique/db/replica_add12.sim -##./test.sh -u -f unique/db/replica_add13.sim -##./test.sh -u -f unique/db/replica_add23.sim -##./test.sh -u -f unique/db/replica_reduce21.sim -##./test.sh -u -f unique/db/replica_reduce32.sim -##./test.sh -u -f unique/db/replica_reduce31.sim -##./test.sh -u -f unique/db/replica_part.sim +#liao wait ./test.sh -u -f unique/db/commit.sim +./test.sh -u -f unique/db/delete.sim +./test.sh -u -f unique/db/delete_part.sim +./test.sh -u -f unique/db/replica_add12.sim +#hongze ./test.sh -u -f unique/db/replica_add13.sim +#hongze wait ./test.sh -u -f unique/db/replica_add23.sim +#hongze wait ./test.sh -u -f unique/db/replica_reduce21.sim +./test.sh -u -f unique/db/replica_reduce32.sim +#hongze wait /test.sh -u -f unique/db/replica_reduce31.sim +./test.sh -u -f unique/db/replica_part.sim -##./test.sh -u -f unique/dnode/balance1.sim -##./test.sh -u -f unique/dnode/balance2.sim -##./test.sh -u -f unique/dnode/balance3.sim -##./test.sh -u -f unique/dnode/balancex.sim -##./test.sh -u -f unique/dnode/offline1.sim -##./test.sh -u -f unique/dnode/offline2.sim -#./test.sh -u -f unique/dnode/remove1.sim -#./test.sh -u -f unique/dnode/remove2.sim -#./test.sh -u -f unique/dnode/vnode_clean.sim +./test.sh -u -f unique/dnode/balance1.sim +./test.sh -u -f unique/dnode/balance2.sim +./test.sh -u -f unique/dnode/balance3.sim +./test.sh -u -f unique/dnode/balancex.sim +./test.sh -u -f unique/dnode/offline1.sim +#hongze wait ./test.sh -u -f unique/dnode/offline2.sim +./test.sh -u -f unique/dnode/remove1.sim +#hongze ./test.sh -u -f unique/dnode/remove2.sim +./test.sh -u -f unique/dnode/vnode_clean.sim -##./test.sh -u -f unique/http/admin.sim -##./test.sh -u -f unique/http/opentsdb.sim +./test.sh -u -f unique/http/admin.sim +./test.sh -u -f unique/http/opentsdb.sim -#./test.sh -u -f unique/import/replica2.sim -#./test.sh -u -f unique/import/replica3.sim +#liao wait ./test.sh -u -f unique/import/replica2.sim +#liao wait ./test.sh -u -f unique/import/replica3.sim -#./test.sh -u -f unique/metrics/balance_replica1.sim -#./test.sh -u -f unique/metrics/dnode2_stop.sim -#./test.sh -u -f unique/metrics/dnode2.sim -#./test.sh -u -f unique/metrics/dnode3.sim -#./test.sh -u -f unique/metrics/replica2_dnode4.sim -#./test.sh -u -f unique/metrics/replica2_vnode3.sim -#./test.sh -u -f unique/metrics/replica3_dnode6.sim -#./test.sh -u -f unique/metrics/replica3_vnode3.sim +#liao wait ./test.sh -u -f unique/stable/balance_replica1.sim +#liao wait ./test.sh -u -f unique/stable/dnode2_stop.sim +#liao wait ./test.sh -u -f unique/stable/dnode2.sim +#liao wait ./test.sh -u -f unique/stable/dnode3.sim +#liao wait ./test.sh -u -f unique/stable/replica2_dnode4.sim +#liao wait ./test.sh -u -f unique/stable/replica2_vnode3.sim +#liao wait ./test.sh -u -f unique/stable/replica3_dnode6.sim +#liao wait ./test.sh -u -f unique/stable/replica3_vnode3.sim ./test.sh -u -f unique/mnode/mgmt22.sim ./test.sh -u -f unique/mnode/mgmt23.sim @@ -297,16 +295,11 @@ cd ../../../debug; make ./test.sh -u -f unique/mnode/mgmt26.sim ./test.sh -u -f unique/mnode/mgmt33.sim ./test.sh -u -f unique/mnode/mgmt34.sim -#./test.sh -u -f unique/mnode/mgmtr2.sim -#./test.sh -u -f unique/mnode/secondIp.sim +./test.sh -u -f unique/mnode/mgmtr2.sim -##./test.sh -u -f unique/table/delete_part.sim - -#./test.sh -u -f unique/vnode/commit.sim -#./test.sh -u -f unique/vnode/many.sim -#./test.sh -u -f unique/vnode/replica2_basic.sim +./test.sh -u -f unique/vnode/many.sim ./test.sh -u -f unique/vnode/replica2_basic2.sim -#./test.sh -u -f unique/vnode/replica2_repeat.sim -##./test.sh -u -f unique/vnode/replica3_basic.sim -#./test.sh -u -f unique/vnode/replica3_repeat.sim -#./test.sh -u -f unique/vnode/replica3_vgroup.sim +./test.sh -u -f unique/vnode/replica2_repeat.sim +./test.sh -u -f unique/vnode/replica3_basic.sim +./test.sh -u -f unique/vnode/replica3_repeat.sim +./test.sh -u -f unique/vnode/replica3_vgroup.sim diff --git a/tests/script/test.sh b/tests/script/test.sh index 8e3959a680..96ab3c5808 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -93,7 +93,7 @@ echo "numOfLogLines 100000000" >> $TAOS_CFG echo "dDebugFlag 135" >> $TAOS_CFG echo "mDebugFlag 135" >> $TAOS_CFG echo "sdbDebugFlag 135" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 151" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 135" >> $TAOS_CFG diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index 10e05971d7..424a80d25a 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -77,7 +77,7 @@ print dnode2 $dnode2Vnodes if $dnode1Vnodes != 2 then return -1 endi -if $dnode2Vnodes != NULL then +if $dnode2Vnodes != null then return -1 endi @@ -146,7 +146,7 @@ print dnode2 $dnode2Vnodes if $dnode1Vnodes != 3 then goto show4 endi -if $dnode2Vnodes != NULL then +if $dnode2Vnodes != null then goto show4 endi @@ -229,7 +229,7 @@ print dnode3 $dnode3Vnodes if $dnode1Vnodes != 3 then goto show8 endi -if $dnode3Vnodes != NULL then +if $dnode3Vnodes != null then goto show8 endi @@ -245,7 +245,7 @@ if $dnode1Role != master then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index d741e03eec..08fdd233e0 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -131,7 +131,7 @@ print dnode3 $dnode3Vnodes if $dnode1Vnodes != 3 then goto show2 endi -if $dnode2Vnodes != NULL then +if $dnode2Vnodes != null then goto show2 endi if $dnode3Vnodes != 3 then @@ -194,7 +194,7 @@ print dnode4 ==> $dnode4Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi if $dnode3Role != slave then @@ -231,7 +231,7 @@ endi if $dnode4Vnodes != 3 then goto show4 endi -if $dnode3Vnodes != NULL then +if $dnode3Vnodes != null then goto show4 endi @@ -248,10 +248,10 @@ print dnode4 ==> $dnode4Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi @@ -339,7 +339,7 @@ print dnode4 $dnode4Vnodes $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes -if $dnode1Vnodes != NULL then +if $dnode1Vnodes != null then goto show6 endi if $dnode4Vnodes != 3 then diff --git a/tests/script/unique/cluster/balance3.sim b/tests/script/unique/cluster/balance3.sim index e9847e21da..407adc7f3b 100644 --- a/tests/script/unique/cluster/balance3.sim +++ b/tests/script/unique/cluster/balance3.sim @@ -110,7 +110,7 @@ endi if $dnode3Vnodes != 3 then goto show1 endi -if $dnode4Vnodes != NULL then +if $dnode4Vnodes != null then goto show1 endi @@ -166,7 +166,7 @@ print dnode4 $dnode4Vnodes if $dnode1Vnodes != 3 then goto show3 endi -if $dnode2Vnodes != NULL then +if $dnode2Vnodes != null then goto show3 endi if $dnode3Vnodes != 3 then @@ -232,7 +232,7 @@ endi if $dnode5Vnodes != 3 then goto show5 endi -if $dnode3Vnodes != NULL then +if $dnode3Vnodes != null then goto show5 endi if $dnode4Vnodes != 3 then @@ -298,7 +298,7 @@ endi if $dnode6Vnodes != 3 then goto show7 endi -if $dnode4Vnodes != NULL then +if $dnode4Vnodes != null then goto show7 endi diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index 41f9db69f3..e23b407828 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -4,26 +4,18 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode2 -c walLevel -v 0 system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 2 system sh/cfg.sh -n dnode2 -c httpMaxThreads -v 2 system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode2 -c http -v 1 +system sh/cfg.sh -n dnode1 -c monitor -v 2 +system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode1 -c enableHttp -v 1 system sh/cfg.sh -n dnode2 -c monitor -v 1 system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/cfg.sh -n dnode2 -c monitorInterval -v 1 -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode2 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode2 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode2 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 -system sh/cfg.sh -n dnode2 -c maxShellConns -v 30000 - system sh/exec.sh -n dnode1 -s start sleep 3000 sql connect @@ -45,13 +37,20 @@ sleep 3000 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 20000 -sql select * from log.dn_192_168_0_1 + +sleep 10000 + +sql show log.tables; +if $rows != 5 then + return -1 +endi + +sql select * from log.dn1 print ===>rows $rows print $data00 $data01 $data02 print $data10 $data11 $data12 print $data20 $data21 $data22 -if $rows < 20 then +if $rows < 10 then return -1 endi diff --git a/tests/script/unique/db/commit.sim b/tests/script/unique/db/commit.sim index 648cd8db2f..5bf6ea6f10 100644 --- a/tests/script/unique/db/commit.sim +++ b/tests/script/unique/db/commit.sim @@ -72,8 +72,8 @@ sql import into tb values (now - 10d , -10 ) sql import into tb values (now - 11d , -11 ) sql select * from tb order by ts desc -print ===> rows $rows -print ===> last $data01 +print ===> rows $rows expect $num +print ===> last $data01 expect $data01 if $rows != $num then return -1 diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index e222db8d70..5688333d20 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -1,7 +1,4 @@ system sh/stop_dnodes.sh - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 @@ -18,33 +15,81 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 10 print ========= start dnodes system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect sql create dnode $hostname2 system sh/exec_up.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec_up.sh -n dnode3 -s start -sleep 3000 print ======== step1 -sql create database db replica 3 ablocks 2 tblocks 5 maxtables 10000 +sql create database db replica 3 blocks 2 maxtables 1000 sql create table db.mt (ts timestamp, tbcol int) TAGS(tgcol int) $tbPrefix = db.t $i = 0 -while $i < 100000 +while $i < 2000 $tb = $tbPrefix . $i sql create table $tb using db.mt tags( $i ) $i = $i + 1 endw +sleep 2500 + +sql show db.vgroups +if $rows != 2 then + return -1 +endi + print ======== step2 -sleep 1000 sql drop database db sql show databases if $rows != 0 then return -1 endi +sleep 3000 +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 0 then + return -1 +endi +if $data2_3 != 0 then + return -1 +endi + +print ======== step3 + +system sh/exec_up.sh -n dnode1 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT + +sleep 1000 +system sh/exec_up.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode3 -s start + +$x = 0 +step3: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show mnodes +print dnode1 role $data2_1 +if $data2_1 != master then + goto step3 +endi + +sleep 1000 + system sh/exec_up.sh -n dnode1 -s stop -x SIGINT system sh/exec_up.sh -n dnode2 -s stop -x SIGINT system sh/exec_up.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index 3d1cc5dc63..179d729d8d 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,44 +31,147 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 print ========= start dnodes system sh/exec_up.sh -n dnode1 -s start +sleep 3000 sql connect sql create dnode $hostname2 system sh/exec_up.sh -n dnode2 -s start -sleep 3000 $loop = 0 begin: $db = db . $loop - print ======== step1 + print ======== step1 $loop sql create database $db sql use $db - $x = 0 - while $x < 32 - $tb = tb . $x - sql create table $tb (ts timestamp, i int) - sql insert into $tb values(now, $x ) - $x = $x + 1 - endw + sql create table t11 (ts timestamp, i int) + sql insert into t11 values(now, 1 ) + sql create table t12 (ts timestamp, i int) + sql insert into t12 values(now, 1 ) + sql create table t13 (ts timestamp, i int) + sql insert into t13 values(now, 1 ) + sql create table t14 (ts timestamp, i int) + sql insert into t14 values(now, 1 ) + sleep 1200 - print ======== step2 - system sh/exec_up.sh -n dnode2 -s stop - sql drop database $db + sql create table t21 (ts timestamp, i int) + sql insert into t21 values(now, 1 ) + sql create table t22 (ts timestamp, i int) + sql insert into t22 values(now, 1 ) + sql create table t23 (ts timestamp, i int) + sql insert into t23 values(now, 1 ) + sql create table t24 (ts timestamp, i int) + sql insert into t24 values(now, 1 ) + sleep 1200 + sql create table t31 (ts timestamp, i int) + sql insert into t31 values(now, 1 ) + sql create table t32 (ts timestamp, i int) + sql insert into t32 values(now, 1 ) + sql create table t33 (ts timestamp, i int) + sql insert into t33 values(now, 1 ) + sql create table t34 (ts timestamp, i int) + sql insert into t34 values(now, 1 ) + sleep 1200 + + sql create table t41 (ts timestamp, i int) + sql insert into t41 values(now, 1 ) + sql create table t42 (ts timestamp, i int) + sql insert into t42 values(now, 1 ) + sql create table t43 (ts timestamp, i int) + sql insert into t43 values(now, 1 ) + sql create table t44 (ts timestamp, i int) + sql insert into t44 values(now, 1 ) + sleep 1200 + + sql create table t51 (ts timestamp, i int) + sql insert into t51 values(now, 1 ) + sql create table t52 (ts timestamp, i int) + sql insert into t52 values(now, 1 ) + sql create table t53 (ts timestamp, i int) + sql insert into t53 values(now, 1 ) + sql create table t54 (ts timestamp, i int) + sql insert into t54 values(now, 1 ) + sleep 1200 + + sql create table t61 (ts timestamp, i int) + sql insert into t61 values(now, 1 ) + sql create table t62 (ts timestamp, i int) + sql insert into t62 values(now, 1 ) + sql create table t63 (ts timestamp, i int) + sql insert into t63 values(now, 1 ) + sql create table t64 (ts timestamp, i int) + sql insert into t64 values(now, 1 ) + sleep 1200 + + sql create table t71 (ts timestamp, i int) + sql insert into t71 values(now, 1 ) + sql create table t72 (ts timestamp, i int) + sql insert into t72 values(now, 1 ) + sql create table t73 (ts timestamp, i int) + sql insert into t73 values(now, 1 ) + sql create table t74 (ts timestamp, i int) + sql insert into t74 values(now, 1 ) + sleep 1200 + + sql create table t81 (ts timestamp, i int) + sql insert into t81 values(now, 1 ) + sql create table t82 (ts timestamp, i int) + sql insert into t82 values(now, 1 ) + sql create table t83 (ts timestamp, i int) + sql insert into t83 values(now, 1 ) + sql create table t84 (ts timestamp, i int) + sql insert into t84 values(now, 1 ) + sleep 1200 + + sql show dnodes + print dnode1 openVnodes $data2_1 + print dnode2 openVnodes $data2_2 + if $data2_1 != 4 then + return -1 + endi + if $data2_2 != 4 then + return -1 + endi + + print ======== step2 $loop - print ======== step3 - sleep 3000 + system sh/exec_up.sh -n dnode2 -s stop + sleep 1000 + print ==> drop database $db + sql drop database $db + + print ======== step3 $loop + sleep 2000 system sh/exec_up.sh -n dnode2 -s start - sleep 20000 + sleep 15000 + + sql show dnodes + print dnode1 openVnodes $data2_1 $data4_1 + print dnode2 openVnodes $data2_2 $data4_2 + if $data2_1 != 0 then + return -1 + endi + if $data2_2 != 0 then + return -1 + endi + if $data4_1 != ready then + return -1 + endi + if $data4_2 != ready then + return -1 + endi print ===> test times : $loop - if $loop > 5 then + if $loop > 3 then return 0 endi $loop = $loop + 1 + + sql reset query cache + sleep 1000 goto begin diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index ac7e3f5c5c..9f66faab0a 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -47,10 +47,10 @@ sql create table d2.t2 (ts timestamp, i int) sql create table d3.t3 (ts timestamp, i int) sql create table d4.t4 (ts timestamp, i int) -sql insert into d1.t1 values(now, 1) -sql insert into d2.t2 values(now, 1) -sql insert into d3.t3 values(now, 1) -sql insert into d4.t4 values(now, 1) +sql insert into d1.t1 values(1589529000011, 1) +sql insert into d2.t2 values(1589529000021, 1) +sql insert into d3.t3 values(1589529000031, 1) +sql insert into d4.t4 values(1589529000041, 1) sql select * from d1.t1 if $rows != 1 then @@ -111,10 +111,10 @@ if $data2_3 != 4 then endi print ======== step4 -sql insert into d1.t1 values(now, 2) -sql insert into d2.t2 values(now, 2) -sql insert into d3.t3 values(now, 2) -sql insert into d4.t4 values(now, 2) +sql insert into d1.t1 values(1589529000012, 2) +sql insert into d2.t2 values(1589529000022, 2) +sql insert into d3.t3 values(1589529000032, 2) +sql insert into d4.t4 values(1589529000042, 2) sql select * from d1.t1 if $rows != 2 then @@ -142,10 +142,10 @@ sleep 1000 system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 5000 -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(1589529000013, 3) +sql insert into d2.t2 values(1589529000023, 3) +sql insert into d3.t3 values(1589529000033, 3) +sql insert into d4.t4 values(1589529000043, 3) sql select * from d1.t1 if $rows != 3 then @@ -173,27 +173,31 @@ sleep 5000 system sh/exec_up.sh -n dnode3 -s stop -x SIGINT sleep 3000 -sql insert into d1.t1 values(now, 4) -sql insert into d2.t2 values(now, 4) -sql insert into d3.t3 values(now, 4) -sql insert into d4.t4 values(now, 4) +sql insert into d1.t1 values(1589529000014, 4) +sql insert into d2.t2 values(1589529000024, 4) +sql insert into d3.t3 values(1589529000034, 4) +sql insert into d4.t4 values(1589529000044, 4) sql select * from d1.t1 +print select * from d1.t1 $rows if $rows != 4 then return -1 endi sql select * from d2.t2 +print select * from d2.t2 $rows if $rows != 4 then return -1 endi sql select * from d3.t3 +print select * from d3.t3 $rows if $rows != 4 then return -1 endi sql select * from d4.t4 +print select * from d4.t4 $rows if $rows != 4 then return -1 endi @@ -204,10 +208,10 @@ sleep 5000 system sh/exec_up.sh -n dnode4 -s stop -x SIGINT sleep 3000 -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) +sql insert into d1.t1 values(1589529000015, 5) +sql insert into d2.t2 values(1589529000025, 5) +sql insert into d3.t3 values(1589529000035, 5) +sql insert into d4.t4 values(1589529000045, 5) sql select * from d1.t1 if $rows != 5 then @@ -235,10 +239,10 @@ sleep 5000 system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sleep 3000 -sql insert into d1.t1 values(now, 6) -sql insert into d2.t2 values(now, 6) -sql insert into d3.t3 values(now, 6) -sql insert into d4.t4 values(now, 6) +sql insert into d1.t1 values(1589529000016, 6) +sql insert into d2.t2 values(1589529000026, 6) +sql insert into d3.t3 values(1589529000036, 6) +sql insert into d4.t4 values(1589529000046, 6) sql select * from d1.t1 if $rows != 6 then diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index f0ffb89015..76e3eaabbe 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -7,9 +7,9 @@ system sh/deploy.sh -n dnode3 -i 3 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode2 -c wallevel -v 2 system sh/cfg.sh -n dnode3 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numOfMPeers -v 2 -system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2 -system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2 +system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 +system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 system sh/cfg.sh -n dnode1 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 34d3310394..9a598e1704 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -53,7 +53,7 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -93,7 +93,7 @@ $x = 0 show4: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -103,7 +103,7 @@ print dnode2 openVnodes $data2_2 if $data2_1 != 2 then goto show4 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show4 endi if $rows != 1 then @@ -131,7 +131,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 0 then goto show5 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show5 endi if $data2_3 != 2 then @@ -155,7 +155,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 0 then return -1 endi -if $data2_2 != NULL then +if $data2_2 != null then return -1 endi if $data2_3 != 3 then @@ -170,7 +170,7 @@ $x = 0 show7: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -182,7 +182,7 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 0 then goto show7 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show7 endi if $data2_3 != 2 then @@ -210,7 +210,7 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 0 then return -1 endi -if $data2_2 != NULL then +if $data2_2 != null then return -1 endi if $data2_3 != 2 then @@ -227,7 +227,7 @@ $x = 0 show9: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -240,10 +240,10 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 0 then goto show9 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show9 endi -if $data2_3 != NULL then +if $data2_3 != null then goto show9 endi if $data2_4 != 4 then diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 9786a854a5..f039579012 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -65,7 +65,7 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -76,7 +76,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 2 then goto show2 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show2 endi if $data2_3 != 2 then @@ -105,7 +105,7 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 0 then goto show3 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show3 endi if $data2_3 != 2 then @@ -132,7 +132,7 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 0 then return -1 endi -if $data2_2 != NULL then +if $data2_2 != null then return -1 endi if $data2_3 != 3 then @@ -150,7 +150,7 @@ $x = 0 show5: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -163,7 +163,7 @@ print dnode5 openVnodes $data2_5 if $data2_1 != 0 then goto show5 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show5 endi if $data2_3 != 2 then @@ -183,7 +183,7 @@ $x = 0 show6: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -196,10 +196,10 @@ print dnode5 openVnodes $data2_5 if $data2_1 != 0 then goto show6 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show6 endi -if $data2_3 != NULL then +if $data2_3 != null then goto show6 endi if $data2_4 != 3 then diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index 6d5bbc77a5..acb0d033d4 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -88,7 +88,7 @@ print dnode4 openVnodes $data2_4 if $data2_1 != 2 then goto show2 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show2 endi if $data2_3 != 2 then @@ -122,7 +122,7 @@ print dnode5 openVnodes $data2_5 if $data2_1 != 0 then goto show3 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show3 endi if $data2_3 != 2 then @@ -162,7 +162,7 @@ print dnode5 openVnodes $data2_5 if $data2_1 != 0 then goto show4 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show4 endi if $data2_3 != 3 then @@ -228,7 +228,7 @@ endi if $data2_6 != 3 then goto show6 endi -if $data2_3 != NULL then +if $data2_3 != null then goto show6 endi if $data2_4 != 3 then diff --git a/tests/script/unique/dnode/balancex.sim b/tests/script/unique/dnode/balancex.sim index 4b46db4b49..202c9b5396 100644 --- a/tests/script/unique/dnode/balancex.sim +++ b/tests/script/unique/dnode/balancex.sim @@ -50,7 +50,7 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -77,7 +77,7 @@ $x = 0 show3: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes @@ -122,7 +122,7 @@ $x = 0 show5: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes @@ -132,7 +132,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show5 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show5 endi if $data2_3 != 3 then diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim index 4d67b5f55c..5e4ab65be3 100644 --- a/tests/script/unique/dnode/offline1.sim +++ b/tests/script/unique/dnode/offline1.sim @@ -63,7 +63,7 @@ print dnode1 $data4_2 if $data4_1 != ready then return -1 endi -if $data4_2 != NULL then +if $data4_2 != null then return -1 endi diff --git a/tests/script/unique/dnode/offline2.sim b/tests/script/unique/dnode/offline2.sim index 6aa85465dd..c526e45b6e 100644 --- a/tests/script/unique/dnode/offline2.sim +++ b/tests/script/unique/dnode/offline2.sim @@ -82,15 +82,18 @@ $x = 0 show4: $x = $x + 1 sleep 5000 - if $x == 50 then + if $x == 10 then return -1 endi sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 if $data4_1 != ready then goto show4 endi -if $data4_2 != NULL then +if $data4_2 != null then goto show4 endi if $data4_3 != ready then diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 545c28a4ea..6b23014b03 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -17,8 +17,8 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1 print ========== step1 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect sql create database d1 maxTables 4 sql create table d1.t1 (t timestamp, i int) @@ -59,17 +59,17 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 -if $data2_1 != 3 then +if $data2_1 != 1 then goto show2 endi -if $data2_2 != 1 then +if $data2_2 != 3 then goto show2 endi @@ -81,7 +81,7 @@ $x = 0 show3: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -97,7 +97,7 @@ $x = 0 show4: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -105,7 +105,7 @@ sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_2 != NULL then +if $data2_2 != null then goto show4 endi @@ -118,8 +118,8 @@ system sh/exec_up.sh -n dnode4 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 2000 + if $x == 10 then return -1 endi sql show dnodes @@ -127,10 +127,10 @@ print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 print dnode4 openVnodes $data2_4 -if $data2_1 != 4 then +if $data2_1 != 0 then goto show5 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show5 endi if $data2_3 != 2 then diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 972b77a35d..77ec1fa630 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -10,15 +10,15 @@ system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mgmtEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode1 -c wallevel -v 1 -system sh/cfg.sh -n dnode2 -c wallevel -v 1 -system sh/cfg.sh -n dnode3 -c wallevel -v 1 -system sh/cfg.sh -n dnode4 -c wallevel -v 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 print ========== step1 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect sql create database d1 maxTables 4 sql create table d1.t1 (t timestamp, i int) @@ -59,46 +59,51 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 -if $data2_1 != 3 then +if $data2_1 != 1 then goto show2 endi -if $data2_2 != 1 then +if $data2_2 != 3 then goto show2 endi print ========== step3 system sh/exec_up.sh -n dnode2 -s stop -x SIGINT sql drop dnode $hostname2 -sleep 7001 +sleep 4000 -$x = 0 -show3: - $x = $x + 1 - sleep 2000 - if $x == 30 then - return -1 - endi - sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 $data5_192.168.0.2 +print dnode2 openVnodes $data2_2 print ========== step4 sql create dnode $hostname3 system sh/exec_up.sh -n dnode3 -s start +sleep 5000 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +if $data2_3 != 0 then + return -1 +endi + +print ============ step 4.1 +system sh/exec_up.sh -n dnode2 -s start + $x = 0 show4: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi @@ -106,13 +111,13 @@ sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_2 != NULL then +if $data2_2 != null then goto show4 endi -if $data2_1 != 3 then +if $data2_1 != 1 then goto show4 endi -if $data2_3 != 1 then +if $data2_3 != 3 then goto show4 endi diff --git a/tests/script/unique/dnode/vnode_clean.sim b/tests/script/unique/dnode/vnode_clean.sim index da34c7bc9b..d46e1a751e 100644 --- a/tests/script/unique/dnode/vnode_clean.sim +++ b/tests/script/unique/dnode/vnode_clean.sim @@ -29,7 +29,7 @@ sql insert into d1.t1 values(now+5s, 11) sql show dnodes print dnode1 openVnodes $data2_1 -if $data2_1 != 3 then +if $data2_1 != 1 then return -1 endi @@ -41,16 +41,16 @@ $x = 0 show2: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 -if $data2_1 != 4 then +if $data2_1 != 0 then goto show2 endi -if $data2_2 != 3 then +if $data2_2 != 1 then goto show2 endi @@ -68,7 +68,7 @@ $x = 0 sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 -if $data2_1 != 4 then +if $data2_1 != 0 then return -1 endi if $data2_2 != 2 then @@ -82,7 +82,7 @@ $x = 0 show4: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes @@ -91,7 +91,7 @@ print dnode2 openVnodes $data2_2 if $data2_1 != 2 then goto show4 endi -if $data2_2 != NULL then +if $data2_2 != null then goto show4 endi if $rows != 1 then @@ -102,13 +102,8 @@ system sh/exec_up.sh -n dnode2 -s stop -x SIGINT print ========== step5 sleep 2000 -sql create dnode $hostname2 -system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c wallevel -v 1 -system sh/exec_up.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec_up.sh -n dnode3 -s start $x = 0 show5: @@ -119,11 +114,11 @@ show5: endi sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 -if $data2_1 != 4 then +print dnode3 openVnodes $data2_3 +if $data2_1 != 0 then goto show5 endi -if $data2_2 != 2 then +if $data2_3 != 2 then goto show5 endi @@ -138,17 +133,17 @@ sql insert into d3.t3 values(now+5s, 31) sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 -if $data2_1 != 4 then +print dnode2 openVnodes $data2_3 +if $data2_1 != 0 then return -1 endi -if $data2_2 != 1 then +if $data2_3 != 3 then return -1 endi print ========== step7 -sql create dnode $hostname3 -system sh/exec_up.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec_up.sh -n dnode4 -s start $x = 0 show7: @@ -160,15 +155,15 @@ show7: sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_1 != 4 then +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then goto show7 endi -if $data2_2 != 2 then +if $data2_3 != 2 then goto show7 endi -if $data2_3 != 3 then +if $data2_4 != 1 then goto show7 endi @@ -185,49 +180,49 @@ $x = 0 show8: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_1 != 4 then - goto show8 -endi -if $data2_2 != 2 then +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then goto show8 endi if $data2_3 != 2 then goto show8 endi +if $data2_4 != 2 then + goto show8 +endi print ========== step9 -sql drop dnode $hostname2 +sql drop dnode $hostname3 $x = 0 show9: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes print dnode1 openVnodes $data2_1 -print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_1 != 4 then +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then goto show9 endi -if $data2_2 != NULL then +if $data2_3 != null then goto show9 endi -if $data2_3 != 0 then +if $data2_4 != 4 then goto show9 endi -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode3 -s stop -x SIGINT print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 914838361a..4e126c4e60 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -1,7 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 +system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec_up.sh -n dnode1 -s start sleep 3000 @@ -12,7 +14,7 @@ print ============================ dnode1 start print =============== step1 - parse system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be NULL"}@ then +if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then return -1 endi @@ -24,7 +26,7 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/ print $system_content -if $system_content != @{"status":"error","code":1057,"desc":"database name can not be NULL"}@ then +if $system_content != @{"status":"error","code":1057,"desc":"database name can not be null"}@ then return -1 endi @@ -73,7 +75,7 @@ endi system_content curl -u root:taosdata -d '[{"metric": "ab1234567890123456789012345678ab1234567890123456789012345678","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1065,"desc":"metric name length can not more than 22"}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"ab1234567890123456789012345678ab1234567890123456789012345678","stable":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb","table":"ab1234567890123456789012345678ab1234567890123456789012345678_d_bbb_lga_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"status":"error","code":-2147483389}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi @@ -123,13 +125,13 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": "web01","group1": "1","group1": "1","group1": "1","group1": "1","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1071,"desc":"tags size too long"}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbbbbbb","table":"sys_cpu_d_bbbbbbb_lga_1_1_1_1_1_web01","timestamp":1346846400,"value":18.000000,"tags":{"dc":"lga","group1":"1","group1":"1","group1":"1","group1":"1","group1":"1","host":"web01"},"status":"error","code":-2147483445}}],"failed":1,"success":0,"affected_rows":0}@ then return -1 endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"": "web01"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1073,"desc":"tag name is NULL"}@ then +if $system_content != @{"status":"error","code":1073,"desc":"tag name is null"}@ then return -1 endi @@ -147,7 +149,7 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400,"value": 18,"tags": {"host": ""}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"status":"error","code":1076,"desc":"tag value is NULL"}@ then +if $system_content != @{"status":"error","code":1076,"desc":"tag value is null"}@ then return -1 endi @@ -162,7 +164,7 @@ endi system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:6020/opentsdb/db/put print $system_content -if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846400000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":0,"status":"succ"}}],"failed":0,"success":1,"affected_rows":0}@ then +if $system_content != @{"errors":[{"datapoint":{"metric":"sys_cpu","stable":"sys_cpu_d_bbb","table":"sys_cpu_d_bbb_lga_1_web01","timestamp":1346846400000,"value":18.000000,"tags":{"dc":"lga","group1":"1","host":"web01"},"affected_rows":1,"status":"succ"}}],"failed":0,"success":1,"affected_rows":1}@ then return -1 endi diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index e4e1235e66..37e38fbf66 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -8,7 +8,7 @@ system sh/cfg.sh -n dnode2 -c numOfMPeers -v 2 system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2 print ============== step1 -system sh/exec_up.sh -n dnode1 -s start -t +system sh/exec_up.sh -n dnode1 -s start sleep 3000 sql connect @@ -20,7 +20,7 @@ if $data2_1 != master then endi print ============== step2 -system sh/exec_up.sh -n dnode2 -s start -t +system sh/exec_up.sh -n dnode2 -s start sql create dnode $hostname2 $x = 0 @@ -41,17 +41,6 @@ if $data2_2 != slave then goto show2 endi -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -system sh/exec_up.sh -n dnode5 -s stop -x SIGINT -system sh/exec_up.sh -n dnode6 -s stop -x SIGINT -system sh/exec_up.sh -n dnode7 -s stop -x SIGINT -system sh/exec_up.sh -n dnode8 -s stop -x SIGINT - -return - print ============== step3 sql_error drop dnode $hostname1 -x error1 print should not drop master diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 56a4b30573..625e42a334 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -9,8 +9,8 @@ system sh/cfg.sh -n dnode3 -c numOfMPeers -v 2 print ============== step1 system sh/exec_up.sh -n dnode1 -s start -sql connect sleep 3000 +sql connect sql show mnodes $dnode1Role = $data2_1 @@ -23,16 +23,15 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $dnode2Role != NULL then +if $dnode2Role != null then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi print ============== step2 sql create dnode $hostname2 -sleep 1700 sql create dnode $hostname3 print ============== step3 @@ -68,10 +67,10 @@ print dnode3 ==> $dnode3Role if $dnode1Role != master then return -1 endi -if $rows != 2 then +if $dnode2Role != slave then return -1 endi -if $dnode3Role != NULL then +if $dnode3Role != null then return -1 endi diff --git a/tests/script/unique/mnode/secondIp.sim b/tests/script/unique/mnode/secondIp.sim deleted file mode 100644 index cfe75ffc84..0000000000 --- a/tests/script/unique/mnode/secondIp.sim +++ /dev/null @@ -1,44 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 - -print ========== step1 dnode2 start -system sh/exec_up.sh -n dnode2 -s start -sql connect - -print ========== step2 connect to dnode2 -sql create dnode $hostname1 -system sh/exec_up.sh -n dnode1 -s start -sleep 3000 - -print ========== step3 -sql show dnodes -print dnode1 openvnodes $data3_1 -print dnode2 openvnodes $data3_2 -print dnode1 totalvnodes $data4_1 -print dnode2 totalvnodes $data4_2 - -if $rows != 2 then - return -1 -endi -if $data3_1 != 0 then - return -1 -endi -if $data3_2 != 0 then - return -1 -endi -if $data4_1 != 4 then - return -1 -endi -if $data4_2 != 4 then - return -1 -endi - -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -system sh/exec_up.sh -n dnode5 -s stop -x SIGINT -system sh/exec_up.sh -n dnode6 -s stop -x SIGINT -system sh/exec_up.sh -n dnode7 -s stop -x SIGINT -system sh/exec_up.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/testSuite.sim b/tests/script/unique/mnode/testSuite.sim index 34d3ce7e53..33df24b860 100644 --- a/tests/script/unique/mnode/testSuite.sim +++ b/tests/script/unique/mnode/testSuite.sim @@ -5,5 +5,4 @@ run unique/mnode/mgmt25.sim run unique/mnode/mgmt26.sim run unique/mnode/mgmt33.sim run unique/mnode/mgmt34.sim -#run unique/mnode/mgmtr2.sim -#run unique/mnode/secondIp.sim +run unique/mnode/mgmtr2.sim diff --git a/tests/script/unique/metrics/balance_replica1.sim b/tests/script/unique/stable/balance_replica1.sim similarity index 97% rename from tests/script/unique/metrics/balance_replica1.sim rename to tests/script/unique/stable/balance_replica1.sim index 52bf558faa..8cf41319a0 100644 --- a/tests/script/unique/metrics/balance_replica1.sim +++ b/tests/script/unique/stable/balance_replica1.sim @@ -1,7 +1,4 @@ system sh/stop_dnodes.sh - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 @@ -26,7 +23,7 @@ $totalNum = 200 print ============== step1 print ========= start dnode1 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect $i = 0 @@ -81,7 +78,7 @@ if $dnode2Vnodes != NULL then endi print =============== step3 start dnode2 sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 8000 $x = 0 diff --git a/tests/script/unique/metrics/dnode2.sim b/tests/script/unique/stable/dnode2.sim similarity index 97% rename from tests/script/unique/metrics/dnode2.sim rename to tests/script/unique/stable/dnode2.sim index cd76b39ba0..2d894c6542 100644 --- a/tests/script/unique/metrics/dnode2.sim +++ b/tests/script/unique/stable/dnode2.sim @@ -1,7 +1,4 @@ system sh/stop_dnodes.sh - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c walLevel -v 0 @@ -10,12 +7,12 @@ system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start $x = 0 createDnode: @@ -25,7 +22,7 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi diff --git a/tests/script/unique/metrics/dnode2_stop.sim b/tests/script/unique/stable/dnode2_stop.sim similarity index 93% rename from tests/script/unique/metrics/dnode2_stop.sim rename to tests/script/unique/stable/dnode2_stop.sim index 996482d11e..55ad5fcf09 100644 --- a/tests/script/unique/metrics/dnode2_stop.sim +++ b/tests/script/unique/stable/dnode2_stop.sim @@ -1,7 +1,4 @@ system sh/stop_dnodes.sh - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c walLevel -v 0 @@ -10,11 +7,11 @@ system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start $x = 0 createDnode: @@ -24,7 +21,7 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi @@ -76,7 +73,7 @@ endi sleep 100 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec_up.sh -n dnode2 -s stop -x SIGINT print =============== step2 sql select count(*) from $mt -x step2 @@ -87,7 +84,7 @@ sql select count(tbcol) from $mt -x step21 return -1 step21: -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sleep 10000 print =============== step3 diff --git a/tests/script/unique/metrics/dnode3.sim b/tests/script/unique/stable/dnode3.sim similarity index 93% rename from tests/script/unique/metrics/dnode3.sim rename to tests/script/unique/stable/dnode3.sim index 3a5419dff4..c9dd31f6f8 100644 --- a/tests/script/unique/metrics/dnode3.sim +++ b/tests/script/unique/stable/dnode3.sim @@ -1,8 +1,4 @@ system sh/stop_dnodes.sh - - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 @@ -15,14 +11,14 @@ system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start sql create dnode $hostname3 -system sh/exec.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode3 -s start $x = 0 createDnode: @@ -32,10 +28,10 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi -if $data4_192.168.0.3 == offline then +if $data4_3 == offline then goto createDnode endi @@ -81,11 +77,11 @@ if $rows != 3 then endi sql show dnodes -$dnode1Vnodes = $data3_192.168.0.1 +$dnode1Vnodes = $data2_1 print dnode1 $dnode1Vnodes -$dnode2Vnodes = $data3_192.168.0.2 +$dnode2Vnodes = $data2_2 print dnode2 $dnode2Vnodes -$dnode3Vnodes = $data3_192.168.0.3 +$dnode3Vnodes = $data2_3 print dnode3 $dnode3Vnodes if $dnode1Vnodes != 3 then diff --git a/tests/script/unique/metrics/replica2_dnode4.sim b/tests/script/unique/stable/replica2_dnode4.sim similarity index 94% rename from tests/script/unique/metrics/replica2_dnode4.sim rename to tests/script/unique/stable/replica2_dnode4.sim index bedc895dd1..f204e05949 100644 --- a/tests/script/unique/metrics/replica2_dnode4.sim +++ b/tests/script/unique/stable/replica2_dnode4.sim @@ -1,9 +1,4 @@ system sh/stop_dnodes.sh - - - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 @@ -20,16 +15,16 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode4 -s start $x = 0 createDnode: @@ -39,13 +34,13 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi -if $data4_192.168.0.3 == offline then +if $data4_3 == offline then goto createDnode endi -if $data4_192.168.0.4 == offline then +if $data4_4 == offline then goto createDnode endi diff --git a/tests/script/unique/metrics/replica2_vnode3.sim b/tests/script/unique/stable/replica2_vnode3.sim similarity index 97% rename from tests/script/unique/metrics/replica2_vnode3.sim rename to tests/script/unique/stable/replica2_vnode3.sim index 9a1d3477be..238e2e4aee 100644 --- a/tests/script/unique/metrics/replica2_vnode3.sim +++ b/tests/script/unique/stable/replica2_vnode3.sim @@ -1,7 +1,4 @@ system sh/stop_dnodes.sh - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c walLevel -v 0 @@ -10,11 +7,11 @@ system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode2 -s start $x = 0 createDnode: @@ -24,7 +21,7 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi diff --git a/tests/script/unique/metrics/replica3_dnode6.sim b/tests/script/unique/stable/replica3_dnode6.sim similarity index 92% rename from tests/script/unique/metrics/replica3_dnode6.sim rename to tests/script/unique/stable/replica3_dnode6.sim index 135e594cdc..1c8a8ae10e 100644 --- a/tests/script/unique/metrics/replica3_dnode6.sim +++ b/tests/script/unique/stable/replica3_dnode6.sim @@ -1,12 +1,4 @@ system sh/stop_dnodes.sh - - - - - - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 @@ -35,7 +27,7 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode6 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 @@ -43,11 +35,11 @@ sql create dnode $hostname3 sql create dnode $hostname4 sql create dnode $hostname5 sql create dnode $hostname6 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start -system sh/exec.sh -n dnode5 -s start -system sh/exec.sh -n dnode6 -s start +system sh/exec_up.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode5 -s start +system sh/exec_up.sh -n dnode6 -s start $x = 0 createDnode: @@ -57,19 +49,19 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi -if $data4_192.168.0.3 == offline then +if $data4_3 == offline then goto createDnode endi -if $data4_192.168.0.4 == offline then +if $data4_4 == offline then goto createDnode endi -if $data4_192.168.0.5 == offline then +if $data4_5 == offline then goto createDnode endi -if $data4_192.168.0.6 == offline then +if $data4_6 == offline then goto createDnode endi diff --git a/tests/script/unique/metrics/replica3_vnode3.sim b/tests/script/unique/stable/replica3_vnode3.sim similarity index 94% rename from tests/script/unique/metrics/replica3_vnode3.sim rename to tests/script/unique/stable/replica3_vnode3.sim index ca147c1ef5..75870af4c4 100644 --- a/tests/script/unique/metrics/replica3_vnode3.sim +++ b/tests/script/unique/stable/replica3_vnode3.sim @@ -1,9 +1,5 @@ system sh/stop_dnodes.sh - - - - system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 @@ -20,16 +16,16 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start +system sh/exec_up.sh -n dnode2 -s start +system sh/exec_up.sh -n dnode3 -s start +system sh/exec_up.sh -n dnode4 -s start $x = 0 createDnode: $x = $x + 1 @@ -38,13 +34,13 @@ createDnode: return -1 endi sql show dnodes; -if $data4_192.168.0.2 == offline then +if $data4_2 == offline then goto createDnode endi -if $data4_192.168.0.3 == offline then +if $data4_3 == offline then goto createDnode endi -if $data4_192.168.0.4 == offline then +if $data4_4 == offline then goto createDnode endi @@ -144,6 +140,7 @@ if $rows != 5 then endi print =============== step7 +print select count(*) from $mt sql select count(*) from $mt print ===> $data00 if $data00 != $totalNum then @@ -208,7 +205,7 @@ endi if $rows != 50 then return -1 endi -return + print =============== clear sql drop database $db sql show databases diff --git a/tests/script/unique/metrics/testSuite.sim b/tests/script/unique/stable/testSuite.sim similarity index 100% rename from tests/script/unique/metrics/testSuite.sim rename to tests/script/unique/stable/testSuite.sim diff --git a/tests/script/unique/table/back_insert.sim b/tests/script/unique/table/back_insert.sim deleted file mode 100644 index 43831cca95..0000000000 --- a/tests/script/unique/table/back_insert.sim +++ /dev/null @@ -1,7 +0,0 @@ -sql connect -$x = 1 -begin: - sql insert into db.tb values(now, $x ) -x begin - #print ===> insert successed $x - $x = $x + 1 -goto begin \ No newline at end of file diff --git a/tests/script/unique/table/delete_part.sim b/tests/script/unique/table/delete_part.sim deleted file mode 100644 index 04cb03598c..0000000000 --- a/tests/script/unique/table/delete_part.sim +++ /dev/null @@ -1,85 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 - -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 - -system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1 -system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1 - -system sh/cfg.sh -n dnode1 -c mgmtEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode2 -c mgmtEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode3 -c mgmtEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode4 -c mgmtEqualVnodeNum -v 4 - -system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4 -system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4 - -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 - -print ========= start dnodes -system sh/exec.sh -n dnode1 -s start -sql connect -sql create dnode $hostname2 -system sh/exec.sh -n dnode2 -s start -sleep 3000 - -sql create database dpdb -sql use dpdb - -$loop = 0 -begin: - - print ======== step1 - - $x = 0 - while $x < 16 - $tb = tb . $x - sql create table $tb (ts timestamp, i int) - sql insert into $tb values(now, $x ) - $x = $x + 1 - endw - - print ======== step2 - system sh/exec.sh -n dnode2 -s stop - $x = 0 - while $x < 16 - $tb = tb . $x - sql drop table $tb - $x = $x + 1 - endw - - print ======== step3 - sleep 2000 - system sh/exec.sh -n dnode2 -s start - sleep 3000 - - print ===> test times : $loop - if $loop > 20 then - return 0 - endi - - $loop = $loop + 1 - -goto begin - -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -system sh/exec_up.sh -n dnode5 -s stop -x SIGINT -system sh/exec_up.sh -n dnode6 -s stop -x SIGINT -system sh/exec_up.sh -n dnode7 -s stop -x SIGINT -system sh/exec_up.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/table/testSuite.sim b/tests/script/unique/table/testSuite.sim deleted file mode 100644 index e0c0a1f2fa..0000000000 --- a/tests/script/unique/table/testSuite.sim +++ /dev/null @@ -1 +0,0 @@ -run unique/table/delete_part.sim diff --git a/tests/script/unique/testSuite.sim b/tests/script/unique/testSuite.sim deleted file mode 100644 index 374f9b7965..0000000000 --- a/tests/script/unique/testSuite.sim +++ /dev/null @@ -1,3 +0,0 @@ -################################# -run unique/mnode/testSuite.sim -################################## diff --git a/tests/script/unique/vnode/commit.sim b/tests/script/unique/vnode/commit.sim deleted file mode 100644 index 29c9f72335..0000000000 --- a/tests/script/unique/vnode/commit.sim +++ /dev/null @@ -1,158 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode2 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numofMpeers -v 3 -system sh/cfg.sh -n dnode2 -c numofMpeers -v 3 -system sh/exec_up.sh -n dnode1 -s start - -sql connect -sql create dnode $hostname2 -system sh/exec_up.sh -n dnode2 -s start -sleep 3000 - -print =================== step 1 create db -sql create database c2db replica 2 days 10 keep 50 -sql use c2db -sql create table tb (ts timestamp, speed int) -sql insert into tb values(now, 0) - -print =================== step2 sleep 2000 and kill dnode2(SIGINT) -sleep 2000 -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -sleep 1000 - -print =================== step3 insert into dnode1 - -$x = 1 -while $x < 100 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 240 -while $x < 400 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 480 -while $x < 700 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 720 -while $x < 809 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 960 -while $x < 1043 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 1200 -while $x < 1244 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 1440 -while $x < 1677 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -$x = 1680 -while $x < 1683 - $time = $x . m - sql insert into tb values (now + $time , $x ) - $x = $x + 1 -endw - -print =================== step4 -sql select count(*) from tb -print select count(*) from tb ==> $data00 (expect 936) -if $data00 != 936 then - return -1 -endi -sql select * from tb order by ts desc -print select * from tb ==> $data00 $data01 $rows -if $data01 != 1682 then - return -1 -endi -if $rows != 936 then - return -1 -endi - -print =================== step5 sleep kill dnode1(SIGINT) then start dnode1 -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -system sh/exec_up.sh -n dnode1 -s start - -sleep 3000 -print =================== step6 start dnode2 and sleep 10000 (wait sync complete) -system sh/exec_up.sh -n dnode2 -s start -sleep 12000 - -print =================== step7 -sql_error insert into tb values(now + 1000h, 100) -sql select count(*) from tb order by ts desc -print select count(*) from tb ==> $data00 (expect <= 936) -if $data00 != 936 then - return -1 -endi -$remainRows = $data00 - -sql select * from tb order by ts desc -print select * from tb ==> $data00 $data01 $data10 $data11 $rows - -if $data11 != 1681 then - return -1 -endi -if $rows != $remainRows then - return -1 -endi - -print =================== step8 kill dnode1(SIGINT) and query -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -sleep 2000 - -print =================== step9 -sql select count(*) from tb order by ts desc -print select count(*) from tb ==> $data00 (expect == $remainRows ) -if $data00 > $remainRows then - return -1 -endi -if $data00 <= 0 then - return -1 -endi - -$remainRows = $data00 -sql select * from tb order by ts desc -print select * from tb ==> $data00 $data01 $rows - -if $rows != $remainRows then - return -1 -endi - -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -system sh/exec_up.sh -n dnode5 -s stop -x SIGINT -system sh/exec_up.sh -n dnode6 -s stop -x SIGINT -system sh/exec_up.sh -n dnode7 -s stop -x SIGINT -system sh/exec_up.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 0504207c2e..bb3e8813bd 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -53,27 +53,27 @@ sql select count(*) from db4.tb4 $lastRows4 = $rows print ======== step2 -run_back cluster/vnode/back_insert_many.sim +run_back unique/vnode/back_insert_many.sim sleep 5000 print ======== step3 system sh/exec_up.sh -n dnode2 -s stop -sleep 10000 +sleep 5000 $x = 0 loop: print ======== step4 system sh/exec_up.sh -n dnode2 -s start -sleep 10000 +sleep 5000 system sh/exec_up.sh -n dnode3 -s stop -sleep 10000 +sleep 5000 print ======== step5 system sh/exec_up.sh -n dnode3 -s start -sleep 10000 +sleep 5000 system sh/exec_up.sh -n dnode2 -s stop -sleep 10000 +sleep 5000 print ======== step6 sql select count(*) from db1.tb1 @@ -108,7 +108,7 @@ print ======== step7 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica2_basic.sim b/tests/script/unique/vnode/replica2_basic.sim deleted file mode 100644 index a0ea7085cb..0000000000 --- a/tests/script/unique/vnode/replica2_basic.sim +++ /dev/null @@ -1,199 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode2 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numofMpeers -v 3 -system sh/cfg.sh -n dnode2 -c numofMpeers -v 3 -system sh/exec_up.sh -n dnode1 -s start - -sql connect -sql create dnode $hostname2 -system sh/exec_up.sh -n dnode2 -s start -sleep 3000 - -$N = 10 -$db = d1 -$table = table_r2 - -print =================== step 1 -sql create database $db replica 3 -sql use $db -sql create table $table (ts timestamp, speed int) -x error_create -return -1 -error_create: -sql drop database $db - -print =================== step 2 - -sql create database $db replica 2 -sql use $db - -sql create table $table (ts timestamp, speed int) -sleep 1000 - -print =================== step 3 -$x = 1 -$y = $x + $N -$expect = $N -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 4 -system sh/exec_up.sh -n dnode2 -s stop -sleep 2000 -$y = $x + $N -$expect = $N * 2 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 5 -system sh/exec_up.sh -n dnode2 -s start -sleep 2000 -$y = $x + $N -$expect = $N * 3 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 6 -system sh/exec_up.sh -n dnode1 -s stop -sleep 2000 -$y = $x + $N -$expect = $N * 4 -while $x < $y -$ms = $x . m -sql insert into $table values (now + $ms , $x ) -$x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then -return -1 -endi - -print =================== step 7 -system sh/exec_up.sh -n dnode1 -s start -sleep 2000 -$y = $x + $N -$expect = $N * 5 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 8 -system sh/ip.sh -i 1 -s down - -sleep 2000 -$y = $x + $N -$expect = $N * 6 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 9 - -sleep 2000 -$y = $x + $N -$expect = $N * 7 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 10 -system sh/ip.sh -i 2 -s down - -sleep 2000 -$y = $x + $N -$expect = $N * 8 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 11 - -sleep 2000 -$y = $x + $N -$expect = $N * 9 -while $x < $y - $ms = $x . m - sql insert into $table values (now + $ms , $x ) - $x = $x + 1 -endw - -sql select * from $table -print sql select * from $table -> $rows points -if $rows != $expect then - return -1 -endi - -print =================== step 12 - -system sh/exec_up.sh -n dnode1 -s stop -x SIGINT -system sh/exec_up.sh -n dnode2 -s stop -x SIGINT -system sh/exec_up.sh -n dnode3 -s stop -x SIGINT -system sh/exec_up.sh -n dnode4 -s stop -x SIGINT -system sh/exec_up.sh -n dnode5 -s stop -x SIGINT -system sh/exec_up.sh -n dnode6 -s stop -x SIGINT -system sh/exec_up.sh -n dnode7 -s stop -x SIGINT -system sh/exec_up.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index f7da1babb7..a6bd226484 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -33,7 +33,7 @@ sql select count(*) from db.tb $lastRows = $rows print ======== step2 -run_back cluster/vnode/back_insert.sim +run_back unique/vnode/back_insert.sim sleep 3000 print ======== step3 @@ -66,7 +66,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 8c3ed902fb..2f311a5d7a 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -36,7 +36,7 @@ sql select count(*) from db.tb $lastRows = $rows print ======== step2 -run_back cluster/vnode/back_insert.sim +run_back unique/vnode/back_insert.sim sleep 3000 print ======== step3 @@ -75,7 +75,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_vgroup.sim b/tests/script/unique/vnode/replica3_vgroup.sim index f63bf1783d..6315a4335c 100644 --- a/tests/script/unique/vnode/replica3_vgroup.sim +++ b/tests/script/unique/vnode/replica3_vgroup.sim @@ -33,7 +33,7 @@ sleep 3001 $tbPre = m -$N = 280 +$N = 300 $x = 0 $y = $x + $N while $x < $y @@ -46,20 +46,20 @@ endw #print =================== step 2 -#$x = 1 -#$y = $x + $N -#$expect = $N -#while $x < $y -# $ms = $x . m -# sql insert into $table values (now + $ms , $x ) -# $x = $x + 1 -#endw +$x = -500 +$y = $x + $N +while $x < $y + $ms = $x . m + sql insert into $table values (now $ms , $x ) + $x = $x + 1 +endw -#sql select * from $table -#print sql select * from $table -> $rows points -#if $rows != $expect then -# return -1 -#endi +$expect = $N + 1 +sql select * from $table +print sql select * from $table -> $rows points expect $expect +if $rows != $expect then + return -1 +endi system sh/exec_up.sh -n dnode1 -s stop -x SIGINT system sh/exec_up.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/vnode/testSuite.sim b/tests/script/unique/vnode/testSuite.sim index 46b01aaa45..3a9db66beb 100644 --- a/tests/script/unique/vnode/testSuite.sim +++ b/tests/script/unique/vnode/testSuite.sim @@ -1,6 +1,4 @@ -run unique/vnode/commit.sim run unique/vnode/many.sim -run unique/vnode/replica2_basic.sim run unique/vnode/replica2_basic2.sim run unique/vnode/replica2_repeat.sim run unique/vnode/replica3_basic.sim