From 5f132980e81b19ffb4c48049f000dc5dc87c01af Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 7 May 2020 10:46:42 +0800 Subject: [PATCH 1/5] if sched is not ready, will cause a deadlock --- src/mnode/src/mgmtDServer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index 726554e490..1e820be0e9 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -52,14 +52,14 @@ int32_t mgmtInitDServer() { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = mgmtDServerRetrieveAuth; + tsMgmtDServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS"); + tsMgmtDServerRpc = rpcOpen(&rpcInit); if (tsMgmtDServerRpc == NULL) { mError("failed to init server connection to dnode"); return -1; } - tsMgmtDServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS"); - mPrint("server connection to dnode is opened"); return 0; } From 32d2570c80d5e44f3e8f372b7c178c628573162f Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Thu, 7 May 2020 05:07:16 +0000 Subject: [PATCH 2/5] [TD-228] fix python connector to support binary interface change --- src/connector/python/linux/python3/taos/cinterface.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 77001609b6..6ef54f1ba5 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -146,6 +146,7 @@ class CTaosInterface(object): libtaos.taos_errstr.restype = ctypes.c_char_p libtaos.taos_subscribe.restype = ctypes.c_void_p libtaos.taos_consume.restype = ctypes.c_void_p + libtaos.taos_fetch_lengths.restype = ctypes.c_void_p def __init__(self, config=None): ''' @@ -314,6 +315,8 @@ class CTaosInterface(object): isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if data == None: @@ -323,7 +326,7 @@ class CTaosInterface(object): if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fields[i]['bytes'], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) From 6d9caf0e5cf2f9c6c071f8737a4ca0c31328a61a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 7 May 2020 13:52:16 +0800 Subject: [PATCH 3/5] add description regarding how to run and add test case. [TD-236] --- README.md | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/README.md b/README.md index 280cf73ba0..158ae040fa 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,251 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo - [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector) - [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector) +# How to run the test cases and how to add a new test case? + +### Prepare development environment + +1. sudo apt install + build-essential cmake net-tools python-pip python-setuptools python3-pip + python3-setuptools valgrind + +2. git clone ; cd TDengine + +3. mkdir debug; cd debug; cmake ..; make ; sudo make install + +4. pip install src/connector/python/linux/python2 ; pip3 install + src/connector/python/linux/python3 + +### How to run TSIM test suite + +1. cd \/tests/script + +2. sudo ./test.sh + +### How to run Python test suite + +1. cd \/tests/pytest + +2. ./smoketest.sh \# for smoke test + +3. ./smoketest.sh -g \# for memory leak detection test with valgrind + +4. ./fulltest.sh \# for full test + +> Note1: TDengine daemon's configuration and data files are stored in +> \/sim directory. As a historical design, it's same place with +> TSIM script. So after the TSIM script ran with sudo privilege, the directory +> has been used by TSIM then the python script cannot write it by a normal +> user. You need to remove the directory completely first before running the +> Python test case. We should consider using two different locations to store +> for TSIM and Python script. + +> Note2: if you need to debug crash problem with a core dump, you need +> manually edit smoketest.sh or fulltest.sh to add "ulimit -c unlimited" +> before the script line. Then you can look for the core file in +> \/tests/pytest after the program crash. + +### How to add a new test case + +**1. add a new TSIM test cases:** + +TSIM test cases are now included in the new development branch and can be +added to the TDengine/tests/script/test.sh script based on the manual test +methods necessary to add test cases as described above. + +**2. add a new Python test cases:** + +**2.1 Please refer to \/tests/pytest/insert/basic.py to add a new +test case.** The new test case must implement 3 functions, where self.init() +and self.stop() simply copy the contents of insert/basic.py and the test +logic is implemented in self.run(). You can refer to the code in the util +directory for more information. + +**2.2 Edit smoketest.sh to add the path and filename of the new test case** + +Note: The Python test framework may continue to be improved in the future, +hopefully, to provide more functionality and ease of writing test cases. The +method of writing the test case above does not exclude that it will also be +affected. + +**2.3 What test.py does in detail:** + +test.py is the entry program for test case execution and monitoring. + +test.py has the following functions. + +\-f --file, Specifies the test case file name to be executed +-p --path, Specifies deployment path + +\-m --master, Specifies the master server IP for cluster deployment +-c--cluster, test cluster function +-s--stop, terminates all running nodes + +\-g--valgrind, load valgrind for memory leak detection test + +\-h--help, display help + +**2.4 What util/log.py does in detail:** + +log.py is quite simple, the main thing is that you can print the output in +different colors as needed. The success() should be called for successful +test case execution and the success() will print green text. The exit() will +print red text and exit the program, exit() should be called for test +failure. + +**util/log.py** + +... + +    def info(self, info): + +        printf("%s %s" % (datetime.datetime.now(), info)) + +  + +    def sleep(self, sec): + +        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec)) + +        time.sleep(sec) + +  + +    def debug(self, err): + +        printf("\\033[1;36m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def success(self, info): + +        printf("\\033[1;32m%s %s\\033[0m" % (datetime.datetime.now(), info)) + +  + +    def notice(self, err): + +        printf("\\033[1;33m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +  + +    def exit(self, err): + +        printf("\\033[1;31m%s %s\\033[0m" % (datetime.datetime.now(), err)) + +        sys.exit(1) + +  + +    def printNoPrefix(self, info): + +        printf("\\033[1;36m%s\\033[0m" % (info) + +... + +**2.5 What util/sql.py does in detail:** + +SQL.py is mainly used to execute SQL statements to manipulate the database, +and the code is extracted and commented as follows: + +**util/sql.py** + +\# prepare() is mainly used to set up the environment for testing table and +data, and to set up the database db for testing. do not call prepare() if you +need to test the database operation command. + +def prepare(self): + +tdLog.info("prepare database:db") + +self.cursor.execute('reset query cache') + +self.cursor.execute('drop database if exists db') + +self.cursor.execute('create database db') + +self.cursor.execute('use db') + +... + +\# query() is mainly used to execute select statements for normal syntax input + +def query(self, sql): + +... + +\# error() is mainly used to execute the select statement with the wrong syntax +input, the error will be caught as a reasonable behavior, if not caught it will +prove that the test failed + +def error() + +... + +\# checkRows() is used to check the number of returned lines after calling +query(select ...) after calling the query(select ...) to check the number of +rows of returned results. + +def checkRows(self, expectRows): + +... + +\# checkData() is used to check the returned result data after calling +query(select ...) after the query(select ...) is called, failure to meet +expectation is + +def checkData(self, row, col, data): + +... + +\# getData() returns the result data after calling query(select ...) to return +the resulting data after calling query(select ...) + +def getData(self, row, col): + +... + +\# execute() used to execute sql and return the number of affected rows + +def execute(self, sql): + +... + +\# executeTimes() Multiple executions of the same sql statement + +def executeTimes(self, sql, times): + +... + +\# CheckAffectedRows() Check if the number of affected rows is as expected + +def checkAffectedRows(self, expectAffectedRows): + +... + +> Note: Both Python2 and Python3 are currently supported by the Python test +> case. Since Python2 is no longer officially supported by January 1, 2020, it +> is recommended that subsequent test case development be guaranteed to run +> correctly on Python3. For Python2, please consider being compatible if +> appropriate without additional +> burden.   + +### CI Covenant submission adoption principle. + +- Every commit / PR compilation must pass. Currently, the warning is treated + as an error, so the warning must also be resolved. + +- Test cases that already exist must pass. + +- Because CI is very important to support build and automatically test + procedure, it is necessary to manually test the test case before adding it + and do as many iterations as possible to ensure that the test case provides + stable and reliable test results when added. + +> Note: In the future, according to the requirements and test development +> progress will add stress testing, performance testing, code style, +> and other features based on functional testing. + ### Third Party Connectors The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them. From 8a4d27127a2898b72a29ef2c9361eb6af1205c3d Mon Sep 17 00:00:00 2001 From: chang Date: Thu, 7 May 2020 14:01:12 +0800 Subject: [PATCH 4/5] docs: fix wrong wrod "ater" to "alter" --- src/mnode/src/mgmtUser.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index b4dd58cb3b..ecd3c217ca 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -396,7 +396,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -439,13 +439,13 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } mgmtSendSimpleResp(pMsg->thandle, code); } else { - mError("user:%s, no rights to ater user", pOperUser->user); + mError("user:%s, no rights to alter user", pOperUser->user); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } From 9b3844ea91b7f744d0d8caac1fb320575e6f9e0e Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 7 May 2020 14:29:06 +0800 Subject: [PATCH 5/5] a dead lock while tsdb commit --- src/tsdb/src/tsdbFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 0c1b9e314e..b1228b0230 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -180,7 +180,7 @@ void tsdbFitRetention(STsdbRepo *pRepo) { int mfid = tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision); - while (pGroup[0].fileId < mfid) { + while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) { tsdbRemoveFileGroup(pFileH, pGroup[0].fileId); } }