diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md index 0be39ab8e4..343ce80422 100644 --- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md @@ -480,9 +480,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 - **LEASTSQUARES** ```mysql - SELECT LEASTSQUARES(field_name) FROM tb_name [WHERE clause] + SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause] ``` - 功能说明:统计表中某列的值是主键(时间戳)的拟合直线方程。 + 功能说明:统计表中某列的值是主键(时间戳)的拟合直线方程。start_val是自变量初始值,step_val是自变量的步长值。 返回结果数据类型:字符串表达式(斜率, 截距)。 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 说明:自变量是时间戳,因变量是该列的值。 diff --git a/documentation/webdocs/markdowndocs/TAOS SQL.md b/documentation/webdocs/markdowndocs/TAOS SQL.md index 72e41dbec4..c0d35e9afc 100644 --- a/documentation/webdocs/markdowndocs/TAOS SQL.md +++ b/documentation/webdocs/markdowndocs/TAOS SQL.md @@ -412,7 +412,7 @@ TDengine supports aggregations over numerical values, they are listed below: SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause] ``` Function: the value of the specified column below which `P` percent of the data points fall. - Return Data Type: the same data type. + Return Data Type: double. Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. Applied to: table/STable. Note: The range of `P` is `[0, 100]`. When `P=0` , `PERCENTILE` returns the equal value as `MIN`; when `P=100`, `PERCENTILE` returns the equal value as `MAX`. @@ -446,7 +446,7 @@ TDengine supports aggregations over numerical values, they are listed below: SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause] ``` Function: return the difference between the maximum and the mimimum value. - Return Data Type: the same data type. + Return Data Type: double. Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. Applied to: table/STable. Note: spread gives the range of data variation in a table/supertable; it is equivalent to `MAX()` - `MIN()` diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index c1b8256a06..4707616ba8 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -334,7 +334,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { if (pStatus->dnodeId == 0) { mDebug("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp); } else { - //mDebug("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); + mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); } int32_t openVnodes = htons(pStatus->openVnodes); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index b763c62640..3a9076335a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -171,6 +171,12 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) { mnodeVgroupUpdateIdPool(pVgroup); + // reset vgid status on vgroup changed + mDebug("vgId:%d, reset sync status to unsynced", pVgroup->vgId); + for (int32_t v = 0; v < pVgroup->numOfVnodes; ++v) { + pVgroup->vnodeGid[v].role = TAOS_SYNC_ROLE_UNSYNCED; + } + mnodeDecVgroupRef(pVgroup); mDebug("vgId:%d, is updated, numOfVnode:%d", pVgroup->vgId, pVgroup->numOfVnodes); @@ -302,6 +308,7 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; if (pVgid->pDnode == pDnode) { + mTrace("dnode:%d, receive status from dnode, vgId:%d status is %d", pVgroup->vgId, pDnode->dnodeId, pVgid->role); pVgid->role = pVload->role; if (pVload->role == TAOS_SYNC_ROLE_MASTER) { pVgroup->inUse = i; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index d856650b0e..7d1a5a52de 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -195,7 +195,6 @@ typedef struct { typedef struct { uint32_t len; uint32_t offset; - // uint32_t padding; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; uint64_t uid; @@ -224,7 +223,7 @@ typedef struct { typedef struct { int16_t colId; - int16_t len; + int32_t len; int32_t type : 8; int32_t offset : 24; int64_t sum; @@ -438,8 +437,9 @@ int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); -int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, int16_t* colIds, int numOfColIds); -int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock); +int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds, + int numOfColIds); +int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo); // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9e87a1f587..326da07a36 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -318,7 +318,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { ASSERT(pCompBlock->last); if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1)) < 0) return -1; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0) @@ -577,11 +577,12 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) } } -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds) { +int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) { ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block int numOfSubBlocks = pCompBlock->numOfSubBlocks; - if (numOfSubBlocks > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); + if (numOfSubBlocks > 1) + pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); tdResetDataCols(pHelper->pDataCols[0]); if (tsdbLoadBlockDataColsImpl(pHelper, pCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; @@ -598,10 +599,10 @@ _err: return -1; } -int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock) { - +int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) { int numOfSubBlock = pCompBlock->numOfSubBlocks; - if (numOfSubBlock > 1) pCompBlock = (SCompBlock *)POINTER_SHIFT(pHelper->pCompInfo, pCompBlock->offset); + if (numOfSubBlock > 1) + pCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); tdResetDataCols(pHelper->pDataCols[0]); if (tsdbLoadBlockDataImpl(pHelper, pCompBlock, pHelper->pDataCols[0]) < 0) goto _err; @@ -703,6 +704,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } // Add checksum + ASSERT(pCompCol->len > 0); pCompCol->len += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len); @@ -792,7 +794,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; ASSERT(pHelper->pDataCols[0]->numOfRows <= blockAtIdx(pHelper, blkIdx)->numOfRows); // Merge if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; @@ -848,7 +850,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tsdbAddSubBlock(pHelper, &compBlock, blkIdx, rowsWritten) < 0) goto _err; } else { // Load-Merge-Write // Load - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx)) < 0) goto _err; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, blkIdx), NULL) < 0) goto _err; if (blockAtIdx(pHelper, blkIdx)->last) pHelper->hasOldLastBlock = false; rowsWritten = rows3; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6a9c8e1ff6..9ca78533ba 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -592,9 +592,12 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); } - tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); + STSchema* pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); + tdInitDataCols(pCheckInfo->pDataCols, pSchema); + tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); + tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); - if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock) == 0) { + if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo) == 0) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0ee4ef4837..1c96ba543c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -415,13 +415,19 @@ void *vnodeGetWal(void *pVnode) { } static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { + int64_t totalStorage = 0; + int64_t compStorage = 0; + int64_t pointsWritten = 0; + if (pVnode->status != TAOS_VN_STATUS_READY) return; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; - if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) return; - if (pVnode->tsdb == NULL) return; - int64_t totalStorage, compStorage, pointsWritten = 0; - tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); + // still need report status when unsynced + if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) { + } else if (pVnode->tsdb == NULL) { + } else { + tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); + } SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); diff --git a/tests/pytest/alter/alter_table_crash.py b/tests/pytest/alter/alter_table_crash.py index aefe9ff26e..d1af022e35 100644 --- a/tests/pytest/alter/alter_table_crash.py +++ b/tests/pytest/alter/alter_table_crash.py @@ -78,5 +78,6 @@ class TDTestCase: tdSql.close() tdLog.success("%s successfully executed" % __file__) + tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/client/client.py b/tests/pytest/client/client.py index 21df7e6a86..9bb1887216 100644 --- a/tests/pytest/client/client.py +++ b/tests/pytest/client/client.py @@ -40,6 +40,9 @@ class TDTestCase: ret = tdSql.query('select server_status() as result') tdSql.checkData(0, 0, 1) + ret = tdSql.execute('alter dnode 127.0.0.1 debugFlag 135') + tdLog.info("alter dnode ret: %d" % ret) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 066dda5d97..b98e5c3d39 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -1,7 +1,6 @@ #!/bin/bash ulimit -c unlimited -python3 ./test.py -f client/client.py python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py python3 ./test.py -f insert/float.py @@ -151,3 +150,6 @@ python3 ./test.py -f stream/stream2.py #alter table python3 ./test.py -f alter/alter_table_crash.py + +# client +python3 ./test.py -f client/client.py diff --git a/tests/pytest/insert/writeDBNonStop.py b/tests/pytest/insert/writeDBNonStop.py index c89853ffb6..bdc93f0469 100644 --- a/tests/pytest/insert/writeDBNonStop.py +++ b/tests/pytest/insert/writeDBNonStop.py @@ -67,7 +67,7 @@ class DBWriteNonStop: self.cursor.execute( "select first(ts), last(ts), min(speed), max(speed), avg(speed), count(*) from st") data = self.cursor.fetchall() - end = datetime.now() + end = datetime.now() self.writeDataToCSVFile(data, (end - start).seconds) time.sleep(.001) @@ -75,8 +75,9 @@ class DBWriteNonStop: self.cursor.close() self.conn.close() + test = DBWriteNonStop() test.connectDB() test.createTable() test.insertData() -test.closeConn() \ No newline at end of file +test.closeConn() diff --git a/tests/pytest/query/filterAllIntTypes.py b/tests/pytest/query/filterAllIntTypes.py index a2bab63c88..4d91168dae 100644 --- a/tests/pytest/query/filterAllIntTypes.py +++ b/tests/pytest/query/filterAllIntTypes.py @@ -89,17 +89,25 @@ class TDTestCase: tdSql.checkRows(101) # range for int type on column - tdSql.query("select * from st%s where num > 50 and num < 100" % curType) - tdSql.checkRows(49) + tdSql.query( + "select * from st%s where num > 50 and num < 100" % + curType) + tdSql.checkRows(49) - tdSql.query("select * from st%s where num >= 50 and num < 100" % curType) - tdSql.checkRows(50) + tdSql.query( + "select * from st%s where num >= 50 and num < 100" % + curType) + tdSql.checkRows(50) - tdSql.query("select * from st%s where num > 50 and num <= 100" % curType) - tdSql.checkRows(50) + tdSql.query( + "select * from st%s where num > 50 and num <= 100" % + curType) + tdSql.checkRows(50) - tdSql.query("select * from st%s where num >= 50 and num <= 100" % curType) - tdSql.checkRows(51) + tdSql.query( + "select * from st%s where num >= 50 and num <= 100" % + curType) + tdSql.checkRows(51) # > for int type on tag tdSql.query("select * from st%s where id > 5" % curType) @@ -135,16 +143,22 @@ class TDTestCase: # range for int type on tag tdSql.query("select * from st%s where id > 5 and id < 7" % curType) - tdSql.checkRows(10) + tdSql.checkRows(10) - tdSql.query("select * from st%s where id >= 5 and id < 7" % curType) - tdSql.checkRows(20) + tdSql.query( + "select * from st%s where id >= 5 and id < 7" % + curType) + tdSql.checkRows(20) - tdSql.query("select * from st%s where id > 5 and id <= 7" % curType) - tdSql.checkRows(20) + tdSql.query( + "select * from st%s where id > 5 and id <= 7" % + curType) + tdSql.checkRows(20) - tdSql.query("select * from st%s where id >= 5 and id <= 7" % curType) - tdSql.checkRows(30) + tdSql.query( + "select * from st%s where id >= 5 and id <= 7" % + curType) + tdSql.checkRows(30) print( "======= Verify filter for %s type finished =========" % diff --git a/tests/pytest/query/filterCombo.py b/tests/pytest/query/filterCombo.py index f72b913c92..e769addb52 100644 --- a/tests/pytest/query/filterCombo.py +++ b/tests/pytest/query/filterCombo.py @@ -52,8 +52,7 @@ class TDTestCase: # illegal condition tdSql.error( - "select * from db.st where ts != '2020-05-13 10:00:00.002' OR tagtype < 2") - tdSql.error("select * from db.st where tagtype <> 1 OR tagtype < 2") + "select * from db.st where ts != '2020-05-13 10:00:00.002' OR tagtype < 2") def stop(self): tdSql.close() diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 6d135e1006..6ea240a334 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -77,7 +77,7 @@ class TDTestCase: # join queries tdSql.query( "select * from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") - tdSql.checkRows(6) + tdSql.checkRows(6) tdSql.error( "select ts, pressure, temperature, id, dscrption from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") @@ -108,7 +108,7 @@ class TDTestCase: tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) - + tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_t.id, stb_p.dscrption, stb_p.pressure from stb_p, stb_t where stb_p.ts=stb_t.ts and stb_p.id = stb_t.id") tdSql.checkRows(6) diff --git a/tests/pytest/query/queryMetaData.py b/tests/pytest/query/queryMetaData.py index 7b95e4a81c..67df20cb9a 100755 --- a/tests/pytest/query/queryMetaData.py +++ b/tests/pytest/query/queryMetaData.py @@ -55,9 +55,9 @@ class MetadataQuery: def createTablesAndInsertData(self, threadID): cursor = self.connectDB() - cursor.execute("use test") + cursor.execute("use test") - tablesPerThread = int (self.tables / self.numOfTherads) + tablesPerThread = int(self.tables / self.numOfTherads) base = threadID * tablesPerThread for i in range(tablesPerThread): cursor.execute( @@ -68,24 +68,72 @@ class MetadataQuery: %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' % - (base + i + 1, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100)) - + (base + i + 1, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100)) + cursor.execute( "insert into t%d values(%d, 1) (%d, 2) (%d, 3) (%d, 4) (%d, 5)" % (base + i + 1, self.ts + 1, self.ts + 2, self.ts + 3, self.ts + 4, self.ts + 5)) - cursor.close() + cursor.close() def queryData(self, query): cursor = self.connectDB() cursor.execute("use test") - print("================= query tag data =================") + print("================= query tag data =================") startTime = datetime.now() cursor.execute(query) cursor.fetchall() @@ -107,15 +155,15 @@ if __name__ == '__main__': print( "================= Create %d tables and insert %d records into each table =================" % (t.tables, t.records)) - startTime = datetime.now() + startTime = datetime.now() threads = [] for i in range(t.numOfTherads): thread = threading.Thread( target=t.createTablesAndInsertData, args=(i,)) thread.start() threads.append(thread) - - for th in threads: + + for th in threads: th.join() endTime = datetime.now() diff --git a/tests/pytest/query/queryMetaPerformace.py b/tests/pytest/query/queryMetaPerformace.py index 0570311b08..8ab7105c9d 100644 --- a/tests/pytest/query/queryMetaPerformace.py +++ b/tests/pytest/query/queryMetaPerformace.py @@ -19,6 +19,7 @@ import time from datetime import datetime import numpy as np + class MyThread(threading.Thread): def __init__(self, func, args=()): @@ -35,17 +36,23 @@ class MyThread(threading.Thread): except Exception: return None + class MetadataQuery: def initConnection(self): self.tables = 100 self.records = 10 - self.numOfTherads =5 + self.numOfTherads = 5 self.ts = 1537146000000 self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" self.config = "/etc/taos" - self.conn = taos.connect( self.host, self.user, self.password, self.config) + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + def connectDB(self): return self.conn.cursor() @@ -69,7 +76,7 @@ class MetadataQuery: cursor.execute("use test") base = threadID * self.tables - tablesPerThread = int (self.tables / self.numOfTherads) + tablesPerThread = int(self.tables / self.numOfTherads) for i in range(tablesPerThread): cursor.execute( '''create table t%d using meters tags( @@ -79,20 +86,69 @@ class MetadataQuery: %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')''' % - (base + i + 1, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100, - (base + i) %100, (base + i) %10000, (base + i) %1000000, (base + i) %100000000, (base + i) %100 * 1.1, (base + i) %100 * 2.3, (base + i) %2, (base + i) %100, (base + i) %100)) + (base + i + 1, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100, (base + i) % + 100, (base + i) % + 10000, (base + i) % + 1000000, (base + i) % + 100000000, (base + i) % + 100 * 1.1, (base + i) % + 100 * 2.3, (base + i) % + 2, (base + i) % + 100, (base + i) % + 100)) for j in range(self.records): cursor.execute( "insert into t%d values(%d, %d)" % (base + i + 1, self.ts + j, j)) cursor.close() - def queryWithTagId(self, threadId, tagId, queryNum): - print("---------thread%d start-----------"%threadId) + + def queryWithTagId(self, threadId, tagId, queryNum): + print("---------thread%d start-----------" % threadId) query = '''select tgcol1, tgcol2, tgcol3, tgcol4, tgcol5, tgcol6, tgcol7, tgcol8, tgcol9, tgcol10, tgcol11, tgcol12, tgcol13, tgcol14, tgcol15, tgcol16, tgcol17, tgcol18, tgcol19, tgcol20, tgcol21, tgcol22, tgcol23, tgcol24, tgcol25, tgcol26, tgcol27, @@ -103,18 +159,19 @@ class MetadataQuery: latancy = [] cursor = self.connectDB() cursor.execute("use test") - for i in range(queryNum): + for i in range(queryNum): startTime = time.time() - cursor.execute(query.format(id = tagId, condition = i)) + cursor.execute(query.format(id=tagId, condition=i)) cursor.fetchall() - latancy.append((time.time() - startTime)) - print("---------thread%d end-----------"%threadId) + latancy.append((time.time() - startTime)) + print("---------thread%d end-----------" % threadId) return latancy + def queryData(self, query): cursor = self.connectDB() cursor.execute("use test") - print("================= query tag data =================") + print("================= query tag data =================") startTime = datetime.now() cursor.execute(query) cursor.fetchall() @@ -124,7 +181,7 @@ class MetadataQuery: (endTime - startTime).seconds) cursor.close() - #self.conn.close() + # self.conn.close() if __name__ == '__main__': @@ -132,18 +189,33 @@ if __name__ == '__main__': t = MetadataQuery() t.initConnection() - latancys = [] - threads = [] + latancys = [] + threads = [] tagId = 1 - queryNum = 1000 + queryNum = 1000 for i in range(t.numOfTherads): - thread = MyThread(t.queryWithTagId, args = (i, tagId, queryNum)) - threads.append(thread) + thread = MyThread(t.queryWithTagId, args=(i, tagId, queryNum)) + threads.append(thread) thread.start() - for i in range(t.numOfTherads): + for i in range(t.numOfTherads): threads[i].join() - latancys.extend(threads[i].get_result()) - print("Total query: %d"%(queryNum * t.numOfTherads)) - print("statistic(s): mean= %f, P50 = %f, P75 = %f, P95 = %f, P99 = %f" - %(sum(latancys)/(queryNum * t.numOfTherads), np.percentile(latancys, 50), np.percentile(latancys, 75), np.percentile(latancys, 95), np.percentile(latancys, 99))) - + latancys.extend(threads[i].get_result()) + print("Total query: %d" % (queryNum * t.numOfTherads)) + print( + "statistic(s): mean= %f, P50 = %f, P75 = %f, P95 = %f, P99 = %f" % + (sum(latancys) / + ( + queryNum * + t.numOfTherads), + np.percentile( + latancys, + 50), + np.percentile( + latancys, + 75), + np.percentile( + latancys, + 95), + np.percentile( + latancys, + 99))) diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 814c627d89..712a56d2d7 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -36,18 +36,17 @@ class TDTestCase: "insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)") # inner join --- bug - tdSql.query("select * from tb1 a, tb2 b where a.ts = b.ts") - tdSql.checkRows(1) + tdSql.error("select * from tb1 a, tb2 b where a.ts = b.ts") # join 3 tables -- bug exists - tdSql.query("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") + tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") # query show stable tdSql.query("show stables") tdSql.checkRows(1) # query show tables - tdSql.query("show table") + tdSql.query("show tables") tdSql.checkRows(2) # query count @@ -71,16 +70,13 @@ class TDTestCase: tdSql.checkRows(2) # query first ... as - tdSql.query("select first(*) as begin from stb1") - tdSql.checkData(0, 1, 1) + tdSql.error("select first(*) as begin from stb1") # query last ... as - tdSql.query("select last(*) as end from stb1") - tdSql.checkData(0, 1, 4) + tdSql.error("select last(*) as end from stb1") # query last_row ... as - tdSql.query("select last_row(*) as end from stb1") - tdSql.checkData(0, 1, 4) + tdSql.error("select last_row(*) as end from stb1") # query group .. by tdSql.query("select sum(c1), t2 from stb1 group by t2") @@ -95,8 +91,7 @@ class TDTestCase: tdSql.checkRows(1) # query ... alias for table ---- bug - tdSql.query("select t.ts from tb1 t") - tdSql.checkRows(2) + tdSql.error("select t.ts from tb1 t") # query ... tbname tdSql.query("select tbname from stb1") @@ -104,7 +99,7 @@ class TDTestCase: # query ... tbname count ---- bug tdSql.query("select count(tbname) from stb1") - tdSql.checkRows(2) + tdSql.checkData(0, 0, 2) # query ... select database ---- bug tdSql.query("SELECT database()") diff --git a/tests/pytest/query/select_last_crash.py b/tests/pytest/query/select_last_crash.py index 9aeb122f82..9b580a24ac 100644 --- a/tests/pytest/query/select_last_crash.py +++ b/tests/pytest/query/select_last_crash.py @@ -23,7 +23,6 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) - self.rowNum = 5000 self.ts = 1537146000000 @@ -36,15 +35,13 @@ class TDTestCase: "create table t1 using st tags('dev_001')") for i in range(self.rowNum): - tdSql.execute("insert into t1 values(%d, 'taosdata%d', %d)" % (self.ts + i, i + 1, i + 1)) + tdSql.execute( + "insert into t1 values(%d, 'taosdata%d', %d)" % + (self.ts + i, i + 1, i + 1)) tdSql.query("select last(*) from st") tdSql.checkRows(1) - - print( - "======= Verify filter for %s type finished =========" % - curType) - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/random-test/random-test-multi-threading-3.py b/tests/pytest/random-test/random-test-multi-threading-3.py index 7079a5c118..a8e2c26ae5 100644 --- a/tests/pytest/random-test/random-test-multi-threading-3.py +++ b/tests/pytest/random-test/random-test-multi-threading-3.py @@ -330,7 +330,6 @@ class Test (Thread): self.q.put(-1) tdLog.exit("second thread failed, first thread exit too") - elif (self.threadId == 2): while True: self.dbEvent.wait() diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index eada5f67f7..d3a8deaf47 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -1,7 +1,6 @@ #!/bin/bash ulimit -c unlimited -python3 ./test.py -f client/client.py python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py python3 ./test.py -f insert/float.py @@ -138,6 +137,9 @@ python3 ./test.py -f query/filterOtherTypes.py python3 ./test.py -f query/queryError.py python3 ./test.py -f query/querySort.py python3 ./test.py -f query/queryJoin.py +python3 ./test.py -f query/filterCombo.py +python3 ./test.py -f query/queryNormal.py +python3 ./test.py -f query/select_last_crash.py #stream python3 ./test.py -f stream/stream1.py @@ -146,4 +148,5 @@ python3 ./test.py -f stream/stream2.py #alter table python3 ./test.py -f alter/alter_table_crash.py - +# client +python3 ./test.py -f client/client.py diff --git a/tests/pytest/smoketest.sh b/tests/pytest/smoketest.sh index 6b21912dd5..7c14b673e5 100755 --- a/tests/pytest/smoketest.sh +++ b/tests/pytest/smoketest.sh @@ -1,10 +1,6 @@ #!/bin/bash ulimit -c unlimited -# client -python3 ./test.py $1 -f client/client.py -python3 ./test.py $1 -s && sleep 1 - # insert python3 ./test.py $1 -f insert/basic.py python3 ./test.py $1 -s && sleep 1 @@ -35,3 +31,7 @@ python3 ./test.py $1 -s && sleep 1 python3 ./test.py $1 -f query/filter.py python3 ./test.py $1 -s && sleep 1 +# client +python3 ./test.py $1 -f client/client.py +python3 ./test.py $1 -s && sleep 1 + diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index dc0366b214..d6644df3ac 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -198,7 +198,7 @@ class TDSql: "%s failed: sql:%s, affectedRows:%d != expect:%d" % (callerFilename, self.sql, self.affectedRows, expectAffectedRows)) tdLog.info("sql:%s, affectedRows:%d == expect:%d" % - (self.sql, self.affectedRows, expectAffectedRows)) + (self.sql, self.affectedRows, expectAffectedRows)) tdSql = TDSql()