From e08fe44fdfebe0cfef0e52b062d88734ed6acbc0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 09:38:04 +0800 Subject: [PATCH 01/56] DB/create: fix create database when default quorum > replica --- src/mnode/src/mnodeDb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..7cc5d41ae1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -389,7 +389,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = tsCompression; if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; - if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->quorum < 0) pCfg->quorum = MIN(tsQuorum, pCfg->replications); if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; if (pCfg->dbType < 0) pCfg->dbType = 0; From 8c06d5952a70589804562409eef0e2a3171545b5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 10:07:35 +0800 Subject: [PATCH 02/56] [TD-4081]: fix vnode dropping --- src/vnode/src/vnodeMain.c | 2 -- src/vnode/src/vnodeMgmt.c | 2 ++ src/vnode/src/vnodeRead.c | 8 ++++++-- src/vnode/src/vnodeWrite.c | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 8ec66316e3..21d4480a73 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -124,8 +124,6 @@ int32_t vnodeDrop(int32_t vgId) { vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; - // remove from hash, so new messages wont be consumed - vnodeRemoveFromHash(pVnode); vnodeRelease(pVnode); vnodeCleanupInMWorker(pVnode); diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 32f9532138..62eb4dadcc 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,6 +118,8 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { + vnodeRemoveFromHash(pVnode); + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index b60fc5a8cd..2f8da0012b 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -117,14 +117,18 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con } int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (pVnode->dropped) { + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); if (pRead == NULL) { assert(terrno != 0); return terrno; } - SVnodeObj *pVnode = vparam; - int32_t code = vnodeCheckRead(pVnode); if (code != TSDB_CODE_SUCCESS) { taosFreeQitem(pRead); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 36516d81df..1b6e2e37ca 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -386,4 +386,6 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued wmsg num:%d", pVnode->vgId, pVnode->queuedWMsg); taosMsleep(10); } + + taosMsleep(1000 * 3); } From cb1064c1a714b116793cce87c0d0304a178c6c49 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 14 May 2021 10:10:54 +0800 Subject: [PATCH 03/56] [TD-4179]: add test case --- .../jdbc/TSDBPreparedStatementTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 8804cc5da0..52858e7f88 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -207,10 +207,69 @@ public class TSDBPreparedStatementTest { while(rs.next()) { rows++; } - Assert.assertEquals(numOfRows, rows); + Assert.assertEquals(numOfRows, rows); } } + @Test + public void bindDataSelectColumnTest() throws SQLException { + Statement stmt = conn.createStatement(); + + int numOfRows = 1000; + + for (int loop = 0; loop < 10; loop++){ + stmt.execute("drop table if exists weather_test"); + stmt.execute("create table weather_test(ts timestamp, f1 nchar(4), f2 float, f3 double, f4 timestamp, f5 int, f6 bool, f7 binary(10))"); + + TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? (ts, f1, f7) values(?, ?, ?)"); + Random r = new Random(); + s.setTableName("weather_test"); + + ArrayList ts = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + ts.add(System.currentTimeMillis() + i); + } + s.setTimestamp(0, ts); + + int random = 10 + r.nextInt(5); + ArrayList s2 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s2.add(null); + }else{ + s2.add("分支" + i % 4); + } + } + s.setNString(1, s2, 4); + + random = 10 + r.nextInt(5); + ArrayList s5 = new ArrayList(); + for(int i = 0; i < numOfRows; i++) { + if(i % random == 0) { + s5.add(null); + }else{ + s5.add("test" + i % 10); + } + } + s.setString(2, s5, 10); + + s.columnDataAddBatch(); + s.columnDataExecuteBatch(); + s.columnDataCloseBatch(); + + String sql = "select * from weather_test"; + PreparedStatement statement = conn.prepareStatement(sql); + ResultSet rs = statement.executeQuery(); + int rows = 0; + while(rs.next()) { + rows++; + } + Assert.assertEquals(numOfRows, rows); + } + } + + + @Test public void setBoolean() throws SQLException { pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); From 70213e65261e6ad7290d497383c7580efe93d16a Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 11:55:49 +0800 Subject: [PATCH 04/56] test/sim: fix null to 0 in show dnodes; --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 1d707bc4a3..11f6762129 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != null then +if $data2_2 != 0 then goto show4 endi if $data2_3 != 3 then From 226f86c8bf4d155f1ecc06ba196b5af95e3eb9e7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 14 May 2021 15:09:49 +0800 Subject: [PATCH 05/56] [TD-4081]: [v3] fix vnode closing --- src/vnode/src/vnodeMain.c | 6 ++++++ src/vnode/src/vnodeMgmt.c | 2 -- src/vnode/src/vnodeStatus.c | 3 +++ src/vnode/src/vnodeSync.c | 11 +++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 21d4480a73..58baefefd2 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -120,6 +120,10 @@ int32_t vnodeDrop(int32_t vgId) { vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } + if (pVnode->dropped) { + vnodeRelease(pVnode); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; @@ -473,6 +477,8 @@ void vnodeCleanUp(SVnodeObj *pVnode) { vnodeSetClosingStatus(pVnode); + vnodeRemoveFromHash(pVnode); + // stop replication module if (pVnode->sync > 0) { int64_t sync = pVnode->sync; diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 62eb4dadcc..32f9532138 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -118,8 +118,6 @@ void vnodeRelease(void *vparam) { tsem_post(&pVnode->sem); } } else { - vnodeRemoveFromHash(pVnode); - vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", pVnode->vgId, refCount, pVnode); vnodeDestroyInMWorker(pVnode); int32_t count = taosHashGetSize(tsVnodesHash); diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index c482d1fd1a..1eaddc3d25 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -66,6 +66,9 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) { } bool vnodeSetClosingStatus(SVnodeObj* pVnode) { + if (pVnode->status == TAOS_VN_STATUS_CLOSING) + return true; + while (!vnodeSetClosingStatusImp(pVnode)) { taosMsleep(1); } diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 05af34a34f..e5a1964915 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -55,6 +55,11 @@ void vnodeNotifyRole(int32_t vgId, int8_t role) { vTrace("vgId:%d, vnode not found while notify role", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while notify role", vgId); + vnodeRelease(pVnode); + return; + } vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; @@ -75,6 +80,11 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } + if (pVnode->dropped) { + vTrace("vgId:%d, vnode dropped while flow ctrl", vgId); + vnodeRelease(pVnode); + return; + } if (pVnode->flowctrlLevel != level) { vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); @@ -129,6 +139,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); + vnodeRelease(pVnode); return TSDB_CODE_VND_INVALID_VGROUP_ID; } From 4fd37ea2a1beae14ebfc636fdcdf53171d3181bc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 12:43:08 +0800 Subject: [PATCH 06/56] [TD-4081]: vnode not close if dropped --- src/vnode/src/vnodeMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 58baefefd2..f02a5bedb0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -392,6 +392,10 @@ int32_t vnodeOpen(int32_t vgId) { int32_t vnodeClose(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return 0; + if (pVnode->dropped) { + vnodeRelease(pVnode); + return 0; + } vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); vnodeRemoveFromHash(pVnode); From 198485c6f43667a8d14b4dfe7dfeeaf93d12307d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:32:31 +0800 Subject: [PATCH 07/56] Revert "test/sim: fix null to 0 in show dnodes;" This reverts commit 70213e65261e6ad7290d497383c7580efe93d16a. --- tests/script/unique/dnode/remove2.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 11f6762129..1d707bc4a3 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -162,7 +162,7 @@ print dnode3 openVnodes $data2_3 if $data2_1 != 1 then goto show4 endi -if $data2_2 != 0 then +if $data2_2 != null then goto show4 endi if $data2_3 != 3 then From 7c93c4217a84a951dde190053abfa4364821b15f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 15:33:19 +0800 Subject: [PATCH 08/56] vnode/write: make last write msg to be written or confirmed --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 1b6e2e37ca..3d16ae1faf 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000 * 3); + taosMsleep(1000); } From 2fb7c16b8757f5a1cf13bd8b5e008ead24aeafb4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 16:46:43 +0800 Subject: [PATCH 09/56] vnode/drop: ingore invalid vgroup id if already dropped --- src/vnode/src/vnodeMain.c | 2 +- src/vnode/src/vnodeWrite.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f02a5bedb0..ee28be3d2f 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -122,7 +122,7 @@ int32_t vnodeDrop(int32_t vgId) { } if (pVnode->dropped) { vnodeRelease(pVnode); - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_SUCCESS; } vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 3d16ae1faf..7c67865a2b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(1000); + taosMsleep(2100); } From dae9262562cedbb6c6c61448f39342fb5f7e2b2e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 15 May 2021 20:42:48 +0800 Subject: [PATCH 10/56] vnode/read: use app not ready instead of invalid vgroup id to make crash_gen happy --- src/vnode/src/vnodeRead.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2f8da0012b..5d344ebf11 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -118,9 +118,8 @@ static SVReadMsg *vnodeBuildVReadMsg(SVnodeObj *pVnode, void *pCont, int32_t con int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; - if (pVnode->dropped) { - return TSDB_CODE_VND_INVALID_VGROUP_ID; + return TSDB_CODE_APP_NOT_READY; } SVReadMsg *pRead = vnodeBuildVReadMsg(vparam, pCont, contLen, qtype, rparam); From 610603478155d87fa71f5c2d2fa97bb6b41e0f96 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 16 May 2021 11:24:35 +0800 Subject: [PATCH 11/56] vnode/close: wait write 900ms --- src/vnode/src/vnodeWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 7c67865a2b..56ea32ccc0 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -387,5 +387,5 @@ void vnodeWaitWriteCompleted(SVnodeObj *pVnode) { taosMsleep(10); } - taosMsleep(2100); + taosMsleep(900); } From e620e2e83a26c8e8784d203010ef914172607a95 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 12:53:35 +0800 Subject: [PATCH 12/56] [td-4209]: add the uid check for super table metadata that is already cached in local buffer. --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 ++++-- src/client/src/tscSchemaUtil.c | 1 + src/client/src/tscServer.c | 28 +++++++++++++--------------- src/client/src/tscUtil.c | 21 ++++++++++----------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index b80dcfb577..47b2865313 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -298,7 +298,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d94d270a7..4bfd3bc88f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -67,14 +67,16 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name + char sTableName[TSDB_TABLE_FNAME_LEN]; // TODO: refactor super table name, not full name + uint64_t suid; // super table id } CChildTableMeta; typedef struct STableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; // super table name + uint64_t suid; // super table id int16_t sversion; int16_t tversion; STableComInfo tableInfo; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2ea382132b..114fc8ee73 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -94,6 +94,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->suid = pTableMetaMsg->suid; pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 33325c7fd7..664f2989a8 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1920,13 +1920,13 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->contLen = htons(pMetaMsg->contLen); + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && @@ -2537,19 +2537,16 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - } else { - //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; } + memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; + pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2565,8 +2562,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { + // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); + int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3d2cf44560..89749202ef 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2941,22 +2941,25 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { assert(pTableMeta != NULL); CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; - cMeta->vgId = pTableMeta->vgId; - cMeta->id = pTableMeta->id; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + cMeta->suid = pTableMeta->suid; tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { +int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { assert(pChild != NULL && buf != NULL); -// uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = buf;//calloc(1, size); - + STableMeta* p = buf; taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); - if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + + // tableMeta exists, build child table meta according to the super table meta + // the uid need to be checked in addition to the general name of the super table. + if (p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -2964,13 +2967,9 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - -// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - -// tfree(p); return -1; } } From 07aed4fa708386e96b441ea1becfb89af80bea50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:23:44 +0800 Subject: [PATCH 13/56] [td-4209] --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 664f2989a8..0e16369cad 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1925,7 +1925,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->suid = htobe64(pMetaMsg->suid); + pMetaMsg->suid = pMetaMsg->suid; pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); From c824330bb5548f963f27255799c7f73d10765dcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 May 2021 15:51:49 +0800 Subject: [PATCH 14/56] [td-225] refactor a sim script. --- tests/script/general/parser/commit.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index dfe521b92b..7c4c883fb1 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -68,7 +68,7 @@ while $loop <= $loops while $i < 10 sql select count(*) from $stb where t1 = $i if $data00 != $rowNum then - print expect $rowNum, actual: $data00 + print expect $rowNum , actual: $data00 return -1 endi $i = $i + 1 From 4cb5bedfaeaeeec84a79e7a72e1a279f4f9111f6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 18 May 2021 13:48:45 +0800 Subject: [PATCH 15/56] [TD-4232]: fix vnode queueWMsg & queuedWMsgSize counting under flow ctrl --- src/vnode/src/vnodeWrite.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 56ea32ccc0..16089c8e91 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -340,8 +340,11 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { if (pWrite->processedCount >= 100) { vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), pWrite->processedCount); - pWrite->processedCount = 1; - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + void *handle = pWrite->rpcMsg.handle; + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + SRpcMsg rpcRsp = {.handle = handle, .code = code}; + rpcSendResponse(&rpcRsp); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { From 6d448dcac1e522e3496c133d7d6d2a72dc2b7d5f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 16:48:17 +0800 Subject: [PATCH 16/56] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 (#6147) * [TD-4136]: taosdemo max records per req < 32767 * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 48 ++++++++++++---------- tests/pytest/tools/taosdemoTestWithJson.py | 22 +++++----- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 12262f7a32..f6ac4a75ba 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -516,6 +516,8 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); +static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -1031,10 +1033,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } @@ -3410,10 +3410,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt2(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -3470,9 +3467,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -5342,6 +5339,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t start_time = pThreadInfo->start_time; int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); for (uint64_t i = 0; i < insertRows;) { @@ -6022,6 +6020,21 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } +} + +static void prompt2() +{ + if (!g_args.answer_yes) { + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + } +} static int insertTestProcess() { @@ -6042,10 +6055,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6317,10 +6327,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6673,10 +6680,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") From 4412c52f27bca02a0b645f24858d94010d1d1039 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:36:07 +0800 Subject: [PATCH 17/56] Hotfix/sangshuduo/td 4187 taosdemo keepprogress (#6149) * [TD-4187]: taosdemo keepProgress. * add consumed. * [TD-4187]: taosdemo keepProgress. add resubAfterConsume process. * print resub msg for test. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/subscribe.json | 40 +++-- src/kit/taosdemo/taosdemo.c | 281 +++++++++++++++++++++++--------- 2 files changed, 234 insertions(+), 87 deletions(-) diff --git a/src/kit/taosdemo/subscribe.json b/src/kit/taosdemo/subscribe.json index fd33a2e2e2..9faf03a03d 100644 --- a/src/kit/taosdemo/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -1,17 +1,37 @@ { - "filetype":"subscribe", + "filetype": "subscribe", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", - "databases": "dbx", - "specified_table_query": - {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] - }, - "super_table_query": - {"stblname": "stb", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", - "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] - } + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select avg(col1) from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f6ac4a75ba..08c1142070 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -227,7 +227,7 @@ typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN + 1]; char dataType[MAX_TB_NAME_SIZE]; uint32_t dataLen; - char note[128]; + char note[128]; } StrColumn; typedef struct SSuperTable_S { @@ -360,10 +360,11 @@ typedef struct SpecifiedQueryInfo_S { uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -374,7 +375,7 @@ typedef struct SuperQueryInfo_S { uint32_t threadCnt; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms - int subscribeRestart; + bool subscribeRestart; int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; @@ -382,6 +383,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -399,7 +401,7 @@ typedef struct SQueryMetaInfo_S { char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; - SuperQueryInfo superQueryInfo; + SuperQueryInfo superQueryInfo; uint64_t totalQueried; } SQueryMetaInfo; @@ -521,8 +523,8 @@ static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, + uint16_t port, char* sqlstr, char *resultFile); /* ************ Global variables ************ */ @@ -1141,12 +1143,14 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } - verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", + __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } -static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile) +static void selectAndGetResult( + threadInfo *pThreadInfo, char *command, char* resultFile) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); @@ -1291,13 +1295,15 @@ static void init_rand_data() { static int printfInsertMeta() { SHOW_PARSE_RESULT_START(); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("host: \033[33m%s:%u\033[0m\n", + g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); printf("configDir: \033[33m%s\033[0m\n", configDir); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); - printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("thread num of create table: \033[33m%d\033[0m\n", + g_Dbs.threadCountByCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%"PRIu64"\033[0m\n", @@ -1309,7 +1315,8 @@ static int printfInsertMeta() { for (int i = 0; i < g_Dbs.dbCount; i++) { printf("database[\033[33m%d\033[0m]:\n", i); - printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); + printf(" database[%d] name: \033[33m%s\033[0m\n", + i, g_Dbs.db[i].dbName); if (0 == g_Dbs.db[i].drop) { printf(" drop: \033[33mno\033[0m\n"); } else { @@ -1317,40 +1324,51 @@ static int printfInsertMeta() { } if (g_Dbs.db[i].dbCfg.blocks > 0) { - printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + printf(" blocks: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.blocks); } if (g_Dbs.db[i].dbCfg.cache > 0) { - printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + printf(" cache: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.cache); } if (g_Dbs.db[i].dbCfg.days > 0) { - printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + printf(" days: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.days); } if (g_Dbs.db[i].dbCfg.keep > 0) { - printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + printf(" keep: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.keep); } if (g_Dbs.db[i].dbCfg.replica > 0) { - printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + printf(" replica: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.replica); } if (g_Dbs.db[i].dbCfg.update > 0) { - printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + printf(" update: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.update); } if (g_Dbs.db[i].dbCfg.minRows > 0) { - printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + printf(" minRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.minRows); } if (g_Dbs.db[i].dbCfg.maxRows > 0) { - printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + printf(" maxRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.maxRows); } if (g_Dbs.db[i].dbCfg.comp > 0) { printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); } if (g_Dbs.db[i].dbCfg.walLevel > 0) { - printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + printf(" walLevel: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.walLevel); } if (g_Dbs.db[i].dbCfg.fsync > 0) { - printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + printf(" fsync: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.fsync); } if (g_Dbs.db[i].dbCfg.quorum > 0) { - printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + printf(" quorum: \033[33m%d\033[0m\n", + g_Dbs.db[i].dbCfg.quorum); } if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) @@ -1543,21 +1561,26 @@ static void printfInsertMetaToFile(FILE* fp) { if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision: %s\n", + g_Dbs.db[i].dbCfg.precision); } else { - fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + fprintf(fp, " precision error: %s\n", + g_Dbs.db[i].dbCfg.precision); } } - fprintf(fp, " super table count: %"PRIu64"\n", g_Dbs.db[i].superTblCount); + fprintf(fp, " super table count: %"PRIu64"\n", + g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { fprintf(fp, " super table[%d]:\n", j); - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + fprintf(fp, " stbName: %s\n", + g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + } else if (AUTO_CREATE_SUBTBL + == g_Dbs.db[i].superTbls[j].autoCreateTable) { fprintf(fp, " autoCreateTable: %s\n", "yes"); } else { fprintf(fp, " autoCreateTable: %s\n", "error"); @@ -1565,7 +1588,8 @@ static void printfInsertMetaToFile(FILE* fp) { if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + } else if (TBL_ALREADY_EXISTS + == g_Dbs.db[i].superTbls[j].childTblExists) { fprintf(fp, " childTblExists: %s\n", "yes"); } else { fprintf(fp, " childTblExists: %s\n", "error"); @@ -1596,8 +1620,10 @@ static void printfInsertMetaToFile(FILE* fp) { */ fprintf(fp, " interlaceRows: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].interlaceRows); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " disorderRange: %d\n", + g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", + g_Dbs.db[i].superTbls[j].disorderRatio); fprintf(fp, " maxSqlLen: %"PRIu64"\n", g_Dbs.db[i].superTbls[j].maxSqlLen); @@ -1605,23 +1631,29 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].timeStampStep); fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + fprintf(fp, " sampleFormat: %s\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", + g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", + g_Dbs.db[i].superTbls[j].tagsFile); - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + fprintf(fp, " columnCount: %d\n ", + g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); if ((0 == strncasecmp( g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + || (0 == strncasecmp( + g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", strlen("nchar")))) { fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + fprintf(fp, "column[%d]:%s ", + k, g_Dbs.db[i].superTbls[j].columns[k].dataType); } } fprintf(fp, "\n"); @@ -1634,7 +1666,8 @@ static void printfInsertMetaToFile(FILE* fp) { "binary", strlen("binary"))) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", strlen("nchar")))) { - fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, + fprintf(fp, "tag[%d]:%s(%d) ", + k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); } else { fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); @@ -4128,7 +4161,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", + errorPrint( + "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", __func__, __LINE__, specifiedQueryTimes->valueint); goto PARSE_OVER; @@ -4145,7 +4179,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + errorPrint( + "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4184,15 +4219,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; + g_queryInfo.specifiedQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + g_queryInfo.specifiedQueryInfo.subscribeRestart = true; } cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); @@ -4237,13 +4272,29 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { printf("ERROR: failed to read json, sql not found\n"); goto PARSE_OVER; } - tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], + sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON* resubAfterConsume = + cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume"); + if (resubAfterConsume + && resubAfterConsume->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] + = resubAfterConsume->valueint; + } else if (!resubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1; + } cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + if ((NULL != result) && (result->type == cJSON_String) + && (result->valuestring != NULL)) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { - memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + memset(g_queryInfo.specifiedQueryInfo.result[j], + 0, MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, super query result file not found\n"); goto PARSE_OVER; @@ -4350,27 +4401,27 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; + g_queryInfo.superQueryInfo.subscribeRestart = false; } else { printf("ERROR: failed to read json, subscribe restart error\n"); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; + g_queryInfo.superQueryInfo.subscribeRestart = true; } - cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (subkeepProgress && - subkeepProgress->type == cJSON_String - && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { + cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (superkeepProgress && + superkeepProgress->type == cJSON_String + && superkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + } else if (0 == strcmp("no", superkeepProgress->valuestring)) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } else { - printf("ERROR: failed to read json, subscribe keepProgress error\n"); + printf("ERROR: failed to read json, subscribe super table keepProgress error\n"); goto PARSE_OVER; } } else { @@ -4408,6 +4459,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(sql, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume[j] = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; + } + cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -5506,7 +5569,8 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != pThreadInfo->superTblInfo->disorderRatio && rand_num < pThreadInfo->superTblInfo->disorderRatio) { - int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); + int64_t d = pThreadInfo->lastTs + - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1); generateRowData(data, d, pThreadInfo->superTblInfo); } else { generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo); @@ -6480,17 +6544,18 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS *taos, char *sql, char* topic, bool restart, + char* resultFileName) { TAOS_SUB* tsub = NULL; if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.specifiedQueryInfo.subscribeInterval); } else { tsub = taos_subscribe(taos, - g_queryInfo.specifiedQueryInfo.subscribeRestart, + restart, topic, sql, NULL, NULL, 0); } @@ -6551,7 +6616,9 @@ static void *superSubscribe(void *sarg) { uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; @@ -6559,17 +6626,22 @@ static void *superSubscribe(void *sarg) { } } + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + // start loop to consume result TAOS_RES* res = NULL; while(1) { - for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taosMsleep(100); // ms + taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[subSeq]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; @@ -6579,6 +6651,28 @@ static void *superSubscribe(void *sarg) { pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[j] ++; + + if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) + && (consumed[j] >= + g_queryInfo.superQueryInfo.resubAfterConsume[j])) { + printf("keepProgress:%d, resub super table query: %d\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, j); + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + consumed[j]= 0; + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl( + pThreadInfo->taos, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[subSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } @@ -6589,8 +6683,7 @@ static void *superSubscribe(void *sarg) { i <= pThreadInfo->end_table_to; i++) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[subSeq], 0); } } @@ -6635,40 +6728,68 @@ static void *specifiedSubscribe(void *sarg) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], + pThreadInfo->threadID); } tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } + // start loop to consume result TAOS_RES* res = NULL; + + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) + consumed[i] = 0; + while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - taosMsleep(1000); // ms + taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.specifiedQueryInfo.result[i], + pThreadInfo->threadID); appendResultToFile(res, tmpFile); } + consumed[i] ++; + + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed[i] >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { + printf("keepProgress:%d, resub specified query: %d\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); + consumed[i] = 0; + taos_unsubscribe(tsub[i], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub[i] = subscribeImpl(pThreadInfo->taos, + g_queryInfo.specifiedQueryInfo.sql[i], topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + tmpFile); + if (NULL == tsub[i]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } } } } taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); @@ -6719,8 +6840,10 @@ static int subscribeTestProcess() { exit(-1); } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + pids = malloc( + g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc( + g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); @@ -6905,17 +7028,21 @@ static void setParaFromArg(){ if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + for (int i = g_Dbs.db[0].superTbls[0].columnCount; + i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } - tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, + "INT", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, + "BINARY", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { From 8e5681f1b004cb8b5d3982fb36f0ffd4d712e2ca Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 May 2021 21:36:35 +0800 Subject: [PATCH 18/56] [TD-4240]: taosdemo subscribe more than max query sql count. (#6154) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 08c1142070..92b5f6abff 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6575,6 +6575,15 @@ static void *superSubscribe(void *sarg) { if (g_queryInfo.superQueryInfo.sqlCount == 0) return NULL; + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, From 5c03dc9ae0f3f2a8efac6d4978878cd69eecd26b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 16:20:38 +0800 Subject: [PATCH 19/56] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6161) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/async-sub.json | 41 +++ src/kit/taosdemo/taosdemo.c | 430 +++++++++++++++++--------------- 2 files changed, 267 insertions(+), 204 deletions(-) create mode 100644 src/kit/taosdemo/async-sub.json diff --git a/src/kit/taosdemo/async-sub.json b/src/kit/taosdemo/async-sub.json new file mode 100644 index 0000000000..a30a1be45c --- /dev/null +++ b/src/kit/taosdemo/async-sub.json @@ -0,0 +1,41 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "specified_table_query": { + "concurrent": 1, + "mode": "async", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "resubAfterConsume": 10, + "sqls": [ + { + "sql": "select col1 from meters where col1 > 1;", + "result": "./subscribe_res0.txt" + }, + { + "sql": "select col2 from meters where col2 > 1;", + "result": "./subscribe_res2.txt" + } + ] + }, + "super_table_query": { + "stblname": "meters", + "threads": 1, + "mode": "sync", + "interval": 1000, + "restart": "yes", + "keepProgress": "yes", + "sqls": [ + { + "sql": "select col1 from xxxx where col1 > 10;", + "result": "./subscribe_res1.txt" + } + ] + } +} diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 92b5f6abff..0831555d22 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -441,8 +441,9 @@ typedef struct SThreadInfo_S { uint64_t maxDelay; uint64_t minDelay; - // query + // seq of query or subscribe uint64_t querySeq; // sequence number of sql command + } threadInfo; #ifdef WINDOWS @@ -1107,7 +1108,6 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } - fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1150,7 +1150,7 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { } static void selectAndGetResult( - threadInfo *pThreadInfo, char *command, char* resultFile) + threadInfo *pThreadInfo, char *command) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { TAOS_RES *res = taos_query(pThreadInfo->taos, command); @@ -1161,8 +1161,8 @@ static void selectAndGetResult( return; } - if ((resultFile) && (strlen(resultFile))) { - appendResultToFile(res, resultFile); + if ((strlen(pThreadInfo->fp))) { + appendResultToFile(res, pThreadInfo->fp); } taos_free_result(res); @@ -1170,7 +1170,7 @@ static void selectAndGetResult( int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - resultFile); + pThreadInfo->fp); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -6230,23 +6230,22 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + } + while(queryTimes --) { if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - st = taosGetTimestampMs(); selectAndGetResult(pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); et = taosGetTimestampMs(); printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", @@ -6332,13 +6331,12 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", + sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6407,7 +6405,7 @@ static int queryTestProcess() { threadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; - int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; + uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; uint64_t startTs = taosGetTimestampMs(); @@ -6421,32 +6419,33 @@ static int queryTestProcess() { ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < nConcurrent; i++) { - for (int j = 0; j < nSqlCount; j++) { - threadInfo *t_info = infos + i * nSqlCount + j; - t_info->threadID = i * nSqlCount + j; - t_info->querySeq = j; + for (uint64_t i = 0; i < nSqlCount; i++) { + for (int j = 0; j < nConcurrent; j++) { + uint64_t seq = i * nConcurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { - taos_close(taos); - free(infos); - free(pids); - errorPrint( "use database %s failed!\n\n", - g_queryInfo.dbName); - return -1; - } + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } + } + + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + seq, NULL, specifiedTableQuery, + t_info); } - - t_info->taos = NULL;// TODO: workaround to use separate taos connection; - - pthread_create(pids + i * nSqlCount + j, NULL, specifiedTableQuery, - t_info); - } } } else { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -6531,7 +6530,8 @@ static int queryTestProcess() { return 0; } -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { +static void stable_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { if (res == NULL || taos_errno(res) != 0) { errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", __func__, __LINE__, code, taos_errstr(res)); @@ -6539,22 +6539,44 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c } if (param) - appendResultToFile(res, (char*)param); + appendResultToFile(res, ((threadInfo *)param)->fp); + // tao_unscribe() will free result. +} + +static void specified_sub_callback( + TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } + + if (param) + appendResultToFile(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } static TAOS_SUB* subscribeImpl( - TAOS *taos, char *sql, char* topic, bool restart, - char* resultFileName) { + threadInfo *pThreadInfo, + char *sql, char* topic, bool restart) +{ TAOS_SUB* tsub = NULL; if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { - tsub = taos_subscribe(taos, + tsub = taos_subscribe( + pThreadInfo->taos, restart, - topic, sql, subscribe_callback, (void*)resultFileName, + topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); + } else if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + tsub = taos_subscribe( + pThreadInfo->taos, + restart, + topic, sql, stable_sub_callback, (void*)pThreadInfo, + g_queryInfo.superQueryInfo.subscribeInterval); } else { - tsub = taos_subscribe(taos, + tsub = taos_subscribe( + pThreadInfo->taos, restart, topic, sql, NULL, NULL, 0); } @@ -6572,13 +6594,8 @@ static void *superSubscribe(void *sarg) { char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - if (g_queryInfo.superQueryInfo.sqlCount == 0) - return NULL; - - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, + if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); exit(-1); @@ -6612,88 +6629,80 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", + i, pThreadInfo->querySeq); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); + replaceChildTblName( + g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], + subSqlstr, i); + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", + __func__, __LINE__, pThreadInfo->threadID, subSqlstr); + tsub[i] = subscribeImpl( + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } - } } - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; - // start loop to consume result + int consumed[MAX_QUERY_SQL_COUNT]; + for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { + consumed[i] = 0; + } TAOS_RES* res = NULL; + while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms - res = taos_consume(tsub[subSeq]); + res = taos_consume(tsub[i]); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[j], + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, pThreadInfo->fp); } - consumed[j] ++; + consumed[i] ++; if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) - && (consumed[j] >= - g_queryInfo.superQueryInfo.resubAfterConsume[j])) { - printf("keepProgress:%d, resub super table query: %d\n", - g_queryInfo.superQueryInfo.subscribeKeepProgress, j); - taos_unsubscribe(tsub[subSeq], + && (consumed[i] >= + g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub, g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[j]= 0; - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", - __func__, __LINE__, subSeq, subSqlstr); - tsub[subSeq] = subscribeImpl( - pThreadInfo->taos, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[subSeq]) { + consumed[i]= 0; + tsub[i] = subscribeImpl( + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart + ); + if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; } } } - } } } taos_free_result(res); for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { - uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; - taos_unsubscribe(tsub[subSeq], 0); - } + taos_unsubscribe(tsub[i], 0); } taos_close(pThreadInfo->taos); @@ -6702,10 +6711,7 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; - - if (g_queryInfo.specifiedQueryInfo.sqlCount == 0) - return NULL; + TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; @@ -6732,76 +6738,61 @@ static void *specifiedSubscribe(void *sarg) { } char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], + sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } + } + tsub = subscribeImpl(pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart); + if (NULL == tsub) { + taos_close(pThreadInfo->taos); + return NULL; } // start loop to consume result TAOS_RES* res = NULL; - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) - consumed[i] = 0; + int consumed; while(1) { - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms - res = taos_consume(tsub[i]); + res = taos_consume(tsub); if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], - pThreadInfo->threadID); - appendResultToFile(res, tmpFile); - } - consumed[i] ++; - - if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) - && (consumed[i] >= - g_queryInfo.specifiedQueryInfo.resubAfterConsume[i])) { - printf("keepProgress:%d, resub specified query: %d\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, i); - consumed[i] = 0; - taos_unsubscribe(tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - tsub[i] = subscribeImpl(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart, - tmpFile); - if (NULL == tsub[i]) { + consumed ++; + if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) + && (consumed >= + g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub specified query: %"PRIu64"\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + consumed = 0; + taos_unsubscribe(tsub, + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + tsub = subscribeImpl( + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart + ); + if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; - } - } + } + } } - } } taos_free_result(res); - - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], 0); - } - + taos_unsubscribe(tsub, 0); taos_close(pThreadInfo->taos); + return NULL; } @@ -6836,7 +6827,11 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query for specified table + + pthread_t *pidsOfStable = NULL; + threadInfo *infosOfStable = NULL; + + //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, @@ -6850,81 +6845,108 @@ static int subscribeTestProcess() { } pids = malloc( - g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(pthread_t)); infos = malloc( - g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + g_queryInfo.specifiedQueryInfo.sqlCount * + g_queryInfo.specifiedQueryInfo.concurrent * + sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); exit(-1); } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + threadInfo *t_info = infos + seq; + t_info->threadID = seq; + t_info->querySeq = i; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, t_info); + } } } - //==== create sub threads for super table query - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - if ((g_queryInfo.superQueryInfo.sqlCount > 0) + //==== create threads for super table query + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + pidsOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfStable = malloc( + g_queryInfo.superQueryInfo.sqlCount * + g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", + if ((NULL == pidsOfStable) || (NULL == infosOfStable)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - // taos_close(taos); - exit(-1); - } + // taos_close(taos); + exit(-1); + } - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; + int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; - int64_t a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } + int64_t a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } + int64_t b = 0; + if (threads != 0) { + b = ntables % threads; + } - uint64_t startFrom = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + uint64_t startFrom = 0; + for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + threadInfo *t_info = infosOfStable + seq; + t_info->threadID = seq; + t_info->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superSubscribe, t_info); - } + t_info->start_table_from = startFrom; + t_info->ntables = jend_table_to = jend_table_to + 1; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfStable + seq, + NULL, superSubscribe, t_info); + } + } - g_queryInfo.superQueryInfo.threadCnt = threads; + g_queryInfo.superQueryInfo.threadCnt = threads; - for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + for (int j = 0; j < threads; j++) { + uint64_t seq = i * threads + j; + pthread_join(pidsOfStable[seq], NULL); + } + } } } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { + uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; + pthread_join(pids[seq], NULL); + } } tmfree((char*)pids); tmfree((char*)infos); - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); + tmfree((char*)pidsOfStable); + tmfree((char*)infosOfStable); // taos_close(taos); return 0; } From 78b8473e03670ec434fe7fe30fbeaf7730beace2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 May 2021 20:12:19 +0800 Subject: [PATCH 20/56] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6162) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * [TD-4238]: taosdemo async subscribe. interval. * [TD-4238]: taosdemo async subscribe. fix super table sub result file. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 61 +++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0831555d22..f8d3f83d47 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -114,12 +114,18 @@ typedef enum TALBE_EXISTS_EN { TBL_EXISTS_BUTT } TALBE_EXISTS_EN; -enum MODE { +enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT }; +typedef enum enumQUERY_CLASS { + SPECIFIED_CLASS, + STABLE_CLASS, + CLASS_BUT +} QUERY_CLASS; + typedef enum enum_INSERT_MODE { PROGRESSIVE_INSERT_MODE, INTERLACE_INSERT_MODE, @@ -6557,18 +6563,21 @@ static void specified_sub_callback( } static TAOS_SUB* subscribeImpl( + QUERY_CLASS class, threadInfo *pThreadInfo, - char *sql, char* topic, bool restart) + char *sql, char* topic, bool restart, uint64_t interval) { TAOS_SUB* tsub = NULL; - if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { + if ((SPECIFIED_CLASS == class) + && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { tsub = taos_subscribe( pThreadInfo->taos, restart, topic, sql, specified_sub_callback, (void*)pThreadInfo, g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + } else if ((STABLE_CLASS == class) + && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { tsub = taos_subscribe( pThreadInfo->taos, restart, @@ -6578,7 +6587,7 @@ static TAOS_SUB* subscribeImpl( tsub = taos_subscribe( pThreadInfo->taos, restart, - topic, sql, NULL, NULL, 0); + topic, sql, NULL, NULL, interval); } if (tsub == NULL) { @@ -6629,9 +6638,14 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", + __func__, __LINE__, + pThreadInfo->threadID, + pThreadInfo->start_table_from, + pThreadInfo->end_table_to, i); sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"", i, pThreadInfo->querySeq); - memset(subSqlstr,0,sizeof(subSqlstr)); + memset(subSqlstr, 0, sizeof(subSqlstr)); replaceChildTblName( g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlstr, i); @@ -6644,8 +6658,10 @@ static void *superSubscribe(void *sarg) { debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", __func__, __LINE__, pThreadInfo->threadID, subSqlstr); tsub[i] = subscribeImpl( + STABLE_CLASS, pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart); + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); return NULL; @@ -6687,8 +6703,10 @@ static void *superSubscribe(void *sarg) { g_queryInfo.superQueryInfo.subscribeKeepProgress); consumed[i]= 0; tsub[i] = subscribeImpl( + STABLE_CLASS, pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval ); if (NULL == tsub[i]) { taos_close(pThreadInfo->taos); @@ -6744,10 +6762,12 @@ static void *specifiedSubscribe(void *sarg) { g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } - tsub = subscribeImpl(pThreadInfo, + tsub = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart); + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; @@ -6777,11 +6797,12 @@ static void *specifiedSubscribe(void *sarg) { taos_unsubscribe(tsub, g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); tsub = subscribeImpl( - pThreadInfo, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, - g_queryInfo.specifiedQueryInfo.subscribeRestart - ); + SPECIFIED_CLASS, + pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], + topic, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + g_queryInfo.specifiedQueryInfo.subscribeInterval); if (NULL == tsub) { taos_close(pThreadInfo->taos); return NULL; @@ -6833,7 +6854,7 @@ static int subscribeTestProcess() { //==== create threads for query for specified table if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount); } else { @@ -6870,10 +6891,10 @@ static int subscribeTestProcess() { } //==== create threads for super table query - if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + if (g_queryInfo.superQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, super table query sqlCount %"PRIu64".\n", __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount); + g_queryInfo.superQueryInfo.sqlCount); } else { if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { @@ -6906,8 +6927,8 @@ static int subscribeTestProcess() { b = ntables % threads; } - uint64_t startFrom = 0; for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + uint64_t startFrom = 0; for (int j = 0; j < threads; j++) { uint64_t seq = i * threads + j; threadInfo *t_info = infosOfStable + seq; From 7316fb89796120fc274700f109612eb1eade2cbd Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 11:06:28 +0800 Subject: [PATCH 21/56] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeMain.c | 1 + src/dnode/src/dnodeSystem.c | 2 ++ src/inc/mnode.h | 2 ++ src/mnode/inc/mnodeSdb.h | 1 + src/mnode/src/mnodeMain.c | 6 ++++++ src/mnode/src/mnodeSdb.c | 12 ++++++++++++ 8 files changed, 26 insertions(+) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 2f4aa4c2b2..abb4d39ee1 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -142,6 +142,7 @@ extern int32_t tsMonitorInterval; extern int8_t tsEnableStream; // internal +extern int8_t tsCompactMnodeWal; extern int8_t tsPrintAuth; extern int8_t tscEmbedded; extern char configDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index db97c3a5af..ab036b021c 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -176,6 +176,7 @@ int32_t tsMonitorInterval = 30; // seconds int8_t tsEnableStream = 1; // internal +int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; int8_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 410e6bb188..48680e20d9 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -40,6 +40,7 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" #include "module.h" +#include "mnode.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..ee37ffdcbb 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { + tsCompactMnodeWal = 1; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2495a42ba2..c66dc60a1b 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -73,6 +73,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); +int32_t mnodeCompactWal(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index e4e4a7a054..13dd8bd06b 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -106,6 +106,7 @@ int32_t sdbGetId(void *pTable); uint64_t sdbGetVersion(); bool sdbCheckRowDeleted(void *pTable, void *pRow); +int32_t mnodeCompactWal(); #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 7ef0488c42..76eebe0d01 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,6 +57,12 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; +/* +static SStep tsMnodeCompactSteps[] = { + {"cluster", mnodeInitCluster, NULL}, +}; +*/ + static void mnodeInitTimer(); static void mnodeCleanupTimer(); static bool mnodeNeedStart() ; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 505d3c519c..64c7713164 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -450,6 +450,12 @@ int32_t sdbInit() { } tsSdbMgmt.status = SDB_STATUS_SERVING; + + if (tsCompactMnodeWal) { + mnodeCompactWal(); + exit(EXIT_SUCCESS); + } + return TSDB_CODE_SUCCESS; } @@ -1138,3 +1144,9 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; } + +int32_t mnodeCompactWal() { + sdbInfo("vgId:1, start compact mnode wal..."); + + return 0; +} \ No newline at end of file From 4697cdfdafbc3e639a253f65fbae6bae4ea5c1a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 14:01:32 +0800 Subject: [PATCH 22/56] Hotfix/sangshuduo/td 4238 taosdemo async subscribe (#6170) * [TD-4238]: taosdemo async subscribe. * [TD-4238]: taosdemo async subscribe subsribe sql command do not use aggregation functions. * [TD-4238]: taosdemo async subscribe. interval. * [TD-4238]: taosdemo async subscribe. fix super table sub result file. * fix specified sub sync mode. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f8d3f83d47..ae7ffd16b9 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6682,13 +6682,18 @@ static void *superSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms res = taos_consume(tsub[i]); if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); appendResultToFile(res, pThreadInfo->fp); } consumed[i] ++; @@ -6783,9 +6788,15 @@ static void *specifiedSubscribe(void *sarg) { continue; } - taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms res = taos_consume(tsub); if (res) { + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) && (consumed >= From a12b0e71755e2d3fce52c3c646e427aa86e02f69 Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 15:12:34 +0800 Subject: [PATCH 23/56] [TD-1920]add mnode compact wal functions --- src/inc/mnode.h | 1 + src/mnode/inc/mnodeAcct.h | 2 ++ src/mnode/inc/mnodeCluster.h | 2 ++ src/mnode/inc/mnodeDb.h | 2 ++ src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 1 + src/mnode/inc/mnodeTable.h | 1 + src/mnode/inc/mnodeUser.h | 2 ++ src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 26 +++++++++++++++ src/mnode/src/mnodeCluster.c | 24 ++++++++++++++ src/mnode/src/mnodeDb.c | 27 ++++++++++++++++ src/mnode/src/mnodeDnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeMain.c | 16 ++++++++-- src/mnode/src/mnodeMnode.c | 27 ++++++++++++++++ src/mnode/src/mnodeSdb.c | 27 ++++++++++++++++ src/mnode/src/mnodeTable.c | 62 ++++++++++++++++++++++++++++++++++++ src/mnode/src/mnodeUser.c | 27 ++++++++++++++++ src/mnode/src/mnodeVgroup.c | 27 ++++++++++++++++ 20 files changed, 301 insertions(+), 3 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index c66dc60a1b..203ac57469 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -74,6 +74,7 @@ void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t mnodeCompactWal(); +int32_t mnodeCompactComponents(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 595dcca413..522070e909 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -35,6 +35,8 @@ void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb); void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser); void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser); +int32_t mnodeCompactAccts(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h index a5af544dc2..db258ae6d6 100644 --- a/src/mnode/inc/mnodeCluster.h +++ b/src/mnode/inc/mnodeCluster.h @@ -25,6 +25,8 @@ void mnodeCleanupCluster(); void mnodeUpdateClusterId(); const char* mnodeGetClusterId(); +int32_t mnodeCompactCluster(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index da0865833d..0fa1a15e2d 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -41,6 +41,8 @@ void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); void mnodeDropAllDbs(SAcctObj *pAcct); +int32_t mnodeCompactDbs(); + // util func void mnodeAddSuperTableIntoDb(SDbObj *pDb); void mnodeRemoveSuperTableFromDb(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index fa1995254e..2db794a173 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -77,6 +77,7 @@ void * mnodeGetDnodeByEp(char *ep); void mnodeUpdateDnode(SDnodeObj *pDnode); int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg); +int32_t mnodeCompactDnodes(); extern int32_t tsAccessSquence; #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index ffdec02eb6..66e9eb1e0e 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -50,6 +50,7 @@ char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); +int32_t mnodeCompactMnodes(); #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 13dd8bd06b..2ae0c47902 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,6 +92,7 @@ void sdbUpdateMnodeRoles(); int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); +int32_t sdbInsertCompactRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index bf04f26a90..c583a60c7a 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -36,6 +36,7 @@ void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); +int32_t mnodeCompactTables(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 156bc7aaeb..b8f0805120 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -33,6 +33,8 @@ char * mnodeGetUserFromMsg(void *pMnodeMsg); int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg); void mnodeDropAllUsers(SAcctObj *pAcct); +int32_t mnodeCompactUsers(); + #ifdef __cplusplus } #endif diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7b798c23f8..73b0e6ae1b 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -32,6 +32,7 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb); void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb); void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); +int32_t mnodeCompactVgroups(); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); void mnodeCancelGetNextVgroup(void *pIter); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index afe474df6b..64cfa28917 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -238,6 +238,32 @@ static int32_t mnodeCreateRootAcct() { return sdbInsertRow(&row); } +int32_t mnodeCompactAccts() { + void *pIter = NULL; + SAcctObj *pAcct = NULL; + + mInfo("start to compact accts table..."); + + while (1) { + pIter = mnodeGetNextAcct(pIter, &pAcct); + if (pAcct == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsAcctSdb, + .pObj = pAcct, + }; + + mInfo("compact accts %s", pAcct->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact accts table..."); + + return 0; +} + #ifndef _ACCT int32_t acctInit() { return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 169d2ebd9d..8b9af6d493 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -237,3 +237,27 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, pShow->numOfReads += numOfRows; return numOfRows; } + +int32_t mnodeCompactCluster() { + SClusterObj *pCluster = NULL; + void *pIter; + + mInfo("start to compact cluster table..."); + + pIter = mnodeGetNextCluster(NULL, &pCluster); + while (pCluster) { + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsClusterSdb, + .pObj = pCluster, + }; + + sdbInsertCompactRow(&row); + + pIter = mnodeGetNextCluster(pIter, &pCluster); + } + + mInfo("end to compact cluster table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 8af20aa862..b5663bc00c 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1271,3 +1271,30 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } + +int32_t mnodeCompactDbs() { + void *pIter = NULL; + SDbObj *pDb = NULL; + + mInfo("start to compact dbs table..."); + + while (1) { + pIter = mnodeGetNextDb(pIter, &pDb); + if (pDb == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), + }; + + mInfo("compact dbs %s", pDb->name); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dbs table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 3525bcac18..51f16e4bc6 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -1270,3 +1270,30 @@ char* dnodeRoles[] = { "vnode", "any" }; + +int32_t mnodeCompactDnodes() { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + + mInfo("start to compact dnodes table..."); + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsDnodeSdb, + .pObj = pDnode, + .rowSize = sizeof(SDnodeObj), + }; + + mInfo("compact dnode %d", pDnode->dnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact dnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 76eebe0d01..01b83bec19 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -57,11 +57,16 @@ static SStep tsMnodeSteps[] = { {"show", mnodeInitShow, mnodeCleanUpShow} }; -/* static SStep tsMnodeCompactSteps[] = { - {"cluster", mnodeInitCluster, NULL}, + {"cluster", mnodeCompactCluster, NULL}, + {"dnodes", mnodeCompactDnodes, NULL}, + {"mnodes", mnodeCompactMnodes, NULL}, + {"accts", mnodeCompactAccts, NULL}, + {"users", mnodeCompactUsers, NULL}, + {"dbs", mnodeCompactDbs, NULL}, + {"vgroups", mnodeCompactVgroups, NULL}, + {"tables", mnodeCompactTables, NULL}, }; -*/ static void mnodeInitTimer(); static void mnodeCleanupTimer(); @@ -77,6 +82,11 @@ static int32_t mnodeInitComponents() { return dnodeStepInit(tsMnodeSteps, stepSize); } +int32_t mnodeCompactComponents() { + int32_t stepSize = sizeof(tsMnodeCompactSteps) / sizeof(SStep); + return dnodeStepInit(tsMnodeCompactSteps, stepSize); +} + int32_t mnodeStartSystem() { if (tsMgmtIsRunning) { mInfo("mnode module already started..."); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ca6d6400ae..ddc9ea59c4 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -566,3 +566,30 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } + +int32_t mnodeCompactMnodes() { + void *pIter = NULL; + SMnodeObj *pMnode = NULL; + + mInfo("start to compact mnodes table..."); + + while (1) { + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsMnodeSdb, + .pObj = pMnode, + .rowSize = sizeof(SMnodeObj), + }; + + mInfo("compact mnode %d", pMnode->mnodeId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact mnodes table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 64c7713164..79081945d9 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -732,6 +732,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * } } +int32_t sdbInsertCompactRow(SSdbRow *pRow) { + SSdbTable *pTable = pRow->pTable; + if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; + return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); +} + int32_t sdbInsertRow(SSdbRow *pRow) { SSdbTable *pTable = pRow->pTable; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; @@ -1148,5 +1154,26 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + + // change wal to wal_bak dir + char temp[TSDB_FILENAME_LEN] = {0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; + sprintf(temp, "%s/wal_tmp", tsMnodeDir); + if (mkdir(temp, 0755) != 0 && errno != EEXIST) { + return -1; + } + tsSdbMgmt.wal = walOpen(temp, &walCfg); + walRenew(tsSdbMgmt.wal); + + // compact memory tables info to wal tmp dir + mnodeCompactComponents(); + + // close wal + walFsync(tsSdbMgmt.wal, true); + walClose(tsSdbMgmt.wal); + return 0; } \ No newline at end of file diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2a8e941fcb..2e5d80b787 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3242,3 +3242,65 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro return numOfRows; } + +static int32_t mnodeCompactSuperTables() { + void *pIter = NULL; + SSTableObj *pTable = NULL; + + mInfo("start to compact super table..."); + + while (1) { + pIter = mnodeGetNextSuperTable(pIter, &pTable); + if (pTable == NULL) break; + + int32_t schemaSize = (pTable->numOfColumns + pTable->numOfTags) * sizeof(SSchema); + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsSuperTableSdb, + .pObj = pTable, + .rowSize = sizeof(SSTableObj) + schemaSize, + }; + + mInfo("compact super %ld", pTable->uid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact super table..."); + + return 0; +} + +static int32_t mnodeCompactChildTables() { + void *pIter = NULL; + SCTableObj *pTable = NULL; + + mInfo("start to compact child table..."); + + while (1) { + pIter = mnodeGetNextChildTable(pIter, &pTable); + if (pTable == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pObj = pTable, + .pTable = tsChildTableSdb, + }; + + mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact child table..."); + + return 0; +} + +int32_t mnodeCompactTables() { + mnodeCompactSuperTables(); + + mnodeCompactChildTables(); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index e77c1b3e59..c5c54791cf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -617,3 +617,30 @@ static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg) { return mnodeRetriveAuth(pAuthMsg->user, &pAuthRsp->spi, &pAuthRsp->encrypt, pAuthRsp->secret, pAuthRsp->ckey); } + +int32_t mnodeCompactUsers() { + void *pIter = NULL; + SUserObj *pUser = NULL; + + mInfo("start to compact users table..."); + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsUserSdb, + .pObj = pUser, + .rowSize = sizeof(SUserObj), + }; + + mInfo("compact users %s", pUser->user); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact users table..."); + + return 0; +} \ No newline at end of file diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7222c8d1a0..67532ad85a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -1302,3 +1302,30 @@ void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); cver[2] = (int8_t)(iver % 100); } + +int32_t mnodeCompactVgroups() { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + + mInfo("start to compact vgroups table..."); + + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + SSdbRow row = { + .type = SDB_OPER_GLOBAL, + .pTable = tsVgroupSdb, + .pObj = pVgroup, + .rowSize = sizeof(SVgObj), + }; + + mInfo("compact vgroups %d", pVgroup->vgId); + + sdbInsertCompactRow(&row); + } + + mInfo("end to compact vgroups table..."); + + return 0; +} \ No newline at end of file From 99f7359390539e659b7de331344015582aa1fbbe Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 16:04:53 +0800 Subject: [PATCH 24/56] [TD-1920]add mnode compact wal functions --- src/mnode/src/mnodeCluster.c | 2 +- src/mnode/src/mnodeMain.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 8b9af6d493..553e8446ab 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -249,7 +249,7 @@ int32_t mnodeCompactCluster() { SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsClusterSdb, - .pObj = pCluster, + .pObj = pCluster, }; sdbInsertCompactRow(&row); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 01b83bec19..d9abe16120 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,6 +59,8 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, + +/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -66,6 +68,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, +*/ }; static void mnodeInitTimer(); From f0b77fa3c3af7564054fe1c4be9210be0d5e8805 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Thu, 20 May 2021 17:16:09 +0800 Subject: [PATCH 25/56] [TD-4258]: speedup nodejs fetchall() with taos_fetch_block (#6165) --- src/connector/nodejs/nodetaos/cinterface.js | 254 ++++++++--------- src/connector/nodejs/nodetaos/cursor.js | 121 ++++----- src/connector/nodejs/package-lock.json | 285 -------------------- src/connector/nodejs/package.json | 2 +- 4 files changed, 188 insertions(+), 474 deletions(-) delete mode 100644 src/connector/nodejs/package-lock.json diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 43a08a800a..f3961e3787 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -9,7 +9,7 @@ const ffi = require('ffi-napi'); const ArrayType = require('ref-array-napi'); const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); -const errors = require ('./error'); +const errors = require('./error'); const TaosObjects = require('./taosobjects'); const { NULL_POINTER } = require('ref-napi'); @@ -22,7 +22,7 @@ function convertMicrosecondsToDatetime(time) { return new TaosObjects.TaosTimestamp(time * 0.001, true); } -function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { timestampConverter = convertMillisecondsToDatetime; if (micro == true) { timestampConverter = convertMicrosecondsToDatetime; @@ -44,14 +44,14 @@ function convertTimestamp(data, num_of_rows, nbytes = 0, offset = 0, micro=false } return res; } -function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = new Array(data.length); for (let i = 0; i < data.length; i++) { if (data[i] == 0) { res[i] = false; } - else if (data[i] == 1){ + else if (data[i] == 1) { res[i] = true; } else if (data[i] == FieldTypes.C_BOOL_NULL) { @@ -60,29 +60,29 @@ function convertBool(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertTinyint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,1); + let d = data.readIntLE(currOffset, 1); res.push(d == FieldTypes.C_TINYINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertSmallint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; while (currOffset < data.length) { - let d = data.readIntLE(currOffset,2); + let d = data.readIntLE(currOffset, 2); res.push(d == FieldTypes.C_SMALLINT_NULL ? null : d); currOffset += nbytes; } return res; } -function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -93,7 +93,7 @@ function convertInt(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -104,7 +104,7 @@ function convertBigint(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -115,7 +115,7 @@ function convertFloat(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -126,7 +126,7 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let currOffset = 0; @@ -142,7 +142,7 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { } return res; } -function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { +function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro = false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; @@ -153,23 +153,23 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { // Object with all the relevant converters from pblock data to javascript readable data let convertFunctions = { - [FieldTypes.C_BOOL] : convertBool, - [FieldTypes.C_TINYINT] : convertTinyint, - [FieldTypes.C_SMALLINT] : convertSmallint, - [FieldTypes.C_INT] : convertInt, - [FieldTypes.C_BIGINT] : convertBigint, - [FieldTypes.C_FLOAT] : convertFloat, - [FieldTypes.C_DOUBLE] : convertDouble, - [FieldTypes.C_BINARY] : convertBinary, - [FieldTypes.C_TIMESTAMP] : convertTimestamp, - [FieldTypes.C_NCHAR] : convertNchar + [FieldTypes.C_BOOL]: convertBool, + [FieldTypes.C_TINYINT]: convertTinyint, + [FieldTypes.C_SMALLINT]: convertSmallint, + [FieldTypes.C_INT]: convertInt, + [FieldTypes.C_BIGINT]: convertBigint, + [FieldTypes.C_FLOAT]: convertFloat, + [FieldTypes.C_DOUBLE]: convertDouble, + [FieldTypes.C_BINARY]: convertBinary, + [FieldTypes.C_TIMESTAMP]: convertTimestamp, + [FieldTypes.C_NCHAR]: convertNchar } // Define TaosField structure var char_arr = ArrayType(ref.types.char); var TaosField = Struct({ - 'name': char_arr, - }); + 'name': char_arr, +}); TaosField.fields.name.type.size = 65; TaosField.defineProperty('type', ref.types.char); TaosField.defineProperty('bytes', ref.types.short); @@ -183,7 +183,7 @@ TaosField.defineProperty('bytes', ref.types.short); * @classdesc The CTaosInterface is the interface through which Node.JS communicates data back and forth with TDengine. It is not advised to * access this class directly and use it unless you understand what these functions do. */ -function CTaosInterface (config = null, pass = false) { +function CTaosInterface(config = null, pass = false) { ref.types.char_ptr = ref.refType(ref.types.char); ref.types.void_ptr = ref.refType(ref.types.void); ref.types.void_ptr2 = ref.refType(ref.types.void_ptr); @@ -196,64 +196,65 @@ function CTaosInterface (config = null, pass = false) { taoslibname = 'libtaos'; } this.libtaos = ffi.Library(taoslibname, { - 'taos_options': [ ref.types.int, [ ref.types.int , ref.types.void_ptr ] ], - 'taos_init': [ ref.types.void, [ ] ], + 'taos_options': [ref.types.int, [ref.types.int, ref.types.void_ptr]], + 'taos_init': [ref.types.void, []], //TAOS *taos_connect(char *ip, char *user, char *pass, char *db, int port) - 'taos_connect': [ ref.types.void_ptr, [ ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int ] ], + 'taos_connect': [ref.types.void_ptr, [ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.char_ptr, ref.types.int]], //void taos_close(TAOS *taos) - 'taos_close': [ ref.types.void, [ ref.types.void_ptr ] ], - //int *taos_fetch_lengths(TAOS_RES *taos); - 'taos_fetch_lengths': [ ref.types.void_ptr, [ ref.types.void_ptr ] ], + 'taos_close': [ref.types.void, [ref.types.void_ptr]], + //int *taos_fetch_lengths(TAOS_RES *res); + 'taos_fetch_lengths': [ref.types.void_ptr, [ref.types.void_ptr]], //int taos_query(TAOS *taos, char *sqlstr) - 'taos_query': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr ] ], - //int taos_affected_rows(TAOS *taos) - 'taos_affected_rows': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_query': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr]], + //int taos_affected_rows(TAOS_RES *res) + 'taos_affected_rows': [ref.types.int, [ref.types.void_ptr]], //int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) - 'taos_fetch_block': [ ref.types.int, [ ref.types.void_ptr, ref.types.void_ptr] ], + 'taos_fetch_block': [ref.types.int, [ref.types.void_ptr, ref.types.void_ptr]], //int taos_num_fields(TAOS_RES *res); - 'taos_num_fields': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_num_fields': [ref.types.int, [ref.types.void_ptr]], //TAOS_ROW taos_fetch_row(TAOS_RES *res) //TAOS_ROW is void **, but we set the return type as a reference instead to get the row - 'taos_fetch_row': [ ref.refType(ref.types.void_ptr2), [ ref.types.void_ptr ] ], + 'taos_fetch_row': [ref.refType(ref.types.void_ptr2), [ref.types.void_ptr]], + 'taos_print_row': [ref.types.int, [ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], //int taos_result_precision(TAOS_RES *res) - 'taos_result_precision': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_result_precision': [ref.types.int, [ref.types.void_ptr]], //void taos_free_result(TAOS_RES *res) - 'taos_free_result': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_free_result': [ref.types.void, [ref.types.void_ptr]], //int taos_field_count(TAOS *taos) - 'taos_field_count': [ ref.types.int, [ ref.types.void_ptr ] ], + 'taos_field_count': [ref.types.int, [ref.types.void_ptr]], //TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) - 'taos_fetch_fields': [ ref.refType(TaosField), [ ref.types.void_ptr ] ], + 'taos_fetch_fields': [ref.refType(TaosField), [ref.types.void_ptr]], //int taos_errno(TAOS *taos) - 'taos_errno': [ ref.types.int, [ ref.types.void_ptr] ], + 'taos_errno': [ref.types.int, [ref.types.void_ptr]], //char *taos_errstr(TAOS *taos) - 'taos_errstr': [ ref.types.char_ptr, [ ref.types.void_ptr] ], + 'taos_errstr': [ref.types.char_ptr, [ref.types.void_ptr]], //void taos_stop_query(TAOS_RES *res); - 'taos_stop_query': [ ref.types.void, [ ref.types.void_ptr] ], + 'taos_stop_query': [ref.types.void, [ref.types.void_ptr]], //char *taos_get_server_info(TAOS *taos); - 'taos_get_server_info': [ ref.types.char_ptr, [ ref.types.void_ptr ] ], + 'taos_get_server_info': [ref.types.char_ptr, [ref.types.void_ptr]], //char *taos_get_client_info(); - 'taos_get_client_info': [ ref.types.char_ptr, [ ] ], + 'taos_get_client_info': [ref.types.char_ptr, []], // ASYNC // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) - 'taos_query_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_query_a': [ref.types.void, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr]], // void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); - 'taos_fetch_rows_a': [ ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr ]], + 'taos_fetch_rows_a': [ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.void_ptr]], // Subscription //TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) - 'taos_subscribe': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int] ], + 'taos_subscribe': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.int, ref.types.char_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.void_ptr, ref.types.int]], // TAOS_RES *taos_consume(TAOS_SUB *tsub) - 'taos_consume': [ ref.types.void_ptr, [ref.types.void_ptr] ], + 'taos_consume': [ref.types.void_ptr, [ref.types.void_ptr]], //void taos_unsubscribe(TAOS_SUB *tsub); - 'taos_unsubscribe': [ ref.types.void, [ ref.types.void_ptr ] ], + 'taos_unsubscribe': [ref.types.void, [ref.types.void_ptr]], // Continuous Query //TAOS_STREAM *taos_open_stream(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), // int64_t stime, void *param, void (*callback)(void *)); - 'taos_open_stream': [ ref.types.void_ptr, [ ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr ] ], + 'taos_open_stream': [ref.types.void_ptr, [ref.types.void_ptr, ref.types.char_ptr, ref.types.void_ptr, ref.types.int64, ref.types.void_ptr, ref.types.void_ptr]], //void taos_close_stream(TAOS_STREAM *tstr); - 'taos_close_stream': [ ref.types.void, [ ref.types.void_ptr ] ] + 'taos_close_stream': [ref.types.void, [ref.types.void_ptr]] }); if (pass == false) { @@ -264,7 +265,7 @@ function CTaosInterface (config = null, pass = false) { try { this._config = ref.allocCString(config); } - catch(err){ + catch (err) { throw "Attribute Error: config is expected as a str"; } } @@ -276,38 +277,38 @@ function CTaosInterface (config = null, pass = false) { return this; } CTaosInterface.prototype.config = function config() { - return this._config; - } -CTaosInterface.prototype.connect = function connect(host=null, user="root", password="taosdata", db=null, port=0) { - let _host,_user,_password,_db,_port; - try { + return this._config; +} +CTaosInterface.prototype.connect = function connect(host = null, user = "root", password = "taosdata", db = null, port = 0) { + let _host, _user, _password, _db, _port; + try { _host = host != null ? ref.allocCString(host) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: host is expected as a str"; } try { _user = ref.allocCString(user) } - catch(err) { + catch (err) { throw "Attribute Error: user is expected as a str"; } try { _password = ref.allocCString(password); } - catch(err) { + catch (err) { throw "Attribute Error: password is expected as a str"; } try { _db = db != null ? ref.allocCString(db) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: db is expected as a str"; } try { _port = ref.alloc(ref.types.int, port); } - catch(err) { + catch (err) { throw TypeError("port is expected as an int") } let connection = this.libtaos.taos_connect(_host, _user, _password, _db, _port); @@ -324,10 +325,10 @@ CTaosInterface.prototype.close = function close(connection) { console.log("Connection is closed"); } CTaosInterface.prototype.query = function query(connection, sql) { - return this.libtaos.taos_query(connection, ref.allocCString(sql)); + return this.libtaos.taos_query(connection, ref.allocCString(sql)); } -CTaosInterface.prototype.affectedRows = function affectedRows(connection) { - return this.libtaos.taos_affected_rows(connection); +CTaosInterface.prototype.affectedRows = function affectedRows(result) { + return this.libtaos.taos_affected_rows(result); } CTaosInterface.prototype.useResult = function useResult(result) { @@ -337,8 +338,8 @@ CTaosInterface.prototype.useResult = function useResult(result) { pfields = ref.reinterpret(pfields, this.fieldsCount(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -347,11 +348,10 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let pblock = this.libtaos.taos_fetch_row(result); - let num_of_rows = 1; - if (ref.isNull(pblock) == true) { - return {block:null, num_of_rows:0}; + let pblock = ref.NULL_POINTER; + let num_of_rows = this.libtaos.taos_fetch_block(result, pblock); + if (ref.isNull(pblock.deref()) == true) { + return { block: null, num_of_rows: 0 }; } var fieldL = this.libtaos.taos_fetch_lengths(result); @@ -359,10 +359,10 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); var fieldlens = []; - + if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 4, i * 4); let len = plen.readInt32LE(0); fieldlens.push(len); } @@ -370,21 +370,23 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { let blocks = new Array(fields.length); blocks.fill(null); - //num_of_rows = Math.abs(num_of_rows); + num_of_rows = Math.abs(num_of_rows); let offset = 0; + let ptr = pblock.deref(); + for (let i = 0; i < fields.length; i++) { - pdata = ref.reinterpret(pblock,8,i*8); - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); - } + pdata = ref.reinterpret(ptr, 8, i * 8); + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, num_of_rows, fieldlens[i], offset, isMicro); + } } - return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} + return { blocks: blocks, num_of_rows } } CTaosInterface.prototype.fetchRow = function fetchRow(result, fields) { let row = this.libtaos.taos_fetch_row(result); @@ -414,7 +416,7 @@ CTaosInterface.prototype.errStr = function errStr(result) { // Async CTaosInterface.prototype.query_a = function query_a(connection, sql, callback, param = ref.ref(ref.NULL)) { // void taos_query_a(TAOS *taos, char *sqlstr, void (*fp)(void *param, TAOS_RES *, int), void *param) - callback = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], callback); + callback = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], callback); this.libtaos.taos_query_a(connection, ref.allocCString(sql), callback, param); return param; } @@ -439,46 +441,46 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, var fieldL = cti.libtaos.taos_fetch_lengths(result); var fieldlens = []; if (ref.isNull(fieldL) == false) { - - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 8, i*8); - let len = ref.get(plen,0,ref.types.int32); + + for (let i = 0; i < fields.length; i++) { + let plen = ref.reinterpret(fieldL, 8, i * 8); + let len = ref.get(plen, 0, ref.types.int32); fieldlens.push(len); } } - if (numOfRows2 > 0){ + if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if(ref.isNull(pdata.readPointer())){ - blocks[i] = new Array(); - }else{ - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; - } + if (ref.isNull(pdata.readPointer())) { + blocks[i] = new Array(); + } else { + if (!convertFunctions[fields[i]['type']]) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row, 8, i * 8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.types.int ], asyncCallbackWrapper); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.types.int], asyncCallbackWrapper); this.libtaos.taos_fetch_rows_a(result, asyncCallbackWrapper, param); return param; } // Fetch field meta data by result handle -CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { +CTaosInterface.prototype.fetchFields_a = function fetchFields_a(result) { let pfields = this.fetchFields(result); let pfieldscount = this.numFields(result); let fields = []; if (ref.isNull(pfields) == false) { - pfields = ref.reinterpret(pfields, 68 * pfieldscount , 0); + pfields = ref.reinterpret(pfields, 68 * pfieldscount, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 64 = name //65 = type, 66 - 67 = bytes - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,65,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 65, i)), type: pfields[i + 65], bytes: pfields[i + 66] }) @@ -488,7 +490,7 @@ CTaosInterface.prototype.fetchFields_a = function fetchFields_a (result) { } // Stop a query by result handle CTaosInterface.prototype.stopQuery = function stopQuery(result) { - if (result != null){ + if (result != null) { this.libtaos.taos_stop_query(result); } else { @@ -509,13 +511,13 @@ CTaosInterface.prototype.subscribe = function subscribe(connection, restart, top try { sql = sql != null ? ref.allocCString(sql) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw "Attribute Error: sql is expected as a str"; } try { topic = topic != null ? ref.allocCString(topic) : ref.alloc(ref.types.char_ptr, ref.NULL); } - catch(err) { + catch (err) { throw TypeError("topic is expected as a str"); } @@ -539,8 +541,8 @@ CTaosInterface.prototype.consume = function consume(subscription) { pfields = ref.reinterpret(pfields, this.numFields(result) * 68, 0); for (let i = 0; i < pfields.length; i += 68) { //0 - 63 = name //64 - 65 = bytes, 66 - 67 = type - fields.push( { - name: ref.readCString(ref.reinterpret(pfields,64,i)), + fields.push({ + name: ref.readCString(ref.reinterpret(pfields, 64, i)), bytes: pfields[i + 64], type: pfields[i + 66] }) @@ -548,7 +550,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { } let data = []; - while(true) { + while (true) { let { blocks, num_of_rows } = this.fetchBlock(result, fields); if (num_of_rows == 0) { break; @@ -559,7 +561,7 @@ CTaosInterface.prototype.consume = function consume(subscription) { for (let j = 0; j < fields.length; j++) { rowBlock[j] = blocks[j][i]; } - data[data.length-1] = (rowBlock); + data[data.length - 1] = (rowBlock); } } return { data: data, fields: fields, result: result }; @@ -570,11 +572,11 @@ CTaosInterface.prototype.unsubscribe = function unsubscribe(subscription) { } // Continuous Query -CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime,stoppingCallback, param = ref.ref(ref.NULL)) { +CTaosInterface.prototype.openStream = function openStream(connection, sql, callback, stime, stoppingCallback, param = ref.ref(ref.NULL)) { try { sql = ref.allocCString(sql); } - catch(err) { + catch (err) { throw "Attribute Error: sql string is expected as a str"; } var cti = this; @@ -587,7 +589,7 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb let offset = 0; if (numOfRows2 > 0) { for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { + if (!convertFunctions[fields[i]['type']]) { throw new errors.DatabaseError("Invalid data type returned from database"); } blocks[i] = convertFunctions[fields[i]['type']](row, numOfRows2, fields[i]['bytes'], offset, isMicro); @@ -596,8 +598,8 @@ CTaosInterface.prototype.openStream = function openStream(connection, sql, callb } callback(param2, result2, blocks, fields); } - asyncCallbackWrapper = ffi.Callback(ref.types.void, [ ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2) ], asyncCallbackWrapper); - asyncStoppingCallbackWrapper = ffi.Callback( ref.types.void, [ ref.types.void_ptr ], stoppingCallback); + asyncCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr, ref.types.void_ptr, ref.refType(ref.types.void_ptr2)], asyncCallbackWrapper); + asyncStoppingCallbackWrapper = ffi.Callback(ref.types.void, [ref.types.void_ptr], stoppingCallback); let streamHandle = this.libtaos.taos_open_stream(connection, sql, asyncCallbackWrapper, stime, param, asyncStoppingCallbackWrapper); if (ref.isNull(streamHandle)) { throw new errors.TDError('Failed to open a stream with TDengine'); diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index e18e6c2500..f879d89d48 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,7 +1,7 @@ const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') -const errors = require ('./error') +const errors = require('./error') const TaosQuery = require('./taosquery') const { PerformanceObserver, performance } = require('perf_hooks'); module.exports = TDengineCursor; @@ -22,7 +22,7 @@ module.exports = TDengineCursor; * @property {fields} - Array of the field objects in order from left to right of the latest data retrieved * @since 1.0.0 */ -function TDengineCursor(connection=null) { +function TDengineCursor(connection = null) { //All parameters are store for sync queries only. this._rowcount = -1; this._connection = null; @@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { callback = options; } if (typeof options != 'object') options = {} @@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback } TDengineCursor.prototype._createAffectedResponse = function (num, time) { - return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype._createSetResponse = function (num, time) { - return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; + return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)"; } TDengineCursor.prototype.executemany = function executemany() { @@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first"); } - let data = []; + let num_of_rows = this._chandle.affectedRows(this._result); + let data = new Array(num_of_rows); + this._rowcount = 0; - //let nodetime = 0; + let time = 0; const obs = new PerformanceObserver((items) => { time += items.getEntries()[0].duration; performance.clearMarks(); }); - /* - const obs2 = new PerformanceObserver((items) => { - nodetime += items.getEntries()[0].duration; - performance.clearMarks(); - }); - obs2.observe({ entryTypes: ['measure'] }); - performance.mark('nodea'); - */ obs.observe({ entryTypes: ['measure'] }); performance.mark('A'); - while(true) { - + while (true) { let blockAndRows = this._chandle.fetchBlock(this._result, this._fields); + // console.log(blockAndRows); + // break; let block = blockAndRows.blocks; let num_of_rows = blockAndRows.num_of_rows; if (num_of_rows == 0) { @@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { this._rowcount += num_of_rows; let numoffields = this._fields.length; for (let i = 0; i < num_of_rows; i++) { - data.push([]); - + // data.push([]); + let rowBlock = new Array(numoffields); for (let j = 0; j < numoffields; j++) { rowBlock[j] = block[j][i]; } - data[data.length-1] = (rowBlock); + data[this._rowcount - num_of_rows + i] = (rowBlock); + // data.push(rowBlock); } } + performance.mark('B'); performance.measure('query', 'A', 'B'); let response = this._createSetResponse(this._rowcount, time) console.log(response); - // this._connection._clearResultSet(); + // this._connection._clearResultSet(); let fields = this.fields; this._reset_result(); this.data = data; @@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) { * @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query * @since 1.0.0 */ -TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) { +TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) { if (operation == undefined) { throw new errors.ProgrammingError('No operation passed as argument'); return null; } - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal } if (resCode >= 0) { -// let fieldCount = cr._chandle.numFields(res2); -// if (fieldCount == 0) { -// //cr._chandle.freeResult(res2); -// return res2; -// } -// else { -// return res2; -// } + // let fieldCount = cr._chandle.numFields(res2); + // if (fieldCount == 0) { + // //cr._chandle.freeResult(res2); + // return res2; + // } + // else { + // return res2; + // } return res2; } @@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal * }) */ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) { - if (typeof options == 'function') { + if (typeof options == 'function') { //we expect the parameter after callback to be param param = callback; callback = options; @@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb for (let k = 0; k < fields.length; k++) { rowBlock[k] = block[k][j]; } - data[data.length-1] = rowBlock; + data[data.length - 1] = rowBlock; } } cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks! - callback(param2, result2, numOfRows2, {data:data,fields:fields}); + callback(param2, result2, numOfRows2, { data: data, fields: fields }); } } ref.writeObject(buf, 0, param); param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param - return {param:param,result:result}; + return { param: param, result: result }; } /** * Stop a query given the result handle. @@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) { */ TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) { while (true) { - let { data, fields, result} = this._chandle.consume(subscription); + let { data, fields, result } = this._chandle.consume(subscription); callback(data, fields, result); } } @@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) { * @return {Buffer} A buffer pointing to the stream handle * @since 1.3.0 */ - TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { - let buf = ref.alloc('Object'); - ref.writeObject(buf, 0, param); +TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) { + let buf = ref.alloc('Object'); + ref.writeObject(buf, 0, param); - let asyncCallbackWrapper = function (param2, result2, blocks, fields) { - let data = []; - let num_of_rows = blocks[0].length; - for (let j = 0; j < num_of_rows; j++) { - data.push([]); - let rowBlock = new Array(fields.length); - for (let k = 0; k < fields.length; k++) { - rowBlock[k] = blocks[k][j]; - } - data[data.length-1] = rowBlock; - } - callback(param2, result2, blocks, fields); - } - return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); - } - /** - * Close a stream - * @param {Buffer} - A buffer pointing to the handle of the stream to be closed - * @since 1.3.0 - */ - TDengineCursor.prototype.closeStream = function closeStream(stream) { - this._chandle.closeStream(stream); - } + let asyncCallbackWrapper = function (param2, result2, blocks, fields) { + let data = []; + let num_of_rows = blocks[0].length; + for (let j = 0; j < num_of_rows; j++) { + data.push([]); + let rowBlock = new Array(fields.length); + for (let k = 0; k < fields.length; k++) { + rowBlock[k] = blocks[k][j]; + } + data[data.length - 1] = rowBlock; + } + callback(param2, result2, blocks, fields); + } + return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf); +} +/** + * Close a stream + * @param {Buffer} - A buffer pointing to the handle of the stream to be closed + * @since 1.3.0 + */ +TDengineCursor.prototype.closeStream = function closeStream(stream) { + this._chandle.closeStream(stream); +} diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json deleted file mode 100644 index 9ca174ccd1..0000000000 --- a/src/connector/nodejs/package-lock.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "name": "td2.0-connector", - "version": "2.0.6", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "array-index": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", - "integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=", - "requires": { - "debug": "^2.2.0", - "es6-symbol": "^3.0.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "requires": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "requires": { - "ms": "2.1.2" - } - }, - "es5-ext": { - "version": "0.10.53", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", - "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "requires": { - "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.3", - "next-tick": "~1.0.0" - } - }, - "es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "requires": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "requires": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, - "ext": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", - "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "requires": { - "type": "^2.0.0" - }, - "dependencies": { - "type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", - "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" - } - } - }, - "ffi-napi": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", - "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", - "requires": { - "debug": "^4.1.1", - "get-uv-event-loop-napi-h": "^1.0.5", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1", - "ref-napi": "^2.0.1", - "ref-struct-di": "^1.1.0" - }, - "dependencies": { - "ref-napi": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", - "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - } - } - }, - "get-symbol-from-current-process-h": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", - "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" - }, - "get-uv-event-loop-napi-h": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", - "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", - "requires": { - "get-symbol-from-current-process-h": "^1.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "next-tick": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" - }, - "node-addon-api": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", - "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" - }, - "node-gyp-build": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", - "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" - }, - "ref-array-napi": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", - "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", - "requires": { - "array-index": "1", - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "ref-napi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", - "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", - "requires": { - "debug": "^4.1.1", - "get-symbol-from-current-process-h": "^1.0.2", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - } - }, - "ref-struct-di": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", - "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "ref-struct-napi": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", - "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", - "requires": { - "debug": "2", - "ref-napi": "^1.4.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "ref-napi": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", - "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", - "requires": { - "debug": "^3.1.0", - "node-addon-api": "^2.0.0", - "node-gyp-build": "^4.2.1" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", - "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - } - } - }, - "type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - } - } -} diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index b39ce2c17d..d21b62108b 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.6", + "version": "2.0.7", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "directories": { From 9958b3d8d27130525b1c3b0a4f95bf08d23b620a Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 20 May 2021 18:19:27 +0800 Subject: [PATCH 26/56] [TD-1920]add mnode compact wal functions --- src/common/inc/tglobal.h | 2 ++ src/common/src/tglobal.c | 2 ++ src/dnode/src/dnodeMain.c | 11 +++++++++++ src/mnode/src/mnodeMain.c | 4 +--- src/mnode/src/mnodeSdb.c | 33 +++++++++++++++++++++++++-------- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index abb4d39ee1..e07c3611d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -149,6 +149,8 @@ extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; extern char tsMnodeDir[]; +extern char tsMnodeBakDir[]; +extern char tsMnodeTmpDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ab036b021c..d6bbc288ad 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -183,6 +183,8 @@ char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeTmpDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeBakDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 48680e20d9..ee4dfcf852 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -217,6 +217,17 @@ static int32_t dnodeInitStorage() { sprintf(tsDnodeDir, "%s/dnode", tsDataDir); // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + if (tsCompactMnodeWal == 1) { + sprintf(tsMnodeTmpDir, "%s/mnode_tmp", tsDataDir); + tfsRmdir(tsMnodeTmpDir); + if (dnodeCreateDir(tsMnodeTmpDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsMnodeTmpDir, strerror(errno)); + return -1; + } + + sprintf(tsMnodeBakDir, "%s/mnode_bak", tsDataDir); + //tfsRmdir(tsMnodeBakDir); + } //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index d9abe16120..d3511a4e62 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -59,8 +59,6 @@ static SStep tsMnodeSteps[] = { static SStep tsMnodeCompactSteps[] = { {"cluster", mnodeCompactCluster, NULL}, - -/* {"dnodes", mnodeCompactDnodes, NULL}, {"mnodes", mnodeCompactMnodes, NULL}, {"accts", mnodeCompactAccts, NULL}, @@ -68,7 +66,7 @@ static SStep tsMnodeCompactSteps[] = { {"dbs", mnodeCompactDbs, NULL}, {"vgroups", mnodeCompactVgroups, NULL}, {"tables", mnodeCompactTables, NULL}, -*/ + }; static void mnodeInitTimer(); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 79081945d9..9e3b87671e 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -20,6 +20,7 @@ #include "tutil.h" #include "tref.h" #include "tbn.h" +#include "tfs.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" @@ -1154,26 +1155,42 @@ int32_t sdbGetReplicaNum() { int32_t mnodeCompactWal() { sdbInfo("vgId:1, start compact mnode wal..."); - // close wal + // close old wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); - // change wal to wal_bak dir - char temp[TSDB_FILENAME_LEN] = {0}; + // reset version,then compacted wal log can start from version 1 + tsSdbMgmt.version = 0; + + // change wal to wal_tmp dir SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; - sprintf(temp, "%s/wal_tmp", tsMnodeDir); - if (mkdir(temp, 0755) != 0 && errno != EEXIST) { - return -1; - } + char temp[TSDB_FILENAME_LEN] = {0}; + sprintf(temp, "%s/wal", tsMnodeTmpDir); tsSdbMgmt.wal = walOpen(temp, &walCfg); walRenew(tsSdbMgmt.wal); // compact memory tables info to wal tmp dir - mnodeCompactComponents(); + if (mnodeCompactComponents() != 0) { + tfsRmdir(tsMnodeTmpDir); + return -1; + } // close wal walFsync(tsSdbMgmt.wal, true); walClose(tsSdbMgmt.wal); + // rename old wal to wal_bak + if (taosRename(tsMnodeDir, tsMnodeBakDir) != 0) { + return -1; + } + + // rename wal_tmp to wal + if (taosRename(tsMnodeTmpDir, tsMnodeDir) != 0) { + return -1; + } + + // del wal_tmp dir + sdbInfo("vgId:1, compact mnode wal success"); + return 0; } \ No newline at end of file From 2a5805b52817939331ae11e5350a5f26f91f318c Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Thu, 20 May 2021 18:39:28 +0800 Subject: [PATCH 27/56] [TD-4181] : update format of FAQ doc. --- documentation20/cn/13.faq/docs.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index e561f91c94..8d18ae269d 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -26,17 +26,17 @@ ## 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/jdbcdriver找不到动态链接库/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/950.html) ## 3. 创建数据表时提示more dnodes are needed -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/创建数据表时提示more-dnodes-are-needed/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/965.html) ## 4. 如何让TDengine crash时生成core文件？ -请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/tdengine-crash时生成core文件的方法/) +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/974.html) -## 5. 遇到错误"Unable to establish connection", 我怎么办？ +## 5. 遇到错误“Unable to establish connection”, 我怎么办？ 客户端遇到连接故障，请按照下面的步骤进行检查： @@ -51,13 +51,13 @@ 4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 -5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 +5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件。如果部署的是TDengine集群，客户端需要能ping通所有集群节点的FQDN。 6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +8. 对于Windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障 @@ -70,7 +70,8 @@ 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 -## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？ +## 6. 遇到错误“Unexpected generic error in RPC”或者“Unable to resolve FQDN”，我怎么办？ + 产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： 1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 @@ -102,7 +103,7 @@ TDengine 目前尚不支持删除功能，未来根据用户需求可能会支 批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 -## 12. windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ +## 12. Windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： ```JAVA @@ -115,15 +116,15 @@ Connection = DriverManager.getConnection(url, properties); ## 13.JDBC报错： the excuted SQL is not a DML or a DDL？ 请更新至最新的JDBC驱动 -```JAVA +```xml com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.27 ``` -## 14. taos connect failed, reason: invalid timestamp +## 14. taos connect failed, reason: invalid timestamp 常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 @@ -157,7 +158,8 @@ ALTER LOCAL RESETLOG; 其含义是，清空本机所有由客户端生成的日志文件。 -## 18. 时间戳的时区信息是怎样处理的？ + +## 18. 时间戳的时区信息是怎样处理的？ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端无关。具体来说，客户端会对 SQL 语句中的时间戳进行时区转换，转为 UTC 时区（即 Unix 时间戳——Unix Timestamp）再交由服务端进行写入和查询；在读取数据时，服务端也是采用 UTC 时区提供原始数据，客户端收到后再根据本地设置，把时间戳转换为本地系统所要求的时区进行显示。 @@ -167,12 +169,13 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。 -## 19. TDengine 都会用到哪些网络端口？ + +## 19. TDengine 都会用到哪些网络端口？ 在 TDengine 2.0 版本中，会用到以下这些网络端口（以默认端口 6030 为前提进行说明，如果修改了配置文件中的设置，那么这里列举的端口都会出现变化），管理员可以参考这里的信息调整防火墙设置： | 协议 | 默认端口 | 用途说明 | 修改方法 | -| --- | --------- | ------------------------------- | ------------------------------ | +| :--- | :-------- | :---------------------------------- | :------------------------------- | | TCP | 6030 | 客户端与服务端之间通讯。 | 由配置文件设置 serverPort 决定。 | | TCP | 6035 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | | TCP | 6040 | 多节点集群的节点间数据同步。 | 随 serverPort 端口变化。 | From 824eb0f3133168b47f2850fc3968914be87b030e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 18:41:35 +0800 Subject: [PATCH 28/56] [TD-4272]: taosdemo specify columns. (#6174) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ebc1074cba..4ca4031b1f 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -198,6 +198,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -537,6 +539,7 @@ static int taosRandom() #endif // ifdef Windows +static void prompt(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -692,8 +695,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -737,7 +741,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -881,20 +884,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -911,12 +925,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -929,16 +943,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -3464,10 +3477,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -5812,19 +5822,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -6172,6 +6176,13 @@ static void *readMetric(void *sarg) { return NULL; } +static void prompt() +{ + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } +} static int insertTestProcess() { @@ -7397,6 +7408,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; From 608e67826de2c23cddab295f5fc9c7498c4bcc7a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 18:41:55 +0800 Subject: [PATCH 29/56] [TD-4272]: taosdemo specify columns. (#6173) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 70 ++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index ae7ffd16b9..34302ac2b0 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -189,6 +189,8 @@ typedef struct { } SColDes; /* Used by main to communicate with parse_opt. */ +static char *g_dupstr = NULL; + typedef struct SArguments_S { char * metaFile; uint32_t test_mode; @@ -526,7 +528,6 @@ static int taosRandom() #endif // ifdef Windows static void prompt(); -static void prompt2(); static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -679,8 +680,9 @@ static void printHelp() { "The data_type of columns, default: INT,INT,INT,INT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s\n", indent, "-l", indent, - "The number of columns per record. Default is 4."); + printf("%s%s%s%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is 4. Max values is ", + MAX_NUM_DATATYPE); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -724,7 +726,6 @@ static bool isStringNumber(char *input) } static void parse_args(int argc, char *argv[], SArguments *arguments) { - char **sptr; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { @@ -851,20 +852,31 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->database = argv[++i]; } else if (strcmp(argv[i], "-l") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); - exit(EXIT_FAILURE); + if (argc == i+1) { + if (!isStringNumber(argv[i+1])) { + printHelp(); + errorPrint("%s", "\n\t-l need a number following!\n"); + exit(EXIT_FAILURE); + } } arguments->num_of_CPR = atoi(argv[++i]); + + if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + prompt(); + arguments->num_of_CPR = MAX_NUM_DATATYPE; + } + + for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + arguments->datatype[col] = NULL; + } + } else if (strcmp(argv[i], "-b") == 0) { if (argc == i+1) { printHelp(); errorPrint("%s", "\n\t-b need valid string following!\n"); exit(EXIT_FAILURE); } - sptr = arguments->datatype; ++i; if (strstr(argv[i], ",") == NULL) { // only one col @@ -881,12 +893,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[0] = argv[i]; + arguments->datatype[0] = argv[i]; } else { // more than one col int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; + g_dupstr = strdup(argv[i]); + char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { if (strcasecmp(token, "INT") @@ -899,16 +911,15 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { printHelp(); - free(dupstr); + free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - sptr[index++] = token; + arguments->datatype[index++] = token; token = strsep(&running, ","); if (index >= MAX_NUM_DATATYPE) break; } - free(dupstr); - sptr[index] = NULL; + arguments->datatype[index] = NULL; } } else if (strcmp(argv[i], "-w") == 0) { if ((argc == i+1) || @@ -3449,7 +3460,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.interlace_rows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - prompt2(); + prompt(); g_args.interlace_rows = g_args.num_of_RPR; } } else if (!interlaceRows) { @@ -5759,19 +5770,13 @@ static void startMultiThreadInsertData(int threads, char* db_name, && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) > superTblInfo->childTblCount)) { printf("WARNING: specified offset + limit > child table count!\n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } if ((superTblInfo->childTblExists != TBL_NO_EXISTS) && (0 == superTblInfo->childTblLimit)) { printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + prompt(); } superTblInfo->childTblName = (char*)calloc(1, @@ -6093,19 +6098,11 @@ static void *readMetric(void *sarg) { static void prompt() { if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); (void)getchar(); } } -static void prompt2() -{ - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } -} - static int insertTestProcess() { setupForAnsiEscape(); @@ -7325,6 +7322,9 @@ int main(int argc, char *argv[]) { } else { testCmdLine(); } + + if (g_dupstr) + free(g_dupstr); } return 0; From 4b016b75bb026f86bd5db56910409d6e2c17fa95 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 20 May 2021 19:08:42 +0800 Subject: [PATCH 30/56] [TD-4263]: update performance test script --- tests/perftest-scripts/perftest-query.sh | 2 +- tests/pytest/tools/taosdemoPerformance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 9a16084683..c6d4687ed7 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,7 +74,7 @@ function runQueryPerfTest { python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT + #python3 perfbenchmark/joinPerformance.py | tee -a $PERFORMANCE_TEST_REPORT } diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index b50180e2b3..a32cba167e 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -104,7 +104,7 @@ class taosdemoPerformace: return output def insertData(self): - os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") From 243a3d16b6c23a0c4cce60a0fc07fe53d6d54dd8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 20 May 2021 23:25:12 +0800 Subject: [PATCH 31/56] Feature/sangshuduo/td 4068 taosdemo stmt (#6178) * merge with develop branch. change query/tests/CMakeLists.txt to allow unused function and variable. * refactor data generating. * refactor. * refactor. * refactor. * refactor. * refactor Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 252 ++++++++++++++++++++++-------------- 1 file changed, 156 insertions(+), 96 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4ca4031b1f..1d3b0d1ade 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -61,6 +61,8 @@ extern char configDir[]; #define QUERY_JSON_NAME "query.json" #define SUBSCRIBE_JSON_NAME "subscribe.json" +#define STR_INSERT_INTO "INSERT INTO " + enum TEST_MODE { INSERT_TEST, // 0 QUERY_TEST, // 1 @@ -5093,7 +5095,8 @@ static int generateStbSQLHead( return len; } -static int64_t generateInterlaceDataBuffer( +static int64_t generateStbInterlaceData( + SSuperTable *superTblInfo, char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, @@ -5103,12 +5106,11 @@ static int64_t generateInterlaceDataBuffer( { assert(buffer); char *pstr = buffer; - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; int headLen = generateStbSQLHead( superTblInfo, tableName, tableSeq, pThreadInfo->db_name, - pstr, *pRemainderBufLen); + pstr, *pRemainderBufLen); if (headLen <= 0) { return 0; @@ -5126,24 +5128,15 @@ static int64_t generateInterlaceDataBuffer( pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); - int64_t k; - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { startTime = taosGetTimestamp(pThreadInfo->time_precision); - } + } - k = generateStbDataTail( + int64_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); - } else { - startTime = 1500000000000; - k = generateDataTailWithoutStb( - batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, - startTime, - /* &(pThreadInfo->samplePos), */&dataLen); - } if (k == batchPerTbl) { pstr += dataLen; @@ -5159,33 +5152,67 @@ static int64_t generateInterlaceDataBuffer( return k; } -static int64_t generateProgressiveDataBuffer( +static int64_t generateInterlaceDataWithoutStb( + char *tableName, uint64_t batchPerTbl, + uint64_t tableSeq, + char *dbName, char *buffer, + int64_t insertRows, + uint64_t *pRemainderBufLen) +{ + assert(buffer); + char *pstr = buffer; + + int headLen = generateSQLHeadWithoutStb( + tableName, dbName, + pstr, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen = 0; + + int64_t startTime = 1500000000000; + int64_t k = generateDataTailWithoutStb( + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &dataLen); + + if (k == batchPerTbl) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + __func__, __LINE__, k, batchPerTbl); + pstr -= headLen; + pstr[0] = '\0'; + k = 0; + } + + return k; +} + +static int64_t generateStbProgressiveData( + SSuperTable *superTblInfo, char *tableName, int64_t tableSeq, - threadInfo *pThreadInfo, char *buffer, + char *dbName, char *buffer, int64_t insertRows, uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *pRemainderBufLen) { - SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - assert(buffer != NULL); char *pstr = buffer; memset(buffer, 0, *pRemainderBufLen); - int64_t headLen; - - if (superTblInfo) { - headLen = generateStbSQLHead( + int64_t headLen = generateStbSQLHead( superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, + tableName, tableSeq, dbName, buffer, *pRemainderBufLen); - } else { - headLen = generateSQLHeadWithoutStb( - tableName, pThreadInfo->db_name, - buffer, *pRemainderBufLen); - } if (headLen <= 0) { return 0; @@ -5194,22 +5221,43 @@ static int64_t generateProgressiveDataBuffer( *pRemainderBufLen -= headLen; int64_t dataLen; - int64_t k; - if (superTblInfo) { - k = generateStbDataTail(superTblInfo, + return generateStbDataTail(superTblInfo, g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); - } else { - k = generateDataTailWithoutStb( +} + +static int64_t generateProgressiveDataWithoutStb( + char *tableName, + int64_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, + int64_t *pRemainderBufLen) +{ + assert(buffer != NULL); + char *pstr = buffer; + + memset(buffer, 0, *pRemainderBufLen); + + int64_t headLen = generateSQLHeadWithoutStb( + tableName, pThreadInfo->db_name, + buffer, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + pstr += headLen; + *pRemainderBufLen -= headLen; + + int64_t dataLen; + + return generateDataTailWithoutStb( g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom, startTime, /*pSamplePos, */&dataLen); - } - - return k; } static void printStatPerThread(threadInfo *pThreadInfo) @@ -5228,6 +5276,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int64_t insertRows; uint64_t interlaceRows; + uint64_t maxSqlLen; + int64_t nTimeStampStep; + uint64_t insert_interval; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; @@ -5240,58 +5291,27 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { interlaceRows = superTblInfo->interlaceRows; } + maxSqlLen = superTblInfo->maxSqlLen; + nTimeStampStep = superTblInfo->timeStampStep; + insert_interval = superTblInfo->insertInterval; } else { insertRows = g_args.num_of_DPT; interlaceRows = g_args.interlace_rows; + maxSqlLen = g_args.max_sql_len; + nTimeStampStep = DEFAULT_TIMESTAMP_STEP; + insert_interval = g_args.insert_interval; } - if (interlaceRows > insertRows) - interlaceRows = insertRows; - - if (interlaceRows > g_args.num_of_RPR) - interlaceRows = g_args.num_of_RPR; - - int progOrInterlace; - - if (interlaceRows > 0) { - progOrInterlace= INTERLACE_INSERT_MODE; - } else { - progOrInterlace = PROGRESSIVE_INSERT_MODE; - } - - uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; - pThreadInfo->buffer = calloc(maxSqlLen, 1); - if (NULL == pThreadInfo->buffer) { - errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", - __func__, __LINE__, maxSqlLen, strerror(errno)); - return NULL; - } - - char tableName[TSDB_TABLE_NAME_LEN]; - - pThreadInfo->totalInsertRows = 0; - pThreadInfo->totalAffectedRows = 0; - - int64_t nTimeStampStep = - superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - - uint64_t insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = UINT64_MAX; - - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); - uint64_t endTs; - - uint64_t tableSeq = pThreadInfo->start_table_from; - debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, pThreadInfo->ntables, insertRows); - int64_t startTime = pThreadInfo->start_time; + if (interlaceRows > insertRows) + interlaceRows = insertRows; + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; uint64_t batchPerTbl = interlaceRows; uint64_t batchPerTblTimes; @@ -5303,13 +5323,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { batchPerTblTimes = 1; } + pThreadInfo->buffer = calloc(maxSqlLen, 1); + if (NULL == pThreadInfo->buffer) { + errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); + return NULL; + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + uint64_t generatedRecPerTbl = 0; bool flagSleep = true; uint64_t sleepTimeTotal = 0; - char *strInsertInto = "insert into "; - int nInsertBufLen = strlen(strInsertInto); - while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampMs(); @@ -5321,13 +5358,16 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { char *pstr = pThreadInfo->buffer; - int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; uint64_t recOfBatch = 0; for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + char tableName[TSDB_TABLE_NAME_LEN]; + getTableName(tableName, pThreadInfo, tableSeq); if (0 == strlen(tableName)) { errorPrint("[%d] %s() LN%d, getTableName return null\n", @@ -5337,13 +5377,25 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } uint64_t oldRemainderLen = remainderBufLen; - int64_t generated = generateInterlaceDataBuffer( - tableName, batchPerTbl, i, batchPerTblTimes, - tableSeq, - pThreadInfo, pstr, - insertRows, - startTime, - &remainderBufLen); + + int64_t generated; + if (superTblInfo) { + generated = generateStbInterlaceData( + superTblInfo, + tableName, batchPerTbl, i, batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, + startTime, + &remainderBufLen); + } else { + generated = generateInterlaceDataWithoutStb( + tableName, batchPerTbl, + tableSeq, + pThreadInfo->db_name, pstr, + insertRows, + &remainderBufLen); + } debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", pThreadInfo->threadID, __func__, __LINE__, generated); @@ -5364,8 +5416,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - if (progOrInterlace == INTERLACE_INSERT_MODE) { - if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; @@ -5383,7 +5434,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) break; - } } verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", @@ -5511,18 +5561,28 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { int64_t remainderBufLen = maxSqlLen; char *pstr = pThreadInfo->buffer; - int nInsertBufLen = strlen("insert into "); - int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into "); + int len = snprintf(pstr, + strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO); pstr += len; remainderBufLen -= len; - int64_t generated = generateProgressiveDataBuffer( + int64_t generated; + if (superTblInfo) { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows, + i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } else { + generated = generateProgressiveDataWithoutStb( tableName, tableSeq, pThreadInfo, pstr, insertRows, i, start_time, &(pThreadInfo->samplePos), &remainderBufLen); + } if (generated > 0) i += generated; else From d324edadfe03a70a77cf73523802e8caf5e1086e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 06:24:17 +0800 Subject: [PATCH 32/56] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 (#6177) * [TD-4136]: taosdemo max records per req < 32767 * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. * add extra prompt. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 34302ac2b0..34f5c87c44 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3494,6 +3494,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3979,10 +3984,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!interlaceRows) { From c820edc32ad6578148c2e3de21a7fa3f3caba1e9 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 21 May 2021 08:33:44 +0800 Subject: [PATCH 33/56] fix bug --- src/query/inc/qHistogram.h | 2 +- src/query/src/qHistogram.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 7742d151a0..3b5c2b4cfb 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -40,7 +40,7 @@ typedef struct SHeapEntry { } SHeapEntry; typedef struct SHistogramInfo { - int32_t numOfElems; + int64_t numOfElems; int32_t numOfEntries; int32_t maxEntries; double min; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index ae25a75234..5fa35d0ee5 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -446,7 +446,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto) { } void tHistogramPrint(SHistogramInfo* pHisto) { - printf("total entries: %d, elements: %d\n", pHisto->numOfEntries, pHisto->numOfElems); + printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); #if defined(USE_ARRAYLIST) for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); From d2de6c5fa3db80271944965fe1fcc6f19baed7fc Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 21 May 2021 08:56:27 +0800 Subject: [PATCH 34/56] [TD-3746]:save grant objects to monitor database --- src/inc/monitor.h | 3 ++- src/inc/taoserror.h | 3 +++ src/plugins/monitor/src/monMain.c | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/inc/monitor.h b/src/inc/monitor.h index 1aefb0b848..d2e5e06487 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -54,7 +54,8 @@ void monCleanupSystem(); void monSaveAcctLog(SAcctMonitorObj *pMonObj); void monSaveLog(int32_t level, const char *const format, ...); void monExecuteSQL(char *sql); - +typedef void (*MonExecuteSQLCbFP)(void *param, TAOS_RES *, int code); +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param); #ifdef __cplusplus } #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ce6f7c4f22..d3ac0ce280 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -427,6 +427,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +// monitor +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") + #ifdef __cplusplus } #endif diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 94af8e3ecd..d9f7d81ebd 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -417,3 +417,13 @@ void monExecuteSQL(char *sql) { monDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); } + +void monExecuteSQLWithResultCallback(char *sql, MonExecuteSQLCbFP callback, void* param) { + if (tsMonitor.conn == NULL) { + callback(param, NULL, TSDB_CODE_MON_CONNECTION_INVALID); + return; + } + + monDebug("execute sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, callback, param); +} From e38362c037c864f1752975fdf792202983a3c306 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 21 May 2021 09:43:05 +0800 Subject: [PATCH 35/56] [TD-1920]fix compile error --- src/mnode/src/mnodeTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2e5d80b787..4d3125a2d1 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3261,7 +3261,7 @@ static int32_t mnodeCompactSuperTables() { .rowSize = sizeof(SSTableObj) + schemaSize, }; - mInfo("compact super %ld", pTable->uid); + mInfo("compact super %" PRIu64, pTable->uid); sdbInsertCompactRow(&row); } @@ -3287,7 +3287,7 @@ static int32_t mnodeCompactChildTables() { .pTable = tsChildTableSdb, }; - mInfo("compact child %ld:%d", pTable->uid, pTable->tid); + mInfo("compact child %" PRIu64 ":%d", pTable->uid, pTable->tid); sdbInsertCompactRow(&row); } From 891886ce56565d88bec3e0bb461a50f1c0888917 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 13:11:52 +0800 Subject: [PATCH 36/56] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6175) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 35 ++++++++-------------- tests/pytest/tools/taosdemoTestWithJson.py | 22 +++++++------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 1d3b0d1ade..b0e1dd28a6 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1088,10 +1088,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { printf("# Print debug info: %d\n", arguments->debug_print); printf("# Print verbose info: %d\n", arguments->verbose_print); printf("###################################################################\n"); - if (!arguments->answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + + prompt(); } } @@ -3513,6 +3511,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { __func__, __LINE__); goto PARSE_OVER; } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n", + numRecPerReq->valueint, MAX_RECORDS_PER_REQ); + printf(" number of records per request value will be set to %d\n\n", + MAX_RECORDS_PER_REQ); + prompt(); numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; @@ -3536,9 +3539,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_args.answer_yes = false; } } else if (!answerPrompt) { - g_args.answer_yes = false; + g_args.answer_yes = true; // default is no, mean answer_yes. } else { - printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n"); goto PARSE_OVER; } @@ -4007,10 +4010,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", g_args.num_of_RPR); - if (!g_args.answer_yes) { - printf(" press Enter key to continue or Ctrl-C to stop."); - (void)getchar(); - } + prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; } } else if (!stbInterlaceRows) { @@ -6263,10 +6263,7 @@ static int insertTestProcess() { if (g_fpOfInsertResult) printfInsertMetaToFile(g_fpOfInsertResult); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); init_rand_data(); @@ -6536,10 +6533,7 @@ static int queryTestProcess() { &g_queryInfo.superQueryInfo.childTblCount); } - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void)getchar(); - } + prompt(); if (g_args.debug_print || g_args.verbose_print) { printfQuerySystemInfo(taos); @@ -6977,10 +6971,7 @@ static int subscribeTestProcess() { printfQueryMeta(); resetAfterAnsiEscape(); - if (!g_args.answer_yes) { - printf("Press enter key to continue\n\n"); - (void) getchar(); - } + prompt(); TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index f57af9ce5c..b2ecd54976 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -23,32 +23,32 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if "taosd" in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + buildPath = root[: len(root) - len("/build/bin")] break return buildPath - + def run(self): tdSql.prepare() buildPath = self.getBuildPath() - if (buildPath == ""): + if buildPath == "": tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -f tools/insert.json" % binPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -f tools/insert.json -y" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") From 972a75c3faaa9f05f7b978e614f055e31c1d6449 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 15:15:28 +0800 Subject: [PATCH 37/56] Hotfix/sangshuduo/td 3913 mips compile support for develop (#6184) * [TD-3913]: mips compile support. * [TD-3913]: mips64 support. * fix osSignal.c * fix tcrc32c.c * fix cmake. * fix osInc.h * fix gcc 4.8 compile error. Co-authored-by: Shuduo Sang --- cmake/platform.inc | 6 +++ src/os/inc/osMips64.h | 87 ++++++++++++++++++++++++++++++++++++ src/os/src/detail/osSignal.c | 10 ++--- src/util/src/tcrc32c.c | 2 +- 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 src/os/inc/osMips64.h diff --git a/cmake/platform.inc b/cmake/platform.inc index dcd0183e27..5f7391c996 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -102,6 +102,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_LINUX TRUE) SET(TD_LINUX_64 FALSE) SET(TD_ARM_64 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + SET(CPUTYPE "mips64") + MESSAGE(STATUS "Set CPUTYPE to mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) ENDIF () ELSE () diff --git a/src/os/inc/osMips64.h b/src/os/inc/osMips64.h new file mode 100644 index 0000000000..ed7b08a311 --- /dev/null +++ b/src/os/inc/osMips64.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MIPS64_H +#define TDENGINE_OS_MIPS64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c index 4467a607b2..33cc39e112 100644 --- a/src/os/src/detail/osSignal.c +++ b/src/os/src/detail/osSignal.c @@ -25,14 +25,14 @@ typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); void taosSetSignal(int32_t signum, FSignalHandler sigfp) { - struct sigaction act = {{0}}; + struct sigaction act; memset(&act, 0, sizeof(act)); #if 1 act.sa_flags = SA_SIGINFO; act.sa_sigaction = (FLinuxSignalHandler)sigfp; -#else - act.sa_handler = sigfp; -#endif - sigaction(signum, &act, NULL); +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); } void taosIgnSignal(int32_t signum) { diff --git a/src/util/src/tcrc32c.c b/src/util/src/tcrc32c.c index 4009973a9f..d2b63eb9ee 100644 --- a/src/util/src/tcrc32c.c +++ b/src/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_ARM_ +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) #include #endif From 83187703a8d4ee3f8d9f0a44eaebc358b2e894bd Mon Sep 17 00:00:00 2001 From: root Date: Fri, 21 May 2021 11:51:33 +0000 Subject: [PATCH 38/56] [TD-4182] specify column for single batch bind --- tests/script/api/stmtBatchTest.c | 975 ++++++++++++++++++++++++++++++- 1 file changed, 950 insertions(+), 25 deletions(-) diff --git a/tests/script/api/stmtBatchTest.c b/tests/script/api/stmtBatchTest.c index 3ce2fefb0d..8bd296db61 100644 --- a/tests/script/api/stmtBatchTest.c +++ b/tests/script/api/stmtBatchTest.c @@ -37,7 +37,7 @@ typedef struct { //void taosMsleep(int mseconds); -int g_rows = 0; +int g_runTimes = 5; unsigned long long getCurrentTime(){ @@ -1336,7 +1336,7 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { result = taos_query(taos, sql); int code = taos_errno(result); if (code != 0) { - printf("failed to query table, reason:%s\n", taos_errstr(result)); + printf("failed to query table: %s, reason:%s\n", tname, taos_errstr(result)); taos_free_result(result); return; } @@ -1358,9 +1358,9 @@ static void checkResult(TAOS *taos, char *tname, int printr, int expected) { } if (rows == expected) { - printf("%d rows are fetched as expectation\n", rows); + printf("%d rows are fetched as expectation from %s\n", rows, tname); } else { - printf("!!!expect %d rows, but %d rows are fetched\n", expected, rows); + printf("!!!expect %d rows, but %d rows are fetched from %s\n", expected, rows, tname); return; } @@ -1451,15 +1451,52 @@ static void prepareV_long(TAOS *taos, int schemaCase, int tableNum, int lenO -//void runcase(TAOS *taos, int idx) { -static void* runCase(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - - TAOS_STMT *stmt = NULL; +static void prepareVcolumn(TAOS *taos, int schemaCase, int tableNum, int lenOfBinaryDef, char* dbName) { + TAOS_RES *result; + int code; + char sqlstr[1024] = {0}; + sprintf(sqlstr, "drop database if exists %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); - (void)idx; + sprintf(sqlstr, "create database %s;", dbName); + result = taos_query(taos, sqlstr); + code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + sprintf(sqlstr, "use %s;", dbName); + result = taos_query(taos, sqlstr); + taos_free_result(result); + + // create table + for (int i = 0 ; i < tableNum; i++) { + char buf[1024]; + if (schemaCase) { + sprintf(buf, "create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp)", i, lenOfBinaryDef, lenOfBinaryDef) ; + } else { + sprintf(buf, "create table m%d (ts timestamp, b int)", i) ; + } + + result = taos_query(taos, buf); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + } + +} + +//void runcase(TAOS *taos, int idx) { +static void runCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; int tableNum; int lenOfBinaryDef; @@ -1975,7 +2012,7 @@ static void* runCase(void *para) { } #endif - return NULL; + return ; } @@ -2166,14 +2203,8 @@ static int stmt_bind_case_001_long(TAOS_STMT *stmt, int tableNum, int rowsOfPerC return 0; } -static void* runCase_long(void *para) { - ThreadInfo* tInfo = (ThreadInfo *)para; - TAOS *taos = tInfo->taos; - int idx = tInfo->idx; - +static void runCase_long(TAOS *taos) { TAOS_STMT *stmt = NULL; - - (void)idx; int tableNum; int lenOfBinaryDef; @@ -2228,7 +2259,7 @@ static void* runCase_long(void *para) { prepareV_long(taos, 1, tableNum, lenOfBinaryDef); totalRowsPerTbl = 0; - for (int i = 0; i < 30000; i++) { + for (int i = 0; i < g_runTimes; i++) { stmt = taos_stmt_init(taos); stmt_bind_case_001_long(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum, &startTs); @@ -2245,17 +2276,895 @@ static void* runCase_long(void *para) { } #endif - return NULL; + return; } +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into ? (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_002(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[i+1].buffer_length = sizeof(int8_t); + params[i+1].buffer = v->b; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + params[i+2].buffer_type = TSDB_DATA_TYPE_INT; + params[i+2].buffer_length = sizeof(int32_t); + params[i+2].buffer = v->v4; + params[i+2].length = NULL; + params[i+2].is_null = is_null; + params[i+2].num = rowsOfPerColum; + + params[i+3].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[i+3].buffer_length = sizeof(float); + params[i+3].buffer = v->f4; + params[i+3].length = NULL; + params[i+3].is_null = is_null; + params[i+3].num = rowsOfPerColum; + + params[i+4].buffer_type = TSDB_DATA_TYPE_BINARY; + params[i+4].buffer_length = (uintptr_t)lenOfBinaryDef; + params[i+4].buffer = v->br; + params[i+4].length = lb; + params[i+4].is_null = is_null; + params[i+4].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into ? (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + char buf[32]; + sprintf(buf, "m%d", zz); + code = taos_stmt_set_tbname(stmt, buf); + if (code != 0){ + printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +/*=======================*/ +/* +test scene: insert into tb1 (ts,f1) values (?,?) +*/ +static int stmt_specifyCol_bind_case_001_maxRows(TAOS_STMT *stmt, int tableNum, int rowsOfPerColum, int bingNum, int lenOfBinaryDef, int lenOfBinaryAct, int columnNum) { + sampleValue* v = (sampleValue *)calloc(1, sizeof(sampleValue)); + + int totalRowsPerTbl = rowsOfPerColum * bingNum; + + v->ts = (int64_t *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * tableNum)); + v->br = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + v->nr = (char *)malloc(sizeof(int64_t) * (size_t)(totalRowsPerTbl * lenOfBinaryDef)); + + int *lb = (int *)malloc(MAX_ROWS_OF_PER_COLUMN * sizeof(int)); + + TAOS_MULTI_BIND *params = calloc(1, sizeof(TAOS_MULTI_BIND) * (size_t)(bingNum * columnNum * (tableNum+1) * rowsOfPerColum)); + char* is_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + char* no_null = malloc(sizeof(char) * MAX_ROWS_OF_PER_COLUMN); + + int64_t tts = 1591060628000; + + for (int i = 0; i < rowsOfPerColum; ++i) { + lb[i] = lenOfBinaryAct; + no_null[i] = 0; + is_null[i] = (i % 10 == 2) ? 1 : 0; + v->b[i] = (int8_t)(i % 2); + v->v1[i] = (int8_t)((i+1) % 2); + v->v2[i] = (int16_t)i; + v->v4[i] = (int32_t)(i+1); + v->v8[i] = (int64_t)(i+2); + v->f4[i] = (float)(i+3); + v->f8[i] = (double)(i+4); + char tbuf[MAX_BINARY_DEF_LEN]; + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "binary-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->br + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + memset(tbuf, 0, MAX_BINARY_DEF_LEN); + sprintf(tbuf, "nchar-%d-0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", i%10); + memcpy(v->nr + i*lenOfBinaryDef, tbuf, (size_t)lenOfBinaryAct); + v->ts2[i] = tts + i; + } + + int i = 0; + for (int j = 0; j < bingNum * tableNum; j++) { + params[i+0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[i+0].buffer_length = sizeof(int64_t); + params[i+0].buffer = &v->ts[j*rowsOfPerColum]; + params[i+0].length = NULL; + params[i+0].is_null = no_null; + params[i+0].num = rowsOfPerColum; + + params[i+1].buffer_type = TSDB_DATA_TYPE_INT; + params[i+1].buffer_length = sizeof(int32_t); + params[i+1].buffer = v->v4; + params[i+1].length = NULL; + params[i+1].is_null = is_null; + params[i+1].num = rowsOfPerColum; + + i+=columnNum; + } + + //int64_t tts = 1591060628000; + for (int i = 0; i < totalRowsPerTbl * tableNum; ++i) { + v->ts[i] = tts + i; + } + + unsigned long long starttime = getCurrentTime(); + +// create table m%d (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, br binary(%d), nr nchar(%d), ts2 timestamp) + char *sql = "insert into m0 (ts,b,v4,f4,br) values(?,?,?,?,?)"; + int code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + + int id = 0; + for (int l = 0; l < bingNum; l++) { + for (int zz = 0; zz < tableNum; zz++) { + //char buf[32]; + //sprintf(buf, "m%d", zz); + //code = taos_stmt_set_tbname(stmt, buf); + //if (code != 0){ + // printf("failed to execute taos_stmt_set_tbname. code:0x%x[%s]\n", code, tstrerror(code)); + // return -1; + //} + + for (int col=0; col < columnNum; ++col) { + code = taos_stmt_bind_single_param_batch(stmt, params + id, col); + if (code != 0){ + printf("failed to execute taos_stmt_bind_single_param_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + id++; + } + + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_add_batch. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("failed to execute taos_stmt_execute. code:0x%x[%s]\n", code, tstrerror(code)); + return -1; + } + } + + unsigned long long endtime = getCurrentTime(); + unsigned long long totalRows = (uint32_t)(totalRowsPerTbl * tableNum); + printf("insert total %d records, used %u seconds, avg:%u useconds per record\n", totalRows, (endtime-starttime)/1000000UL, (endtime-starttime)/totalRows); + + free(v->ts); + free(v->br); + free(v->nr); + free(v); + free(lb); + free(params); + free(is_null); + free(no_null); + + return 0; +} + +static void SpecifyColumnBatchCase(TAOS *taos) { + TAOS_STMT *stmt = NULL; + + int tableNum; + int lenOfBinaryDef; + int rowsOfPerColum; + int bingNum; + int lenOfBinaryAct; + int columnNum; + + int totalRowsPerTbl; + +//=======================================================================// +//=============================== single table ==========================// +//========== case 1: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 1 check result end\n\n"); +} +#endif + + //========== case 2: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2 check result end\n\n"); +} +#endif + + //========== case 2-1: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32767; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_1"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 2-1 check result end\n\n"); + } +#endif + //========== case 2-2: ======================// +#if 1 + { + printf("====case 2-2 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 32768; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 15; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db2_2"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 2-2 check result end\n\n"); + } +#endif + + + //========== case 3: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db3"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 3 check result end\n\n"); + } +#endif + + //========== case 4: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db4"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + //checkResult(taos, "m1", 0, totalRowsPerTbl); + //checkResult(taos, "m2", 0, totalRowsPerTbl); + //checkResult(taos, "m3", 0, totalRowsPerTbl); + //checkResult(taos, "m4", 0, totalRowsPerTbl); + //checkResult(taos, "m5", 0, totalRowsPerTbl); + //checkResult(taos, "m6", 0, totalRowsPerTbl); + //checkResult(taos, "m7", 0, totalRowsPerTbl); + //checkResult(taos, "m8", 0, totalRowsPerTbl); + //checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 4 check result end\n\n"); + } +#endif + +//=======================================================================// +//=============================== multi tables ==========================// + //========== case 5: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 16; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db5"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 5 check result end\n\n"); + } +#endif + + //========== case 6: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 1; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 20; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db6"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 6 check result end\n\n"); + } +#endif + + //========== case 7: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db7"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 7 check result end\n\n"); + } +#endif + + //========== case 8: ======================// +#if 1 +{ + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 40; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db8"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 8 check result end\n\n"); +} +#endif + + //=======================================================================// + //=============================== multi-rows to single table ==========================// + //========== case 9: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23740; + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db9"); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 9 check result end\n\n"); + } +#endif + + //========== case 10: ======================// +#if 0 + { + printf("====case 10 error test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 1; + rowsOfPerColum = 23741; // WAL size exceeds limit + bingNum = 1; + lenOfBinaryDef = 40; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareV(taos, 1, tableNum, lenOfBinaryDef); + stmt_specifyCol_bind_case_001(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("====case 10 check result end\n\n"); + } +#endif + + + //=======================================================================// + //=============================== multi tables, multi bind one same table ==========================// + //========== case 13: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 1; + bingNum = 5; + lenOfBinaryDef = 40; + lenOfBinaryAct = 28; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db13"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 13 check result end\n\n"); + } +#endif + + //========== case 14: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 5; + rowsOfPerColum = 5; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 33; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db14"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m1", 0, totalRowsPerTbl); + checkResult(taos, "m2", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m4", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 14 check result end\n\n"); + } +#endif + + + //========== case 15: ======================// +#if 1 + { + stmt = taos_stmt_init(taos); + + tableNum = 1000; + rowsOfPerColum = 10; + bingNum = 5; + lenOfBinaryDef = 1000; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db15"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m111", 0, totalRowsPerTbl); + checkResult(taos, "m222", 0, totalRowsPerTbl); + checkResult(taos, "m333", 0, totalRowsPerTbl); + checkResult(taos, "m500", 0, totalRowsPerTbl); + checkResult(taos, "m999", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 15 check result end\n\n"); + } +#endif + + //========== case 17: ======================// +#if 1 + { + //printf("case 17 test start\n"); + stmt = taos_stmt_init(taos); + + tableNum = 10; + rowsOfPerColum = 100; + bingNum = 1; + lenOfBinaryDef = 100; + lenOfBinaryAct = 8; + columnNum = 5; + + prepareVcolumn(taos, 1, tableNum, lenOfBinaryDef, "db17"); + stmt_specifyCol_bind_case_002(stmt, tableNum, rowsOfPerColum, bingNum, lenOfBinaryDef, lenOfBinaryAct, columnNum); + + totalRowsPerTbl = rowsOfPerColum * bingNum; + checkResult(taos, "m0", 0, totalRowsPerTbl); + checkResult(taos, "m3", 0, totalRowsPerTbl); + checkResult(taos, "m5", 0, totalRowsPerTbl); + checkResult(taos, "m8", 0, totalRowsPerTbl); + checkResult(taos, "m9", 0, totalRowsPerTbl); + taos_stmt_close(stmt); + printf("case 17 check result end\n\n"); + } +#endif + + return ; + +} int main(int argc, char *argv[]) { TAOS *taos; char host[32] = "127.0.0.1"; char* serverIp = NULL; - int threadNum = 2; + int threadNum = 1; // connect to server if (argc == 1) { @@ -2268,9 +3177,12 @@ int main(int argc, char *argv[]) } else if (argc == 4) { serverIp = argv[1]; threadNum = atoi(argv[2]); - g_rows = atoi(argv[3]); + g_runTimes = atoi(argv[3]); } + printf("server:%s, runTimes:%d\n\n", serverIp, g_runTimes); + +#if 0 printf("server:%s, threadNum:%d, rows:%d\n\n", serverIp, threadNum, g_rows); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), (size_t)threadNum); @@ -2287,7 +3199,8 @@ int main(int argc, char *argv[]) tInfo->taos = taos; tInfo->idx = i; if (0 == i) { - pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + //pthread_create(&(pThreadList[0]), NULL, runCase, (void *)tInfo); + pthread_create(&(pThreadList[0]), NULL, SpecifyColumnBatchCase, (void *)tInfo); } else if (1 == i){ pthread_create(&(pThreadList[0]), NULL, runCase_long, (void *)tInfo); } @@ -2300,6 +3213,18 @@ int main(int argc, char *argv[]) free(pThreadList); free(threadInfo); +#endif + + taos = taos_connect(serverIp, "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to TDengine, reason:%s\n", taos_errstr(taos)); + return -1; + } + + runCase(taos); + runCase_long(taos); + SpecifyColumnBatchCase(taos); + return 0; } From c74651a6cb906f71368357fe7acd61a260bae8d0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 21:35:41 +0800 Subject: [PATCH 39/56] Hotfix/sangshuduo/td 4136 taosdemo records morethan 32767 for develop (#6189) * [TD-4136]: taosdemo records per req more than 32767. for develop branch. * change taosRandom() to rand_bool() for bool data. * [TD-4136]: taosdemo check insert rows not more than 32767. check insert rows for progressive. * fix with answer_yes. * fix interlace rows checking position. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index e8075c69ba..6c1f56e44a 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3492,16 +3492,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.interlace_rows = interlaceRows->valueint; - - // rows per table need be less than insert batch - if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", - g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", - g_args.num_of_RPR); - prompt(); - g_args.interlace_rows = g_args.num_of_RPR; - } } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { @@ -3567,6 +3557,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", + g_args.num_of_RPR); + prompt(); + g_args.interlace_rows = g_args.num_of_RPR; + } + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); if (!dbs || dbs->type != cJSON_Array) { printf("ERROR: failed to read json, databases not found\n"); From fc394200e4a7bb01f36810da14cb93bf12ba2d01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 May 2021 23:20:42 +0800 Subject: [PATCH 40/56] Hotfix/sangshuduo/td 4240 taosdemo subscribe toomuch for develop (#6193) * [TD-4240]: taosdemo subscribe more than max query sql count. * [TD-4240]: taosdemo subscribe more than 100. fix tsub sequence bug. * [TD-4240]: taosdemo subscribe more than 100. fix auto create table. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 102 +++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6c1f56e44a..feab1aab90 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4890,10 +4890,12 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) return affectedRows; } -static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq) +static void getTableName(char *pTblName, + threadInfo* pThreadInfo, uint64_t tableSeq) { SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - if (superTblInfo) { + if ((superTblInfo) + && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) { if (superTblInfo->childTblLimit > 0) { snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", superTblInfo->childTblName + @@ -6768,6 +6770,7 @@ static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; + uint64_t tsubSeq; if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n", @@ -6776,6 +6779,15 @@ static void *superSubscribe(void *sarg) { exit(-1); } + if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { + errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", + g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, + g_queryInfo.superQueryInfo.sqlCount, + pThreadInfo->ntables, + MAX_QUERY_SQL_COUNT); + exit(-1); + } + if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, @@ -6804,6 +6816,8 @@ static void *superSubscribe(void *sarg) { char topic[32] = {0}; for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + + tsubSeq = i - pThreadInfo->start_table_from; verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n", __func__, __LINE__, pThreadInfo->threadID, @@ -6823,12 +6837,12 @@ static void *superSubscribe(void *sarg) { debugPrint("%s() LN%d, [%d] subSqlstr: %s\n", __func__, __LINE__, pThreadInfo->threadID, subSqlstr); - tsub[i] = subscribeImpl( + tsub[tsubSeq] = subscribeImpl( STABLE_CLASS, pThreadInfo, subSqlstr, topic, g_queryInfo.superQueryInfo.subscribeRestart, g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[i]) { + if (NULL == tsub[tsubSeq]) { taos_close(pThreadInfo->taos); return NULL; } @@ -6844,54 +6858,56 @@ static void *superSubscribe(void *sarg) { while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } + tsubSeq = i - pThreadInfo->start_table_from; + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } - res = taos_consume(tsub[i]); - if (res) { - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); - } - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); - } - consumed[i] ++; + res = taos_consume(tsub[tsubSeq]); + if (res) { + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + sprintf(pThreadInfo->fp, "%s-%d", + g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], + pThreadInfo->threadID); + appendResultToFile(res, pThreadInfo->fp); + } + consumed[tsubSeq] ++; - if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) - && (consumed[i] >= - g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { - printf("keepProgress:%d, resub super table query: %"PRIu64"\n", - g_queryInfo.superQueryInfo.subscribeKeepProgress, - pThreadInfo->querySeq); - taos_unsubscribe(tsub, + if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) + && (consumed[tsubSeq] >= + g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + printf("keepProgress:%d, resub super table query: %"PRIu64"\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress, + pThreadInfo->querySeq); + taos_unsubscribe(tsub[tsubSeq], g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[i]= 0; - tsub[i] = subscribeImpl( - STABLE_CLASS, - pThreadInfo, subSqlstr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval - ); - if (NULL == tsub[i]) { - taos_close(pThreadInfo->taos); - return NULL; - } - } - } + consumed[tsubSeq]= 0; + tsub[tsubSeq] = subscribeImpl( + STABLE_CLASS, + pThreadInfo, subSqlstr, topic, + g_queryInfo.superQueryInfo.subscribeRestart, + g_queryInfo.superQueryInfo.subscribeInterval + ); + if (NULL == tsub[tsubSeq]) { + taos_close(pThreadInfo->taos); + return NULL; + } + } + } } } taos_free_result(res); for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { - taos_unsubscribe(tsub[i], 0); + tsubSeq = i - pThreadInfo->start_table_from; + taos_unsubscribe(tsub[tsubSeq], 0); } taos_close(pThreadInfo->taos); From 171ca11a0ad9be9e2973512d4dd63d8592e4a921 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 11:54:50 +0800 Subject: [PATCH 41/56] Update README-CN.md (#6191) --- README-CN.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index d4c10e71d6..afb242d621 100644 --- a/README-CN.md +++ b/README-CN.md @@ -116,7 +116,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -在X86-64、X86、arm64 和 arm32 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 +在X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 aarch64： @@ -130,6 +130,12 @@ aarch32： cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64： + +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### Windows 系统 如果你使用的是 Visual Studio 2013 版本： From 2d43e7f06abf5d373edc1f654cd1f12f89b52a01 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 11:55:12 +0800 Subject: [PATCH 42/56] add mips64 build instruction. (#6190) Co-authored-by: Shuduo Sang --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78f902babe..2c33119e34 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 and arm32 platform. +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64, arm32 and mips64 platform. You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: @@ -123,6 +123,11 @@ aarch32: cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` +mips64: +```bash +cmake .. -DCPUTYPE=mips64 && cmake --build . +``` + ### On Windows platform If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". From 795503ccbe4f9ebc5e970e00414283822d209328 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 16:17:03 +0800 Subject: [PATCH 43/56] [TD-4296]: taosdemo sub keepProgress. (#6201) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index feab1aab90..aa7a022f91 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -403,7 +403,7 @@ typedef struct SuperQueryInfo_S { uint64_t sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; - int resubAfterConsume[MAX_QUERY_SQL_COUNT]; + int resubAfterConsume; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; char* childTblName; @@ -4495,6 +4495,18 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; } + cJSON* superResubAfterConsume = + cJSON_GetObjectItem(superQuery, "resubAfterConsume"); + if (superResubAfterConsume + && superResubAfterConsume->type == cJSON_Number) { + g_queryInfo.superQueryInfo.resubAfterConsume = + superResubAfterConsume->valueint; + } else if (!superResubAfterConsume) { + //printf("failed to read json, subscribe interval no found\n"); + ////goto PARSE_OVER; + g_queryInfo.superQueryInfo.resubAfterConsume = 1; + } + // sqls cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); if (!subsqls) { @@ -4526,18 +4538,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - cJSON* superResubAfterConsume = - cJSON_GetObjectItem(sql, "resubAfterConsume"); - if (superResubAfterConsume - && superResubAfterConsume->type == cJSON_Number) { - g_queryInfo.superQueryInfo.resubAfterConsume[j] = - superResubAfterConsume->valueint; - } else if (!superResubAfterConsume) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1; - } - cJSON *result = cJSON_GetObjectItem(sql, "result"); if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ @@ -6855,6 +6855,8 @@ static void *superSubscribe(void *sarg) { } TAOS_RES* res = NULL; + uint64_t st = 0, et = 0; + while(1) { for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { @@ -6863,7 +6865,12 @@ static void *superSubscribe(void *sarg) { continue; } + st = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et)); res = taos_consume(tsub[tsubSeq]); + et = taosGetTimestampMs(); + performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st)); + if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", @@ -6881,7 +6888,7 @@ static void *superSubscribe(void *sarg) { if ((g_queryInfo.superQueryInfo.subscribeKeepProgress) && (consumed[tsubSeq] >= - g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { + g_queryInfo.superQueryInfo.resubAfterConsume)) { printf("keepProgress:%d, resub super table query: %"PRIu64"\n", g_queryInfo.superQueryInfo.subscribeKeepProgress, pThreadInfo->querySeq); From 091b8e6e565a8dc53b6b5ff4001cf62611438bce Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Sat, 22 May 2021 19:54:28 +0800 Subject: [PATCH 44/56] [TD-4299]Circle CI startup (#6197) * [TD-4299]Circle CI startup * test * test * revert * udpate case --- .circleci/config.yml | 13 +++++++++++++ tests/script/unique/cluster/cache.sim | 2 +- tests/script/unique/dnode/monitor.sim | 2 +- tests/script/unique/dnode/monitor_bug.sim | 6 +++--- 4 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..6f98693add --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,13 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 +# Use a package of configuration called an orb. +orbs: + # Declare a dependency on the welcome-orb + welcome: circleci/welcome-orb@0.4.1 +# Orchestrate or schedule a set of jobs +workflows: + # Name the workflow "welcome" + welcome: + # Run the welcome/run job in its own container + jobs: + - welcome/run diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index 33aaea425c..740eddfb0d 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -41,7 +41,7 @@ sql create dnode $hostname2 sleep 10000 sql show log.tables; -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor.sim b/tests/script/unique/dnode/monitor.sim index b9b5e41889..0b41a4137c 100644 --- a/tests/script/unique/dnode/monitor.sim +++ b/tests/script/unique/dnode/monitor.sim @@ -56,7 +56,7 @@ print $data30 print $data40 print $data50 -if $rows != 5 then +if $rows > 6 then return -1 endi diff --git a/tests/script/unique/dnode/monitor_bug.sim b/tests/script/unique/dnode/monitor_bug.sim index efdf5e94b9..60c6524d9c 100644 --- a/tests/script/unique/dnode/monitor_bug.sim +++ b/tests/script/unique/dnode/monitor_bug.sim @@ -19,7 +19,7 @@ sleep 3000 sql show dnodes print dnode1 openVnodes $data2_1 -if $data2_1 != 1 then +if $data2_1 > 2 then return -1 endi @@ -41,7 +41,7 @@ print dnode2 openVnodes $data2_2 if $data2_1 != 0 then goto show2 endi -if $data2_2 != 1 then +if $data2_2 > 2 then goto show2 endi @@ -55,7 +55,7 @@ print $data30 print $data40 print $data50 -if $rows != 4 then +if $rows > 5 then return -1 endi From beac10a45094ea2c20e224beec4db920b4477619 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 22 May 2021 21:08:13 +0800 Subject: [PATCH 45/56] Hotfix/sangshuduo/td 4240 taosdemo subscribe toomuch for develop (#6203) * [TD-4240]: taosdemo subscribe more than max query sql count. * [TD-4240]: taosdemo subscribe more than 100. fix tsub sequence bug. * [TD-4240]: taosdemo subscribe more than 100. fix auto create table. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index aa7a022f91..eace05af8d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6779,15 +6779,6 @@ static void *superSubscribe(void *sarg) { exit(-1); } - if (g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint("The number %"PRId64" of sql count(%"PRIu64") multiple the table number(%"PRId64") of the thread is more than max query sql count: %d\n", - g_queryInfo.superQueryInfo.sqlCount * pThreadInfo->ntables, - g_queryInfo.superQueryInfo.sqlCount, - pThreadInfo->ntables, - MAX_QUERY_SQL_COUNT); - exit(-1); - } - if (pThreadInfo->taos == NULL) { TAOS * taos = NULL; taos = taos_connect(g_queryInfo.host, From 4729a13ed94c073aa49b34ff467efd7a7a805b37 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Sun, 23 May 2021 14:40:29 +0800 Subject: [PATCH 46/56] change version number (#6205) Co-authored-by: plum-lihui --- .gitmodules | 6 +++--- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index 74afbbf997..0e65b02221 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,12 @@ [submodule "src/connector/go"] path = src/connector/go - url = https://github.com/taosdata/driver-go + url = git@github.com:taosdata/driver-go.git [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin - url = https://github.com/taosdata/grafanaplugin + url = git@github.com:taosdata/grafanaplugin.git [submodule "src/connector/hivemq-tdengine-extension"] path = src/connector/hivemq-tdengine-extension - url = https://github.com/huskar-t/hivemq-tdengine-extension.git + url = git@github.com:taosdata/hivemq-tdengine-extension.git [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git diff --git a/cmake/version.inc b/cmake/version.inc index 0ee23f319a..ed8e7a156c 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.1.0.0") + SET(TD_VER_NUMBER "2.1.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 43006928a6..d85c8e5871 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,7 +1,7 @@ name: tdengine base: core18 -version: '2.1.0.0' +version: '2.1.1.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -73,7 +73,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.1.0.0 + - usr/lib/libtaos.so.2.1.1.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/connector/go b/src/connector/go index 7a26c432f8..8ce6d86558 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 7a26c432f8b4203e42344ff3290b9b9b01b983d5 +Subproject commit 8ce6d86558afc8c0b50c10f990fd2b4270cf06fc diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 32e2c97a4c..3530c6df09 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df +Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa From 105d6b040a60e99ad0c0300b63e1494b6f73255f Mon Sep 17 00:00:00 2001 From: yaofei517 <41509375+yaofei517@users.noreply.github.com> Date: Sun, 23 May 2021 17:11:06 +0800 Subject: [PATCH 47/56] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dwindows=E4=B8=8Bvscode?= =?UTF-8?q?=E8=B0=83=E8=AF=95go=E7=A8=8B=E5=BA=8F=E5=90=8D=E6=97=A0?= =?UTF-8?q?=E5=90=8E=E7=BC=80=E7=9A=84=E6=83=85=E5=86=B5=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=A8=8B=E5=BA=8Fcrash=20(#6176)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修复windows下vscode调试go程序名无后缀的情况导致程序crash * 修复windows下vscode调试go程序时临时程序_debug_bin无后缀的情况，导致调用接口发生crash * 修复由于头文件排序导致编译错误 --- src/os/src/windows/wSemphone.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index a3f0367ee1..878ceba791 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "os.h" #include "taosdef.h" #include "tglobal.h" @@ -24,7 +25,7 @@ bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } -void taosResetPthread(pthread_t *thread) { thread->p = 0; } +void taosResetPthread(pthread_t* thread) { thread->p = 0; } int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION @@ -34,27 +35,24 @@ int64_t taosGetPthreadId(pthread_t thread) { #endif } -int64_t taosGetSelfPthreadId() { - return GetCurrentThreadId(); -} +int64_t taosGetSelfPthreadId() { return GetCurrentThreadId(); } -bool taosComparePthread(pthread_t first, pthread_t second) { - return first.p == second.p; -} +bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } -int32_t taosGetPId() { - return GetCurrentProcessId(); -} +int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char *name, int32_t* len) { +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); - *strrchr(filepath,'.') = '\0'; + char* sub = strrchr(filepath, '.'); + if (sub != NULL) { + *sub = '\0'; + } strcpy(name, filepath); if (len != NULL) { - *len = (int32_t) strlen(filepath); + *len = (int32_t)strlen(filepath); } return 0; From d26ff7cede73bdc099c23b292638d4ea9e3d7479 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 23 May 2021 19:35:13 +0800 Subject: [PATCH 48/56] [TD-4313]: odbc example compile failed on Windows. (#6206) * [TD-4313]: odbc example compile failed on Windows. * [TD-4313]: odbc example compile failed on Windows. few functions also need change. --- src/connector/odbc/examples/c/main.c | 6 +----- src/os/src/detail/osMemory.c | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/connector/odbc/examples/c/main.c b/src/connector/odbc/examples/c/main.c index 3ad32dbe53..de01d2b85e 100644 --- a/src/connector/odbc/examples/c/main.c +++ b/src/connector/odbc/examples/c/main.c @@ -312,11 +312,7 @@ static int test_sqls_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, const cha size_t len = 0; ssize_t n = 0; -#ifdef _MSC_VER - n = taosGetlineImp(&line, &len, f); -#else - n = getline(&line, &len, f); -#endif + n = tgetline(&line, &len, f); if (n==-1) break; const char *p = NULL; diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 291a54b669..d8194feab4 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -62,7 +62,7 @@ static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_ static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrdup(str); } static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { @@ -70,11 +70,11 @@ static char* taosRandomStrndup(const char* str, size_t size, const char* file, u if (len > size) { len = size; } - return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); + return taosRandomAllocFail(len + 1, file, line) ? NULL : tstrndup(str, len); } static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); + return taosRandomAllocFail(*n, file, line) ? -1 : tgetline(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -242,7 +242,7 @@ static char* taosStrndupDetectLeak(const char* str, size_t size, const char* fil static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; - ssize_t size = taosGetlineImp(&buf, &bufSize, stream); + ssize_t size = tgetline(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); @@ -372,7 +372,7 @@ void taosFreeMem(void* ptr, const char* file, uint32_t line) { char* taosStrdupMem(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrdupImp(str); + return tstrdup(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrdup(str, file, line); @@ -380,13 +380,13 @@ char* taosStrdupMem(const char* str, const char* file, uint32_t line) { case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrdupDetectLeak(str, file, line); } - return taosStrdupImp(str); + return tstrdup(str); } char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosStrndupImp(str, size); + return tstrndup(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomStrndup(str, size, file, line); @@ -394,13 +394,13 @@ char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t li case TAOS_ALLOC_MODE_DETECT_LEAK: return taosStrndupDetectLeak(str, size, file, line); } - return taosStrndupImp(str, size); + return tstrndup(str, size); } ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: return taosRandomGetline(lineptr, n, stream, file, line); @@ -408,7 +408,7 @@ ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file case TAOS_ALLOC_MODE_DETECT_LEAK: return taosGetlineDetectLeak(lineptr, n, stream, file, line); } - return taosGetlineImp(lineptr, n, stream); + return tgetline(lineptr, n, stream); } static void taosCloseAllocLog() { @@ -517,4 +517,4 @@ void* taosTZfree(void* ptr) { free((void*)((char*)ptr - sizeof(size_t))); } return NULL; -} \ No newline at end of file +} From 8e6a5acbc39b2b3fa88a8db8835f8bfbd3447080 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 12:07:45 +0800 Subject: [PATCH 49/56] Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#6208) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. * [TD-3197]: taosdemo coverity scan issue. configDir buffer overwrite. * [TD-3197]: coverity scan issue. taosdump argument validation. * [TD-3197]: taosdemo and taosdump coverity scan issues. * [TD-3197]: taosdemo coverity scan. append result buf to file. for develop branch. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index eace05af8d..32c1be372c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1147,19 +1147,18 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { static void appendResultBufToFile(char *resultBuf, char *resultFile) { - FILE *fp = NULL; - if (resultFile[0] != 0) { - fp = fopen(resultFile, "at"); - if (fp == NULL) { - errorPrint( - "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); - return; + FILE *fp = NULL; + if (resultFile[0] != 0) { + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + fprintf(fp, "%s", resultBuf); + tmfclose(fp); } - } - - fprintf(fp, "%s", resultBuf); - tmfclose(fp); } static void appendResultToFile(TAOS_RES *res, char* resultFile) { From 9facb54b3ff22b54ed2b83bdec53bf6a4d815da0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 15:21:20 +0800 Subject: [PATCH 50/56] [TD-4322]: taosdemo query then fetch result. (#6211) * [TD-4322]: taosdemo query then fetch result. * build again. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 32c1be372c..adb6e836de 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1147,21 +1147,20 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { static void appendResultBufToFile(char *resultBuf, char *resultFile) { - FILE *fp = NULL; - if (resultFile[0] != 0) { - fp = fopen(resultFile, "at"); - if (fp == NULL) { - errorPrint( - "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); - return; - } - fprintf(fp, "%s", resultBuf); - tmfclose(fp); - } + FILE *fp = NULL; + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + + fprintf(fp, "%s", resultBuf); + tmfclose(fp); } -static void appendResultToFile(TAOS_RES *res, char* resultFile) { +static void fetchResult(TAOS_RES *res, char* resultFile) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -1179,10 +1178,11 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if (totalLen >= 100*1024*1024 - 32000) { - appendResultBufToFile(databuf, resultFile); - totalLen = 0; - memset(databuf, 0, 100*1024*1024); + if ((resultFile) + && (totalLen >= 100*1024*1024 - 32000)) { + appendResultBufToFile(databuf, resultFile); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); } num_rows++; int len = taos_print_row(temp, row, fields, num_fields); @@ -1194,7 +1194,9 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); - appendResultBufToFile(databuf, resultFile); + if (resultFile) { + appendResultBufToFile(databuf, resultFile); + } free(databuf); } @@ -1210,9 +1212,7 @@ static void selectAndGetResult( return; } - if ((strlen(pThreadInfo->fp))) { - appendResultToFile(res, pThreadInfo->fp); - } + fetchResult(res, pThreadInfo->fp); taos_free_result(res); } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { @@ -2053,13 +2053,13 @@ static void printfQuerySystemInfo(TAOS * taos) { // show variables res = taos_query(taos, "show variables;"); - //appendResultToFile(res, filename); + //fetchResult(res, filename); xDumpResultToFile(filename, res); // show dnodes res = taos_query(taos, "show dnodes;"); xDumpResultToFile(filename, res); - //appendResultToFile(res, filename); + //fetchResult(res, filename); // show databases res = taos_query(taos, "show databases;"); @@ -6712,7 +6712,7 @@ static void stable_sub_callback( } if (param) - appendResultToFile(res, ((threadInfo *)param)->fp); + fetchResult(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } @@ -6725,7 +6725,7 @@ static void specified_sub_callback( } if (param) - appendResultToFile(res, ((threadInfo *)param)->fp); + fetchResult(res, ((threadInfo *)param)->fp); // tao_unscribe() will free result. } @@ -6866,13 +6866,13 @@ static void *superSubscribe(void *sarg) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } consumed[tsubSeq] ++; @@ -6973,7 +6973,7 @@ static void *specifiedSubscribe(void *sarg) { sprintf(pThreadInfo->fp, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - appendResultToFile(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo->fp); } consumed ++; From 432f961e6122c96d62de0c5e94cd83fc0650853e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 16:22:25 +0800 Subject: [PATCH 51/56] Hotfix/sangshuduo/td 4296 taosdemo sub keepprogress for develop (#6215) * [TD-4296]: taosdemo sub keepProgress. * make print sqlcount quiet. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index adb6e836de..5afc8c44a1 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -7082,7 +7082,7 @@ static int subscribeTestProcess() { //==== create threads for super table query if (g_queryInfo.superQueryInfo.sqlCount <= 0) { - printf("%s() LN%d, super table query sqlCount %"PRIu64".\n", + debugPrint("%s() LN%d, super table query sqlCount %"PRIu64".\n", __func__, __LINE__, g_queryInfo.superQueryInfo.sqlCount); } else { From 71c41bdd3b0926fbb0ad4d6c6331a3f106af4d6f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 24 May 2021 17:03:40 +0800 Subject: [PATCH 52/56] [TD-4327]jenkins migration --- tests/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 709b39d077..2f2b9f6933 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { stage('Parallel test stage') { parallel { stage('pytest') { - agent{label '184'} + agent{label 'slad1'} steps { pre_test() sh ''' @@ -62,7 +62,7 @@ pipeline { } stage('test_crash_gen') { - agent{label "185"} + agent{label "slad2"} steps { pre_test() sh ''' @@ -149,7 +149,7 @@ pipeline { } stage('test_valgrind') { - agent{label "186"} + agent{label "slad3"} steps { pre_test() From d1f15a82eb825f12f4d422ab853951c12e73f289 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 20:16:26 +0800 Subject: [PATCH 53/56] Hotfix/sangshuduo/td 4322 taosdemo fetch result for develop (#6220) * [TD-4322]: taosdemo query then fetch result. * build again. * [TD-4322]: taosdemo fetch result. check result file name length. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5afc8c44a1..0f3adc29be 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1178,7 +1178,7 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((resultFile) + if ((resultFile) && (strlen(resultFile) > 0) && (totalLen >= 100*1024*1024 - 32000)) { appendResultBufToFile(databuf, resultFile); totalLen = 0; @@ -1194,7 +1194,7 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); - if (resultFile) { + if ((resultFile) && (strlen(resultFile) > 0)) { appendResultBufToFile(databuf, resultFile); } free(databuf); @@ -2231,7 +2231,7 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); - if (resultFile) { + if ((resultFile) && (strlen(resultFile) > 0)) { appendResultBufToFile(response_buf, resultFile); } From 21c0341b32142cebbd06d8b3909049013839cf00 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 24 May 2021 22:10:55 +0800 Subject: [PATCH 54/56] Hotfix/sangshuduo/td 4296 taosdemo sub keepprogress for develop (#6222) * [TD-4296]: taosdemo sub keepProgress. * make print sqlcount quiet. * [TD-4296]: taosdemo subscribe keepprogress. specified query topic thread safe. * cherry pick from master branch. * cherry pick from master branch. * fix compile warning for mac. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 407 ++++++++++++++++++------------------ 1 file changed, 206 insertions(+), 201 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0f3adc29be..6f651c18cf 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -375,7 +375,7 @@ typedef struct SDbs_S { typedef struct SpecifiedQueryInfo_S { uint64_t queryInterval; // 0: unlimit > 0 loop/s - uint64_t concurrent; + uint32_t concurrent; uint64_t sqlCount; uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms @@ -386,6 +386,9 @@ typedef struct SpecifiedQueryInfo_S { char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; int resubAfterConsume[MAX_QUERY_SQL_COUNT]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + char topic[MAX_QUERY_SQL_COUNT][32]; + int consumed[MAX_QUERY_SQL_COUNT]; + TAOS_RES* res[MAX_QUERY_SQL_COUNT]; uint64_t totalQueried; } SpecifiedQueryInfo; @@ -431,7 +434,8 @@ typedef struct SThreadInfo_S { int threadID; char db_name[MAX_DB_NAME_SIZE+1]; uint32_t time_precision; - char fp[4096]; + char filePath[4096]; + FILE *fp; char tb_prefix[MAX_TB_NAME_SIZE]; uint64_t start_table_from; uint64_t end_table_to; @@ -546,7 +550,7 @@ static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); static int postProceSql(char *host, struct sockaddr_in *pServAddr, - uint16_t port, char* sqlstr, char *resultFile); + uint16_t port, char* sqlstr, threadInfo *pThreadInfo); /* ************ Global variables ************ */ @@ -1128,7 +1132,8 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to execute %s, command length: %d, reason: %s\n", + command, (int)strlen(command), taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -1145,22 +1150,22 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void appendResultBufToFile(char *resultBuf, char *resultFile) +static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo) { - FILE *fp = NULL; - fp = fopen(resultFile, "at"); - if (fp == NULL) { + pThreadInfo->fp = fopen(pThreadInfo->filePath, "at"); + if (pThreadInfo->fp == NULL) { errorPrint( "%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFile); + __func__, __LINE__, pThreadInfo->filePath); return; } - fprintf(fp, "%s", resultBuf); - tmfclose(fp); + fprintf(pThreadInfo->fp, "%s", resultBuf); + tmfclose(pThreadInfo->fp); + pThreadInfo->fp = NULL; } -static void fetchResult(TAOS_RES *res, char* resultFile) { +static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); @@ -1178,9 +1183,9 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((resultFile) && (strlen(resultFile) > 0) + if ((strlen(pThreadInfo->filePath) > 0) && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, resultFile); + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } @@ -1193,9 +1198,9 @@ static void fetchResult(TAOS_RES *res, char* resultFile) { } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", - __func__, __LINE__, databuf, resultFile); - if ((resultFile) && (strlen(resultFile) > 0)) { - appendResultBufToFile(databuf, resultFile); + __func__, __LINE__, databuf, pThreadInfo->filePath); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(databuf, pThreadInfo); } free(databuf); } @@ -1212,14 +1217,14 @@ static void selectAndGetResult( return; } - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); taos_free_result(res); } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { int retCode = postProceSql( g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, command, - pThreadInfo->fp); + pThreadInfo); if (0 != retCode) { printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); } @@ -1756,7 +1761,7 @@ static void printfQueryMeta() { printf("query interval: \033[33m%"PRIu64" ms\033[0m\n", g_queryInfo.specifiedQueryInfo.queryInterval); printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times); - printf("concurrent: \033[33m%"PRIu64"\033[0m\n", + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("mod: \033[33m%s\033[0m\n", (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync"); @@ -2095,7 +2100,7 @@ static void printfQuerySystemInfo(TAOS * taos) { } static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, - char* sqlstr, char *resultFile) + char* sqlstr, threadInfo *pThreadInfo) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -2231,8 +2236,8 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); - if ((resultFile) && (strlen(resultFile) > 0)) { - appendResultBufToFile(response_buf, resultFile); + if (strlen(pThreadInfo->filePath) > 0) { + appendResultBufToFile(response_buf, pThreadInfo); } free(request_buf); @@ -2994,19 +2999,19 @@ static int startMultiThreadCreateChildTable( int64_t b = 0; b = ntables % threads; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->superTblInfo = superTblInfo; + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->superTblInfo = superTblInfo; verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); - t_info->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (t_info->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", __func__, __LINE__, taos_errstr(NULL)); free(pids); @@ -3014,14 +3019,14 @@ static int startMultiThreadCreateChildTable( return -1; } - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->use_metric = true; - t_info->cols = cols; - t_info->minDelay = UINT64_MAX; - pthread_create(pids + i, NULL, createTable, t_info); + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->use_metric = true; + pThreadInfo->cols = cols; + pThreadInfo->minDelay = UINT64_MAX; + pthread_create(pids + i, NULL, createTable, pThreadInfo); } for (int i = 0; i < threads; i++) { @@ -3029,8 +3034,8 @@ static int startMultiThreadCreateChildTable( } for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - taos_close(t_info->taos); + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); } free(pids); @@ -4246,7 +4251,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (concurrent && concurrent->type == cJSON_Number) { if (concurrent->valueint <= 0) { errorPrint( - "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + "%s() LN%d, query sqlCount %"PRIu64" or concurrent %d is not correct.\n", __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, g_queryInfo.specifiedQueryInfo.concurrent); @@ -4313,24 +4318,28 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } // sqls - cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); - if (!superSqls) { + cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); + if (!specifiedSqls) { g_queryInfo.specifiedQueryInfo.sqlCount = 0; - } else if (superSqls->type != cJSON_Array) { + } else if (specifiedSqls->type != cJSON_Array) { errorPrint("%s() LN%d, failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(superSqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", - __func__, __LINE__, MAX_QUERY_SQL_COUNT); + int superSqlSize = cJSON_GetArraySize(specifiedSqls); + if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent + > MAX_QUERY_SQL_COUNT) { + errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n", + __func__, __LINE__, + superSqlSize, + g_queryInfo.specifiedQueryInfo.concurrent, + MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(superSqls, j); + cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -4506,16 +4515,16 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.resubAfterConsume = 1; } - // sqls - cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); - if (!subsqls) { + // supert table sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (subsqls->type != cJSON_Array) { + } else if (superSqls->type != cJSON_Array) { errorPrint("%s() LN%d: failed to read json, super sqls not found\n", __func__, __LINE__); goto PARSE_OVER; } else { - int superSqlSize = cJSON_GetArraySize(subsqls); + int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", __func__, __LINE__, MAX_QUERY_SQL_COUNT); @@ -4524,7 +4533,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.superQueryInfo.sqlCount = superSqlSize; for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(subsqls, j); + cJSON* sql = cJSON_GetArrayItem(superSqls, j); if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); @@ -5970,22 +5979,22 @@ static void startMultiThreadInsertData(int threads, char* db_name, memset(infos, 0, threads * sizeof(threadInfo)); for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->time_precision = timePrec; - t_info->superTblInfo = superTblInfo; + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE); + pThreadInfo->time_precision = timePrec; + pThreadInfo->superTblInfo = superTblInfo; - t_info->start_time = start_time; - t_info->minDelay = UINT64_MAX; + pThreadInfo->start_time = start_time; + pThreadInfo->minDelay = UINT64_MAX; if ((NULL == superTblInfo) || (superTblInfo->insertMode != REST_IFACE)) { - //t_info->taos = taos; - t_info->taos = taos_connect( + //pThreadInfo->taos = taos; + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == t_info->taos) { + if (NULL == pThreadInfo->taos) { errorPrint( "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n", __func__, __LINE__, @@ -5995,8 +6004,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, } if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) { - t_info->stmt = taos_stmt_init(t_info->taos); - if (NULL == t_info->stmt) { + pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); + if (NULL == pThreadInfo->stmt) { errorPrint( "%s() LN%d, failed init stmt, reason: %s\n", __func__, __LINE__, @@ -6007,27 +6016,27 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } } else { - t_info->taos = NULL; + pThreadInfo->taos = NULL; } /* if ((NULL == superTblInfo) || (0 == superTblInfo->multiThreadWriteOneTbl)) { */ - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; /* } else { - t_info->start_table_from = 0; - t_info->ntables = superTblInfo->childTblCount; - t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = superTblInfo->childTblCount; + pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); } */ - tsem_init(&(t_info->lock_sem), 0, 0); + tsem_init(&(pThreadInfo->lock_sem), 0, 0); if (ASYNC_MODE == g_Dbs.asyncMode) { - pthread_create(pids + i, NULL, asyncWrite, t_info); + pthread_create(pids + i, NULL, asyncWrite, pThreadInfo); } else { - pthread_create(pids + i, NULL, syncWrite, t_info); + pthread_create(pids + i, NULL, syncWrite, pThreadInfo); } } @@ -6042,31 +6051,32 @@ static void startMultiThreadInsertData(int threads, char* db_name, double avgDelay = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; + threadInfo *pThreadInfo = infos + i; - tsem_destroy(&(t_info->lock_sem)); + tsem_destroy(&(pThreadInfo->lock_sem)); - if (t_info->stmt) { - taos_stmt_close(t_info->stmt); + if (pThreadInfo->stmt) { + taos_stmt_close(pThreadInfo->stmt); } - taos_close(t_info->taos); + tsem_destroy(&(pThreadInfo->lock_sem)); + taos_close(pThreadInfo->taos); debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n", __func__, __LINE__, - t_info->threadID, t_info->totalInsertRows, - t_info->totalAffectedRows); + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); if (superTblInfo) { - superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalInsertRows += t_info->totalInsertRows; + superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows; + superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows; } else { - g_args.totalAffectedRows += t_info->totalAffectedRows; - g_args.totalInsertRows += t_info->totalInsertRows; + g_args.totalAffectedRows += pThreadInfo->totalAffectedRows; + g_args.totalInsertRows += pThreadInfo->totalInsertRows; } - totalDelay += t_info->totalDelay; - cntDelay += t_info->cntDelay; - if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; + totalDelay += pThreadInfo->totalDelay; + cntDelay += pThreadInfo->cntDelay; + if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay; + if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay; } cntDelay -= 1; @@ -6122,26 +6132,26 @@ static void startMultiThreadInsertData(int threads, char* db_name, static void *readTable(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - uint64_t sTime = rinfo->start_time; - char *tb_prefix = rinfo->tb_prefix; - FILE *fp = fopen(rinfo->fp, "a"); + uint64_t sTime = pThreadInfo->start_time; + char *tb_prefix = pThreadInfo->tb_prefix; + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } int64_t num_of_DPT; -/* if (rinfo->superTblInfo) { - num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; +/* if (pThreadInfo->superTblInfo) { + num_of_DPT = pThreadInfo->superTblInfo->insertRows; // nrecords_per_table; } else { */ num_of_DPT = g_args.num_of_DPT; // } - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6194,17 +6204,17 @@ static void *readTable(void *sarg) { static void *readMetric(void *sarg) { #if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; + threadInfo *pThreadInfo = (threadInfo *)sarg; + TAOS *taos = pThreadInfo->taos; char command[BUFFER_SIZE] = "\0"; - FILE *fp = fopen(rinfo->fp, "a"); + FILE *fp = fopen(pThreadInfo->filePath, "a"); if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno)); return NULL; } - int64_t num_of_DPT = rinfo->superTblInfo->insertRows; - int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; + int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows; + int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; @@ -6403,8 +6413,8 @@ static void *specifiedTableQuery(void *sarg) { uint64_t lastPrintTime = taosGetTimestampMs(); uint64_t startTs = taosGetTimestampMs(); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } @@ -6504,8 +6514,8 @@ static void *superTableQuery(void *sarg) { for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { memset(sqlstr,0,sizeof(sqlstr)); replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); - if (g_queryInfo.superQueryInfo.result[j][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + if (g_queryInfo.superQueryInfo.result[j] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } @@ -6595,9 +6605,9 @@ static int queryTestProcess() { for (uint64_t i = 0; i < nSqlCount; i++) { for (int j = 0; j < nConcurrent; j++) { uint64_t seq = i * nConcurrent + j; - threadInfo *t_info = infos + seq; - t_info->threadID = seq; - t_info->querySeq = i; + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { @@ -6614,10 +6624,10 @@ static int queryTestProcess() { } } - t_info->taos = NULL;// TODO: workaround to use separate taos connection; + pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection; pthread_create(pids + seq, NULL, specifiedTableQuery, - t_info); + pThreadInfo); } } } else { @@ -6657,15 +6667,15 @@ static int queryTestProcess() { uint64_t startFrom = 0; for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; + threadInfo *pThreadInfo = infosOfSub + i; + pThreadInfo->threadID = i; - t_info->start_table_from = startFrom; - t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; - startFrom = t_info->end_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info); + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -6712,7 +6722,7 @@ static void stable_sub_callback( } if (param) - fetchResult(res, ((threadInfo *)param)->fp); + fetchResult(res, (threadInfo *)param); // tao_unscribe() will free result. } @@ -6725,7 +6735,7 @@ static void specified_sub_callback( } if (param) - fetchResult(res, ((threadInfo *)param)->fp); + fetchResult(res, (threadInfo *)param); // tao_unscribe() will free result. } @@ -6779,18 +6789,15 @@ static void *superSubscribe(void *sarg) { } if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } @@ -6820,7 +6827,7 @@ static void *superSubscribe(void *sarg) { g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlstr, i); if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } @@ -6863,16 +6870,16 @@ static void *superSubscribe(void *sarg) { if (res) { if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); } if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(res, pThreadInfo); } consumed[tsubSeq] ++; @@ -6913,21 +6920,18 @@ static void *superSubscribe(void *sarg) { static void *specifiedSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB* tsub = NULL; +// TAOS_SUB* tsub = NULL; if (pThreadInfo->taos == NULL) { - TAOS * taos = NULL; - taos = taos_connect(g_queryInfo.host, + pThreadInfo->taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", pThreadInfo->threadID, taos_errstr(NULL)); return NULL; - } else { - pThreadInfo->taos = taos; } } @@ -6939,69 +6943,70 @@ static void *specifiedSubscribe(void *sarg) { return NULL; } - char topic[32] = {0}; - sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq); - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], + "taosdemo-subscribe-%"PRIu64"-%d", + pThreadInfo->querySeq, + pThreadInfo->threadID); + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq] != NULL) { + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); } - tsub = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( + SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeRestart, g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == tsub) { + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { taos_close(pThreadInfo->taos); return NULL; } // start loop to consume result - TAOS_RES* res = NULL; - - int consumed; + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; while(1) { if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } - res = taos_consume(tsub); - if (res) { + g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); + if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - sprintf(pThreadInfo->fp, "%s-%d", + sprintf(pThreadInfo->filePath, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - fetchResult(res, pThreadInfo->fp); + fetchResult(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], pThreadInfo); } - consumed ++; + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++; if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress) - && (consumed >= + && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >= g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) { printf("keepProgress:%d, resub specified query: %"PRIu64"\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, pThreadInfo->querySeq); - consumed = 0; - taos_unsubscribe(tsub, + g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - tsub = subscribeImpl( + g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( SPECIFIED_CLASS, pThreadInfo, g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], - topic, + g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], g_queryInfo.specifiedQueryInfo.subscribeRestart, g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == tsub) { + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { taos_close(pThreadInfo->taos); return NULL; } } } } - taos_free_result(res); - taos_unsubscribe(tsub, 0); + taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); + taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0); taos_close(pThreadInfo->taos); return NULL; @@ -7071,11 +7076,11 @@ static int subscribeTestProcess() { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) { uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *t_info = infos + seq; - t_info->threadID = seq; - t_info->querySeq = i; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pids + seq, NULL, specifiedSubscribe, t_info); + threadInfo *pThreadInfo = infos + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo); } } } @@ -7121,17 +7126,17 @@ static int subscribeTestProcess() { uint64_t startFrom = 0; for (int j = 0; j < threads; j++) { uint64_t seq = i * threads + j; - threadInfo *t_info = infosOfStable + seq; - t_info->threadID = seq; - t_info->querySeq = i; + threadInfo *pThreadInfo = infosOfStable + seq; + pThreadInfo->threadID = seq; + pThreadInfo->querySeq = i; - t_info->start_table_from = startFrom; - t_info->ntables = jend_table_to = jend_table_to + 1; - t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pThreadInfo->start_table_from = startFrom; + pThreadInfo->ntables = jend_table_to = jend_table_to + 1; + pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pidsOfStable + seq, - NULL, superSubscribe, t_info); + NULL, superSubscribe, pThreadInfo); } } @@ -7410,47 +7415,47 @@ static void queryResult() { // query data pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - assert(rInfo); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_from = 0; + threadInfo *pThreadInfo = malloc(sizeof(threadInfo)); + assert(pThreadInfo); + pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_table_from = 0; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; if (g_args.use_metric) { - rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; - rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - tstrncpy(rInfo->tb_prefix, + pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(pThreadInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); } else { - rInfo->ntables = g_args.num_of_tables; - rInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + pThreadInfo->ntables = g_args.num_of_tables; + pThreadInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); } - rInfo->taos = taos_connect( + pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - if (rInfo->taos == NULL) { + if (pThreadInfo->taos == NULL) { errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - free(rInfo); + free(pThreadInfo); exit(-1); } - tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN); if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); + pthread_create(&read_id, NULL, readTable, pThreadInfo); } else { - pthread_create(&read_id, NULL, readMetric, rInfo); + pthread_create(&read_id, NULL, readMetric, pThreadInfo); } pthread_join(read_id, NULL); - taos_close(rInfo->taos); - free(rInfo); + taos_close(pThreadInfo->taos); + free(pThreadInfo); } static void testCmdLine() { From 27def2b35e7649080cd6911971daab0761ce74cf Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Tue, 25 May 2021 14:32:15 +0800 Subject: [PATCH 55/56] [TD-4181] : update SQL related doc. --- .../cn/12.taos-sql/01.error-code/docs.md | 340 +++++++++--------- documentation20/cn/12.taos-sql/docs.md | 168 +++++---- 2 files changed, 264 insertions(+), 244 deletions(-) diff --git a/documentation20/cn/12.taos-sql/01.error-code/docs.md b/documentation20/cn/12.taos-sql/01.error-code/docs.md index 95975dba5a..0a0d694ee3 100644 --- a/documentation20/cn/12.taos-sql/01.error-code/docs.md +++ b/documentation20/cn/12.taos-sql/01.error-code/docs.md @@ -1,172 +1,172 @@ # TDengine 2.0 错误码以及对应的十进制码 -| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | -|-----------------------| :---: | :---------: | :------------------------ | ---------------- | -|TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| -|TSDB_CODE_RPC_AUTH_REQUIRED| 0 | 0x0002 | "Authentication required"| -2147483646| -|TSDB_CODE_RPC_AUTH_FAILURE| 0| 0x0003 | "Authentication failure"| -2147483645| -|TSDB_CODE_RPC_REDIRECT |0 | 0x0004| "Redirect"| -2147483644| -|TSDB_CODE_RPC_NOT_READY| 0 | 0x0005 | "System not ready"| -2147483643| -|TSDB_CODE_RPC_ALREADY_PROCESSED| 0 | 0x0006 |"Message already processed"| -2147483642| -|TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED| 0 |0x0007| "Last session not finished"| -2147483641| -|TSDB_CODE_RPC_MISMATCHED_LINK_ID| 0| 0x0008 | "Mismatched meter id"| -2147483640| -|TSDB_CODE_RPC_TOO_SLOW| 0 | 0x0009 | "Processing of request timed out"| -2147483639| -|TSDB_CODE_RPC_MAX_SESSIONS| 0 | 0x000A | "Number of sessions reached limit"| -2147483638| -|TSDB_CODE_RPC_NETWORK_UNAVAIL| 0 |0x000B | "Unable to establish connection" |-2147483637| -|TSDB_CODE_RPC_APP_ERROR| 0| 0x000C | "Unexpected generic error in RPC"| -2147483636| -|TSDB_CODE_RPC_UNEXPECTED_RESPONSE| 0 |0x000D | "Unexpected response"| -2147483635| -|TSDB_CODE_RPC_INVALID_VALUE| 0 | 0x000E | "Invalid value"| -2147483634| -|TSDB_CODE_RPC_INVALID_TRAN_ID| 0 | 0x000F | "Invalid transaction id"| -2147483633| -|TSDB_CODE_RPC_INVALID_SESSION_ID| 0| 0x0010 | "Invalid session id"| -2147483632| -|TSDB_CODE_RPC_INVALID_MSG_TYPE| 0| 0x0011| "Invalid message type"| -2147483631| -|TSDB_CODE_RPC_INVALID_RESPONSE_TYPE| 0 | 0x0012| "Invalid response type"| -2147483630| -|TSDB_CODE_RPC_INVALID_TIME_STAMP| 0| 0x0013| "Invalid timestamp"| -2147483629| -|TSDB_CODE_COM_OPS_NOT_SUPPORT| 0 | 0x0100| "Operation not supported"| -2147483392| -|TSDB_CODE_COM_MEMORY_CORRUPTED |0| 0x0101 | "Memory corrupted"| -2147483391| -|TSDB_CODE_COM_OUT_OF_MEMORY| 0| 0x0102| "Out of memory"| -2147483390| -|TSDB_CODE_COM_INVALID_CFG_MSG| 0 | 0x0103| "Invalid config message"| -2147483389| -|TSDB_CODE_COM_FILE_CORRUPTED| 0| 0x0104| "Data file corrupted" |-2147483388| -|TSDB_CODE_TSC_INVALID_SQL| 0| 0x0200 | "Invalid SQL statement"| -2147483136| -|TSDB_CODE_TSC_INVALID_QHANDLE| 0 | 0x0201 | "Invalid qhandle"| -2147483135| -|TSDB_CODE_TSC_INVALID_TIME_STAMP| 0 | 0x0202 | "Invalid combination of client/service time"| -2147483134| -|TSDB_CODE_TSC_INVALID_VALUE| 0 | 0x0203| "Invalid value in client"| -2147483133| -|TSDB_CODE_TSC_INVALID_VERSION| 0 | 0x0204 | "Invalid client version" |-2147483132| -|TSDB_CODE_TSC_INVALID_IE| 0 | 0x0205 | "Invalid client ie" |-2147483131| -|TSDB_CODE_TSC_INVALID_FQDN| 0 | 0x0206| "Invalid host name"| -2147483130| -|TSDB_CODE_TSC_INVALID_USER_LENGTH| 0 | 0x0207| "Invalid user name"| -2147483129| -|TSDB_CODE_TSC_INVALID_PASS_LENGTH| 0 | 0x0208 | "Invalid password"| -2147483128| -|TSDB_CODE_TSC_INVALID_DB_LENGTH| 0 | 0x0209| "Database name too long"| -2147483127| -|TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH| 0 | 0x020A | "Table name too long"| -2147483126| -|TSDB_CODE_TSC_INVALID_CONNECTION| 0 | 0x020B| "Invalid connection"| -2147483125| -|TSDB_CODE_TSC_OUT_OF_MEMORY| 0 | 0x020C | "System out of memory" |-2147483124| -|TSDB_CODE_TSC_NO_DISKSPACE| 0 | 0x020D | "System out of disk space"| -2147483123| -|TSDB_CODE_TSC_QUERY_CACHE_ERASED| 0 | 0x020E| "Query cache erased"| -2147483122| -|TSDB_CODE_TSC_QUERY_CANCELLED| 0 | 0x020F |"Query terminated"| -2147483121| -|TSDB_CODE_TSC_SORTED_RES_TOO_MANY| 0 |0x0210 | "Result set too large to be sorted"| -2147483120| -|TSDB_CODE_TSC_APP_ERROR| 0 | 0x0211 | "Application error"| -2147483119| -|TSDB_CODE_TSC_ACTION_IN_PROGRESS| 0 |0x0212 | "Action in progress"| -2147483118| -|TSDB_CODE_TSC_DISCONNECTED| 0 | 0x0213 |"Disconnected from service" |-2147483117| -|TSDB_CODE_TSC_NO_WRITE_AUTH| 0 | 0x0214 | "No write permission" |-2147483116| -|TSDB_CODE_MND_MSG_NOT_PROCESSED| 0| 0x0300| "Message not processed"| -2147482880| -|TSDB_CODE_MND_ACTION_IN_PROGRESS| 0 | 0x0301 |"Message is progressing"| -2147482879| -|TSDB_CODE_MND_ACTION_NEED_REPROCESSED| 0 | 0x0302 |"Messag need to be reprocessed"| -2147482878| -|TSDB_CODE_MND_NO_RIGHTS| 0 | 0x0303| "Insufficient privilege for operation"| -2147482877| -|TSDB_CODE_MND_APP_ERROR| 0 | 0x0304 | "Unexpected generic error in mnode"| -2147482876| -|TSDB_CODE_MND_INVALID_CONNECTION| 0 | 0x0305 | "Invalid message connection"| -2147482875| -|TSDB_CODE_MND_INVALID_MSG_VERSION| 0 | 0x0306 | "Incompatible protocol version"| -2147482874| -|TSDB_CODE_MND_INVALID_MSG_LEN| 0| 0x0307 | "Invalid message length"| -2147482873| -|TSDB_CODE_MND_INVALID_MSG_TYPE| 0 | 0x0308 | "Invalid message type" |-2147482872| -|TSDB_CODE_MND_TOO_MANY_SHELL_CONNS| 0 |0x0309 | "Too many connections"| -2147482871| -|TSDB_CODE_MND_OUT_OF_MEMORY| 0 |0x030A | "Out of memory in mnode"| -2147482870| -|TSDB_CODE_MND_INVALID_SHOWOBJ| 0 | 0x030B |"Data expired"| -2147482869| -|TSDB_CODE_MND_INVALID_QUERY_ID |0 | 0x030C |"Invalid query id" |-2147482868| -|TSDB_CODE_MND_INVALID_STREAM_ID| 0 |0x030D | "Invalid stream id"| -2147482867| -|TSDB_CODE_MND_INVALID_CONN_ID| 0| 0x030E | "Invalid connection id" |-2147482866| -|TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE| 0 | 0x0320| "Object already there"| -2147482848| -|TSDB_CODE_MND_SDB_ERROR| 0 |0x0321 | "Unexpected generic error in sdb" |-2147482847| -|TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE| 0 | 0x0322| "Invalid table type" |-2147482846| -|TSDB_CODE_MND_SDB_OBJ_NOT_THERE| 0 | 0x0323 |"Object not there" |-2147482845| -|TSDB_CODE_MND_SDB_INVAID_META_ROW| 0 | 0x0324| "Invalid meta row" |-2147482844| -|TSDB_CODE_MND_SDB_INVAID_KEY_TYPE| 0 | 0x0325 |"Invalid key type" |-2147482843| -|TSDB_CODE_MND_DNODE_ALREADY_EXIST| 0 | 0x0330 | "DNode already exists"| -2147482832| -|TSDB_CODE_MND_DNODE_NOT_EXIST| 0 | 0x0331| "DNode does not exist" |-2147482831| -|TSDB_CODE_MND_VGROUP_NOT_EXIST| 0 | 0x0332 |"VGroup does not exist"| -2147482830| -|TSDB_CODE_MND_NO_REMOVE_MASTER |0 | 0x0333 | "Master DNode cannot be removed"| -2147482829| -|TSDB_CODE_MND_NO_ENOUGH_DNODES |0 | 0x0334| "Out of DNodes"| -2147482828| -|TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT |0 | 0x0335 | "Cluster cfg inconsistent"| -2147482827| -|TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION| 0 | 0x0336 | "Invalid dnode cfg option"| -2147482826| -|TSDB_CODE_MND_BALANCE_ENABLED| 0 | 0x0337 | "Balance already enabled" |-2147482825| -|TSDB_CODE_MND_VGROUP_NOT_IN_DNODE| 0 |0x0338 | "Vgroup not in dnode"| -2147482824| -|TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE| 0 | 0x0339 | "Vgroup already in dnode"| -2147482823| -|TSDB_CODE_MND_DNODE_NOT_FREE |0 | 0x033A |"Dnode not avaliable"| -2147482822| -|TSDB_CODE_MND_INVALID_CLUSTER_ID |0 |0x033B | "Cluster id not match"| -2147482821| -|TSDB_CODE_MND_NOT_READY| 0 | 0x033C |"Cluster not ready"| -2147482820| -|TSDB_CODE_MND_ACCT_ALREADY_EXIST| 0 | 0x0340 | "Account already exists" |-2147482816| -|TSDB_CODE_MND_INVALID_ACCT| 0 | 0x0341| "Invalid account"| -2147482815| -|TSDB_CODE_MND_INVALID_ACCT_OPTION| 0 | 0x0342 | "Invalid account options"| -2147482814| -|TSDB_CODE_MND_USER_ALREADY_EXIST| 0 | 0x0350 | "User already exists"| -2147482800| -|TSDB_CODE_MND_INVALID_USER |0 | 0x0351 | "Invalid user" |-2147482799| -|TSDB_CODE_MND_INVALID_USER_FORMAT| 0 |0x0352 |"Invalid user format" |-2147482798| -|TSDB_CODE_MND_INVALID_PASS_FORMAT| 0| 0x0353 | "Invalid password format"| -2147482797| -|TSDB_CODE_MND_NO_USER_FROM_CONN| 0 | 0x0354 | "Can not get user from conn"| -2147482796| -|TSDB_CODE_MND_TOO_MANY_USERS| 0 | 0x0355| "Too many users"| -2147482795| -|TSDB_CODE_MND_TABLE_ALREADY_EXIST| 0| 0x0360| "Table already exists"| -2147482784| -|TSDB_CODE_MND_INVALID_TABLE_ID| 0| 0x0361| "Table name too long"| -2147482783| -|TSDB_CODE_MND_INVALID_TABLE_NAME| 0| 0x0362 | "Table does not exist"| -2147482782| -|TSDB_CODE_MND_INVALID_TABLE_TYPE| 0| 0x0363 | "Invalid table type in tsdb"| -2147482781| -|TSDB_CODE_MND_TOO_MANY_TAGS| 0 | 0x0364| "Too many tags"| -2147482780| -|TSDB_CODE_MND_TOO_MANY_TIMESERIES| 0| 0x0366| "Too many time series"| -2147482778| -|TSDB_CODE_MND_NOT_SUPER_TABLE| 0 |0x0367| "Not super table"| -2147482777| -|TSDB_CODE_MND_COL_NAME_TOO_LONG| 0| 0x0368| "Tag name too long"| -2147482776| -|TSDB_CODE_MND_TAG_ALREAY_EXIST| 0| 0x0369| "Tag already exists"| -2147482775| -|TSDB_CODE_MND_TAG_NOT_EXIST| 0 |0x036A | "Tag does not exist" |-2147482774| -|TSDB_CODE_MND_FIELD_ALREAY_EXIST| 0 | 0x036B| "Field already exists"| -2147482773| -|TSDB_CODE_MND_FIELD_NOT_EXIST| 0 | 0x036C | "Field does not exist"| -2147482772| -|TSDB_CODE_MND_INVALID_STABLE_NAME |0 | 0x036D |"Super table does not exist" |-2147482771| -|TSDB_CODE_MND_DB_NOT_SELECTED| 0 | 0x0380 | "Database not specified or available"| -2147482752| -|TSDB_CODE_MND_DB_ALREADY_EXIST| 0 | 0x0381 | "Database already exists"| -2147482751| -|TSDB_CODE_MND_INVALID_DB_OPTION| 0 | 0x0382 | "Invalid database options"| -2147482750| -|TSDB_CODE_MND_INVALID_DB| 0 | 0x0383 | "Invalid database name"| -2147482749| -|TSDB_CODE_MND_MONITOR_DB_FORBIDDEN| 0 | 0x0384 | "Cannot delete monitor database"| -2147482748| -|TSDB_CODE_MND_TOO_MANY_DATABASES| 0| 0x0385 | "Too many databases for account"| -2147482747| -|TSDB_CODE_MND_DB_IN_DROPPING| 0 | 0x0386| "Database not available" |-2147482746| -|TSDB_CODE_DND_MSG_NOT_PROCESSED| 0| 0x0400 | "Message not processed"| -2147482624| -|TSDB_CODE_DND_OUT_OF_MEMORY |0 | 0x0401 | "Dnode out of memory"| -2147482623| -|TSDB_CODE_DND_NO_WRITE_ACCESS| 0 | 0x0402 | "No permission for disk files in dnode"| -2147482622| -|TSDB_CODE_DND_INVALID_MSG_LEN| 0 | 0x0403 | "Invalid message length"| -2147482621| -|TSDB_CODE_VND_ACTION_IN_PROGRESS |0 |0x0500| "Action in progress" |-2147482368| -|TSDB_CODE_VND_MSG_NOT_PROCESSED| 0 |0x0501 | "Message not processed" |-2147482367| -|TSDB_CODE_VND_ACTION_NEED_REPROCESSED |0 |0x0502| "Action need to be reprocessed"| -2147482366| -|TSDB_CODE_VND_INVALID_VGROUP_ID |0 | 0x0503| "Invalid Vgroup ID"| -2147482365| -|TSDB_CODE_VND_INIT_FAILED| 0 | 0x0504 | "Vnode initialization failed"| -2147482364| -|TSDB_CODE_VND_NO_DISKSPACE| 0 |0x0505| "System out of disk space" |-2147482363| -|TSDB_CODE_VND_NO_DISK_PERMISSIONS| 0 | 0x0506| "No write permission for disk files" |-2147482362| -|TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR| 0 | 0x0507 | "Missing data file"| -2147482361| -|TSDB_CODE_VND_OUT_OF_MEMORY |0| 0x0508 | "Out of memory"| -2147482360| -|TSDB_CODE_VND_APP_ERROR| 0| 0x0509 | "Unexpected generic error in vnode"| -2147482359| -|TSDB_CODE_VND_INVALID_STATUS |0| 0x0510 | "Database not ready"| -2147482352| -|TSDB_CODE_VND_NOT_SYNCED| 0 | 0x0511 | "Database suspended"| -2147482351| -|TSDB_CODE_VND_NO_WRITE_AUTH| 0 | 0x0512| "Write operation denied" |-2147482350| -|TSDB_CODE_TDB_INVALID_TABLE_ID |0 | 0x0600 | "Invalid table ID"| -2147482112| -|TSDB_CODE_TDB_INVALID_TABLE_TYPE| 0| 0x0601 |"Invalid table type"| -2147482111| -|TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION| 0| 0x0602| "Invalid table schema version"| -2147482110| -|TSDB_CODE_TDB_TABLE_ALREADY_EXIST| 0 | 0x0603| "Table already exists"| -2147482109| -|TSDB_CODE_TDB_INVALID_CONFIG| 0 | 0x0604| "Invalid configuration"| -2147482108| -|TSDB_CODE_TDB_INIT_FAILED| 0 | 0x0605| "Tsdb init failed"| -2147482107| -|TSDB_CODE_TDB_NO_DISKSPACE| 0 | 0x0606| "No diskspace for tsdb"| -2147482106| -|TSDB_CODE_TDB_NO_DISK_PERMISSIONS| 0 | 0x0607| "No permission for disk files"| -2147482105| -|TSDB_CODE_TDB_FILE_CORRUPTED| 0 | 0x0608| "Data file(s) corrupted"| -2147482104| -|TSDB_CODE_TDB_OUT_OF_MEMORY| 0 | 0x0609| "Out of memory"| -2147482103| -|TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE| 0 | 0x060A| "Tag too old"| -2147482102| -|TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE |0| 0x060B | "Timestamp data out of range"| -2147482101| -|TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP| 0| 0x060C| "Submit message is messed up"| -2147482100| -|TSDB_CODE_TDB_INVALID_ACTION| 0 | 0x060D | "Invalid operation"| -2147482099| -|TSDB_CODE_TDB_INVALID_CREATE_TB_MSG| 0 | 0x060E| "Invalid creation of table"| -2147482098| -|TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM| 0 | 0x060F| "No table data in memory skiplist" |-2147482097| -|TSDB_CODE_TDB_FILE_ALREADY_EXISTS| 0 | 0x0610| "File already exists"| -2147482096| -|TSDB_CODE_TDB_TABLE_RECONFIGURE| 0 | 0x0611| "Need to reconfigure table"| -2147482095| -|TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO| 0 | 0x0612| "Invalid information to create table"| -2147482094| -|TSDB_CODE_QRY_INVALID_QHANDLE| 0 | 0x0700| "Invalid handle"| -2147481856| -|TSDB_CODE_QRY_INVALID_MSG| 0 | 0x0701| "Invalid message"| -2147481855| -|TSDB_CODE_QRY_NO_DISKSPACE| 0 | 0x0702 | "No diskspace for query"| -2147481854| -|TSDB_CODE_QRY_OUT_OF_MEMORY| 0 | 0x0703 | "System out of memory"| -2147481853| -|TSDB_CODE_QRY_APP_ERROR| 0 | 0x0704 | "Unexpected generic error in query"| -2147481852| -|TSDB_CODE_QRY_DUP_JOIN_KEY| 0 | 0x0705| "Duplicated join key"| -2147481851| -|TSDB_CODE_QRY_EXCEED_TAGS_LIMIT| 0 | 0x0706 | "Tag conditon too many"| -2147481850| -|TSDB_CODE_QRY_NOT_READY |0| 0x0707 | "Query not ready" |-2147481849| -|TSDB_CODE_QRY_HAS_RSP| 0 | 0x0708| "Query should response"| -2147481848| -|TSDB_CODE_GRANT_EXPIRED| 0 | 0x0800| "License expired"| -2147481600| -|TSDB_CODE_GRANT_DNODE_LIMITED| 0 | 0x0801 | "DNode creation limited by licence"| -2147481599| -|TSDB_CODE_GRANT_ACCT_LIMITED |0| 0x0802 |"Account creation limited by license"| -2147481598| -|TSDB_CODE_GRANT_TIMESERIES_LIMITED| 0 | 0x0803 | "Table creation limited by license"| -2147481597| -|TSDB_CODE_GRANT_DB_LIMITED| 0 | 0x0804 | "DB creation limited by license"| -2147481596| -|TSDB_CODE_GRANT_USER_LIMITED| 0 | 0x0805 | "User creation limited by license"| -2147481595| -|TSDB_CODE_GRANT_CONN_LIMITED| 0| 0x0806 | "Conn creation limited by license" |-2147481594| -|TSDB_CODE_GRANT_STREAM_LIMITED| 0 | 0x0807 | "Stream creation limited by license"| -2147481593| -|TSDB_CODE_GRANT_SPEED_LIMITED| 0 | 0x0808 | "Write speed limited by license" |-2147481592| -|TSDB_CODE_GRANT_STORAGE_LIMITED| 0 |0x0809 | "Storage capacity limited by license"| -2147481591| -|TSDB_CODE_GRANT_QUERYTIME_LIMITED| 0 | 0x080A | "Query time limited by license" |-2147481590| -|TSDB_CODE_GRANT_CPU_LIMITED| 0 |0x080B |"CPU cores limited by license"| -2147481589| -|TSDB_CODE_SYN_INVALID_CONFIG| 0 | 0x0900| "Invalid Sync Configuration"| -2147481344| -|TSDB_CODE_SYN_NOT_ENABLED| 0 | 0x0901 | "Sync module not enabled" |-2147481343| -|TSDB_CODE_WAL_APP_ERROR| 0| 0x1000 | "Unexpected generic error in wal" |-2147479552| \ No newline at end of file +| 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | +| :-------------------------------------- | :--: | :----------------: | :------------------------------------------- | :--------------- | +| TSDB_CODE_RPC_ACTION_IN_PROGRESS | 0 | 0x0001 | "Action in progress" | -2147483647 | +| TSDB_CODE_RPC_AUTH_REQUIRED | 0 | 0x0002 | "Authentication required" | -2147483646 | +| TSDB_CODE_RPC_AUTH_FAILURE | 0 | 0x0003 | "Authentication failure" | -2147483645 | +| TSDB_CODE_RPC_REDIRECT | 0 | 0x0004 | "Redirect" | -2147483644 | +| TSDB_CODE_RPC_NOT_READY | 0 | 0x0005 | "System not ready" | -2147483643 | +| TSDB_CODE_RPC_ALREADY_PROCESSED | 0 | 0x0006 | "Message already processed" | -2147483642 | +| TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED | 0 | 0x0007 | "Last session not finished" | -2147483641 | +| TSDB_CODE_RPC_MISMATCHED_LINK_ID | 0 | 0x0008 | "Mismatched meter id" | -2147483640 | +| TSDB_CODE_RPC_TOO_SLOW | 0 | 0x0009 | "Processing of request timed out" | -2147483639 | +| TSDB_CODE_RPC_MAX_SESSIONS | 0 | 0x000A | "Number of sessions reached limit" | -2147483638 | +| TSDB_CODE_RPC_NETWORK_UNAVAIL | 0 | 0x000B | "Unable to establish connection" | -2147483637 | +| TSDB_CODE_RPC_APP_ERROR | 0 | 0x000C | "Unexpected generic error in RPC" | -2147483636 | +| TSDB_CODE_RPC_UNEXPECTED_RESPONSE | 0 | 0x000D | "Unexpected response" | -2147483635 | +| TSDB_CODE_RPC_INVALID_VALUE | 0 | 0x000E | "Invalid value" | -2147483634 | +| TSDB_CODE_RPC_INVALID_TRAN_ID | 0 | 0x000F | "Invalid transaction id" | -2147483633 | +| TSDB_CODE_RPC_INVALID_SESSION_ID | 0 | 0x0010 | "Invalid session id" | -2147483632 | +| TSDB_CODE_RPC_INVALID_MSG_TYPE | 0 | 0x0011 | "Invalid message type" | -2147483631 | +| TSDB_CODE_RPC_INVALID_RESPONSE_TYPE | 0 | 0x0012 | "Invalid response type" | -2147483630 | +| TSDB_CODE_RPC_INVALID_TIME_STAMP | 0 | 0x0013 | "Invalid timestamp" | -2147483629 | +| TSDB_CODE_COM_OPS_NOT_SUPPORT | 0 | 0x0100 | "Operation not supported" | -2147483392 | +| TSDB_CODE_COM_MEMORY_CORRUPTED | 0 | 0x0101 | "Memory corrupted" | -2147483391 | +| TSDB_CODE_COM_OUT_OF_MEMORY | 0 | 0x0102 | "Out of memory" | -2147483390 | +| TSDB_CODE_COM_INVALID_CFG_MSG | 0 | 0x0103 | "Invalid config message" | -2147483389 | +| TSDB_CODE_COM_FILE_CORRUPTED | 0 | 0x0104 | "Data file corrupted" | -2147483388 | +| TSDB_CODE_TSC_INVALID_SQL | 0 | 0x0200 | "Invalid SQL statement" | -2147483136 | +| TSDB_CODE_TSC_INVALID_QHANDLE | 0 | 0x0201 | "Invalid qhandle" | -2147483135 | +| TSDB_CODE_TSC_INVALID_TIME_STAMP | 0 | 0x0202 | "Invalid combination of client/service time" | -2147483134 | +| TSDB_CODE_TSC_INVALID_VALUE | 0 | 0x0203 | "Invalid value in client" | -2147483133 | +| TSDB_CODE_TSC_INVALID_VERSION | 0 | 0x0204 | "Invalid client version" | -2147483132 | +| TSDB_CODE_TSC_INVALID_IE | 0 | 0x0205 | "Invalid client ie" | -2147483131 | +| TSDB_CODE_TSC_INVALID_FQDN | 0 | 0x0206 | "Invalid host name" | -2147483130 | +| TSDB_CODE_TSC_INVALID_USER_LENGTH | 0 | 0x0207 | "Invalid user name" | -2147483129 | +| TSDB_CODE_TSC_INVALID_PASS_LENGTH | 0 | 0x0208 | "Invalid password" | -2147483128 | +| TSDB_CODE_TSC_INVALID_DB_LENGTH | 0 | 0x0209 | "Database name too long" | -2147483127 | +| TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH | 0 | 0x020A | "Table name too long" | -2147483126 | +| TSDB_CODE_TSC_INVALID_CONNECTION | 0 | 0x020B | "Invalid connection" | -2147483125 | +| TSDB_CODE_TSC_OUT_OF_MEMORY | 0 | 0x020C | "System out of memory" | -2147483124 | +| TSDB_CODE_TSC_NO_DISKSPACE | 0 | 0x020D | "System out of disk space" | -2147483123 | +| TSDB_CODE_TSC_QUERY_CACHE_ERASED | 0 | 0x020E | "Query cache erased" | -2147483122 | +| TSDB_CODE_TSC_QUERY_CANCELLED | 0 | 0x020F | "Query terminated" | -2147483121 | +| TSDB_CODE_TSC_SORTED_RES_TOO_MANY | 0 | 0x0210 | "Result set too large to be sorted" | -2147483120 | +| TSDB_CODE_TSC_APP_ERROR | 0 | 0x0211 | "Application error" | -2147483119 | +| TSDB_CODE_TSC_ACTION_IN_PROGRESS | 0 | 0x0212 | "Action in progress" | -2147483118 | +| TSDB_CODE_TSC_DISCONNECTED | 0 | 0x0213 | "Disconnected from service" | -2147483117 | +| TSDB_CODE_TSC_NO_WRITE_AUTH | 0 | 0x0214 | "No write permission" | -2147483116 | +| TSDB_CODE_MND_MSG_NOT_PROCESSED | 0 | 0x0300 | "Message not processed" | -2147482880 | +| TSDB_CODE_MND_ACTION_IN_PROGRESS | 0 | 0x0301 | "Message is progressing" | -2147482879 | +| TSDB_CODE_MND_ACTION_NEED_REPROCESSED | 0 | 0x0302 | "Messag need to be reprocessed" | -2147482878 | +| TSDB_CODE_MND_NO_RIGHTS | 0 | 0x0303 | "Insufficient privilege for operation" | -2147482877 | +| TSDB_CODE_MND_APP_ERROR | 0 | 0x0304 | "Unexpected generic error in mnode" | -2147482876 | +| TSDB_CODE_MND_INVALID_CONNECTION | 0 | 0x0305 | "Invalid message connection" | -2147482875 | +| TSDB_CODE_MND_INVALID_MSG_VERSION | 0 | 0x0306 | "Incompatible protocol version" | -2147482874 | +| TSDB_CODE_MND_INVALID_MSG_LEN | 0 | 0x0307 | "Invalid message length" | -2147482873 | +| TSDB_CODE_MND_INVALID_MSG_TYPE | 0 | 0x0308 | "Invalid message type" | -2147482872 | +| TSDB_CODE_MND_TOO_MANY_SHELL_CONNS | 0 | 0x0309 | "Too many connections" | -2147482871 | +| TSDB_CODE_MND_OUT_OF_MEMORY | 0 | 0x030A | "Out of memory in mnode" | -2147482870 | +| TSDB_CODE_MND_INVALID_SHOWOBJ | 0 | 0x030B | "Data expired" | -2147482869 | +| TSDB_CODE_MND_INVALID_QUERY_ID | 0 | 0x030C | "Invalid query id" | -2147482868 | +| TSDB_CODE_MND_INVALID_STREAM_ID | 0 | 0x030D | "Invalid stream id" | -2147482867 | +| TSDB_CODE_MND_INVALID_CONN_ID | 0 | 0x030E | "Invalid connection id" | -2147482866 | +| TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE | 0 | 0x0320 | "Object already there" | -2147482848 | +| TSDB_CODE_MND_SDB_ERROR | 0 | 0x0321 | "Unexpected generic error in sdb" | -2147482847 | +| TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE | 0 | 0x0322 | "Invalid table type" | -2147482846 | +| TSDB_CODE_MND_SDB_OBJ_NOT_THERE | 0 | 0x0323 | "Object not there" | -2147482845 | +| TSDB_CODE_MND_SDB_INVAID_META_ROW | 0 | 0x0324 | "Invalid meta row" | -2147482844 | +| TSDB_CODE_MND_SDB_INVAID_KEY_TYPE | 0 | 0x0325 | "Invalid key type" | -2147482843 | +| TSDB_CODE_MND_DNODE_ALREADY_EXIST | 0 | 0x0330 | "DNode already exists" | -2147482832 | +| TSDB_CODE_MND_DNODE_NOT_EXIST | 0 | 0x0331 | "DNode does not exist" | -2147482831 | +| TSDB_CODE_MND_VGROUP_NOT_EXIST | 0 | 0x0332 | "VGroup does not exist" | -2147482830 | +| TSDB_CODE_MND_NO_REMOVE_MASTER | 0 | 0x0333 | "Master DNode cannot be removed" | -2147482829 | +| TSDB_CODE_MND_NO_ENOUGH_DNODES | 0 | 0x0334 | "Out of DNodes" | -2147482828 | +| TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT | 0 | 0x0335 | "Cluster cfg inconsistent" | -2147482827 | +| TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION | 0 | 0x0336 | "Invalid dnode cfg option" | -2147482826 | +| TSDB_CODE_MND_BALANCE_ENABLED | 0 | 0x0337 | "Balance already enabled" | -2147482825 | +| TSDB_CODE_MND_VGROUP_NOT_IN_DNODE | 0 | 0x0338 | "Vgroup not in dnode" | -2147482824 | +| TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE | 0 | 0x0339 | "Vgroup already in dnode" | -2147482823 | +| TSDB_CODE_MND_DNODE_NOT_FREE | 0 | 0x033A | "Dnode not avaliable" | -2147482822 | +| TSDB_CODE_MND_INVALID_CLUSTER_ID | 0 | 0x033B | "Cluster id not match" | -2147482821 | +| TSDB_CODE_MND_NOT_READY | 0 | 0x033C | "Cluster not ready" | -2147482820 | +| TSDB_CODE_MND_ACCT_ALREADY_EXIST | 0 | 0x0340 | "Account already exists" | -2147482816 | +| TSDB_CODE_MND_INVALID_ACCT | 0 | 0x0341 | "Invalid account" | -2147482815 | +| TSDB_CODE_MND_INVALID_ACCT_OPTION | 0 | 0x0342 | "Invalid account options" | -2147482814 | +| TSDB_CODE_MND_USER_ALREADY_EXIST | 0 | 0x0350 | "User already exists" | -2147482800 | +| TSDB_CODE_MND_INVALID_USER | 0 | 0x0351 | "Invalid user" | -2147482799 | +| TSDB_CODE_MND_INVALID_USER_FORMAT | 0 | 0x0352 | "Invalid user format" | -2147482798 | +| TSDB_CODE_MND_INVALID_PASS_FORMAT | 0 | 0x0353 | "Invalid password format" | -2147482797 | +| TSDB_CODE_MND_NO_USER_FROM_CONN | 0 | 0x0354 | "Can not get user from conn" | -2147482796 | +| TSDB_CODE_MND_TOO_MANY_USERS | 0 | 0x0355 | "Too many users" | -2147482795 | +| TSDB_CODE_MND_TABLE_ALREADY_EXIST | 0 | 0x0360 | "Table already exists" | -2147482784 | +| TSDB_CODE_MND_INVALID_TABLE_ID | 0 | 0x0361 | "Table name too long" | -2147482783 | +| TSDB_CODE_MND_INVALID_TABLE_NAME | 0 | 0x0362 | "Table does not exist" | -2147482782 | +| TSDB_CODE_MND_INVALID_TABLE_TYPE | 0 | 0x0363 | "Invalid table type in tsdb" | -2147482781 | +| TSDB_CODE_MND_TOO_MANY_TAGS | 0 | 0x0364 | "Too many tags" | -2147482780 | +| TSDB_CODE_MND_TOO_MANY_TIMESERIES | 0 | 0x0366 | "Too many time series" | -2147482778 | +| TSDB_CODE_MND_NOT_SUPER_TABLE | 0 | 0x0367 | "Not super table" | -2147482777 | +| TSDB_CODE_MND_COL_NAME_TOO_LONG | 0 | 0x0368 | "Tag name too long" | -2147482776 | +| TSDB_CODE_MND_TAG_ALREAY_EXIST | 0 | 0x0369 | "Tag already exists" | -2147482775 | +| TSDB_CODE_MND_TAG_NOT_EXIST | 0 | 0x036A | "Tag does not exist" | -2147482774 | +| TSDB_CODE_MND_FIELD_ALREAY_EXIST | 0 | 0x036B | "Field already exists" | -2147482773 | +| TSDB_CODE_MND_FIELD_NOT_EXIST | 0 | 0x036C | "Field does not exist" | -2147482772 | +| TSDB_CODE_MND_INVALID_STABLE_NAME | 0 | 0x036D | "Super table does not exist" | -2147482771 | +| TSDB_CODE_MND_DB_NOT_SELECTED | 0 | 0x0380 | "Database not specified or available" | -2147482752 | +| TSDB_CODE_MND_DB_ALREADY_EXIST | 0 | 0x0381 | "Database already exists" | -2147482751 | +| TSDB_CODE_MND_INVALID_DB_OPTION | 0 | 0x0382 | "Invalid database options" | -2147482750 | +| TSDB_CODE_MND_INVALID_DB | 0 | 0x0383 | "Invalid database name" | -2147482749 | +| TSDB_CODE_MND_MONITOR_DB_FORBIDDEN | 0 | 0x0384 | "Cannot delete monitor database" | -2147482748 | +| TSDB_CODE_MND_TOO_MANY_DATABASES | 0 | 0x0385 | "Too many databases for account" | -2147482747 | +| TSDB_CODE_MND_DB_IN_DROPPING | 0 | 0x0386 | "Database not available" | -2147482746 | +| TSDB_CODE_DND_MSG_NOT_PROCESSED | 0 | 0x0400 | "Message not processed" | -2147482624 | +| TSDB_CODE_DND_OUT_OF_MEMORY | 0 | 0x0401 | "Dnode out of memory" | -2147482623 | +| TSDB_CODE_DND_NO_WRITE_ACCESS | 0 | 0x0402 | "No permission for disk files in dnode" | -2147482622 | +| TSDB_CODE_DND_INVALID_MSG_LEN | 0 | 0x0403 | "Invalid message length" | -2147482621 | +| TSDB_CODE_VND_ACTION_IN_PROGRESS | 0 | 0x0500 | "Action in progress" | -2147482368 | +| TSDB_CODE_VND_MSG_NOT_PROCESSED | 0 | 0x0501 | "Message not processed" | -2147482367 | +| TSDB_CODE_VND_ACTION_NEED_REPROCESSED | 0 | 0x0502 | "Action need to be reprocessed" | -2147482366 | +| TSDB_CODE_VND_INVALID_VGROUP_ID | 0 | 0x0503 | "Invalid Vgroup ID" | -2147482365 | +| TSDB_CODE_VND_INIT_FAILED | 0 | 0x0504 | "Vnode initialization failed" | -2147482364 | +| TSDB_CODE_VND_NO_DISKSPACE | 0 | 0x0505 | "System out of disk space" | -2147482363 | +| TSDB_CODE_VND_NO_DISK_PERMISSIONS | 0 | 0x0506 | "No write permission for disk files" | -2147482362 | +| TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR | 0 | 0x0507 | "Missing data file" | -2147482361 | +| TSDB_CODE_VND_OUT_OF_MEMORY | 0 | 0x0508 | "Out of memory" | -2147482360 | +| TSDB_CODE_VND_APP_ERROR | 0 | 0x0509 | "Unexpected generic error in vnode" | -2147482359 | +| TSDB_CODE_VND_INVALID_STATUS | 0 | 0x0510 | "Database not ready" | -2147482352 | +| TSDB_CODE_VND_NOT_SYNCED | 0 | 0x0511 | "Database suspended" | -2147482351 | +| TSDB_CODE_VND_NO_WRITE_AUTH | 0 | 0x0512 | "Write operation denied" | -2147482350 | +| TSDB_CODE_TDB_INVALID_TABLE_ID | 0 | 0x0600 | "Invalid table ID" | -2147482112 | +| TSDB_CODE_TDB_INVALID_TABLE_TYPE | 0 | 0x0601 | "Invalid table type" | -2147482111 | +| TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION | 0 | 0x0602 | "Invalid table schema version" | -2147482110 | +| TSDB_CODE_TDB_TABLE_ALREADY_EXIST | 0 | 0x0603 | "Table already exists" | -2147482109 | +| TSDB_CODE_TDB_INVALID_CONFIG | 0 | 0x0604 | "Invalid configuration" | -2147482108 | +| TSDB_CODE_TDB_INIT_FAILED | 0 | 0x0605 | "Tsdb init failed" | -2147482107 | +| TSDB_CODE_TDB_NO_DISKSPACE | 0 | 0x0606 | "No diskspace for tsdb" | -2147482106 | +| TSDB_CODE_TDB_NO_DISK_PERMISSIONS | 0 | 0x0607 | "No permission for disk files" | -2147482105 | +| TSDB_CODE_TDB_FILE_CORRUPTED | 0 | 0x0608 | "Data file(s) corrupted" | -2147482104 | +| TSDB_CODE_TDB_OUT_OF_MEMORY | 0 | 0x0609 | "Out of memory" | -2147482103 | +| TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE | 0 | 0x060A | "Tag too old" | -2147482102 | +| TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE | 0 | 0x060B | "Timestamp data out of range" | -2147482101 | +| TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP | 0 | 0x060C | "Submit message is messed up" | -2147482100 | +| TSDB_CODE_TDB_INVALID_ACTION | 0 | 0x060D | "Invalid operation" | -2147482099 | +| TSDB_CODE_TDB_INVALID_CREATE_TB_MSG | 0 | 0x060E | "Invalid creation of table" | -2147482098 | +| TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM | 0 | 0x060F | "No table data in memory skiplist" | -2147482097 | +| TSDB_CODE_TDB_FILE_ALREADY_EXISTS | 0 | 0x0610 | "File already exists" | -2147482096 | +| TSDB_CODE_TDB_TABLE_RECONFIGURE | 0 | 0x0611 | "Need to reconfigure table" | -2147482095 | +| TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO | 0 | 0x0612 | "Invalid information to create table" | -2147482094 | +| TSDB_CODE_QRY_INVALID_QHANDLE | 0 | 0x0700 | "Invalid handle" | -2147481856 | +| TSDB_CODE_QRY_INVALID_MSG | 0 | 0x0701 | "Invalid message" | -2147481855 | +| TSDB_CODE_QRY_NO_DISKSPACE | 0 | 0x0702 | "No diskspace for query" | -2147481854 | +| TSDB_CODE_QRY_OUT_OF_MEMORY | 0 | 0x0703 | "System out of memory" | -2147481853 | +| TSDB_CODE_QRY_APP_ERROR | 0 | 0x0704 | "Unexpected generic error in query" | -2147481852 | +| TSDB_CODE_QRY_DUP_JOIN_KEY | 0 | 0x0705 | "Duplicated join key" | -2147481851 | +| TSDB_CODE_QRY_EXCEED_TAGS_LIMIT | 0 | 0x0706 | "Tag conditon too many" | -2147481850 | +| TSDB_CODE_QRY_NOT_READY | 0 | 0x0707 | "Query not ready" | -2147481849 | +| TSDB_CODE_QRY_HAS_RSP | 0 | 0x0708 | "Query should response" | -2147481848 | +| TSDB_CODE_GRANT_EXPIRED | 0 | 0x0800 | "License expired" | -2147481600 | +| TSDB_CODE_GRANT_DNODE_LIMITED | 0 | 0x0801 | "DNode creation limited by licence" | -2147481599 | +| TSDB_CODE_GRANT_ACCT_LIMITED | 0 | 0x0802 | "Account creation limited by license" | -2147481598 | +| TSDB_CODE_GRANT_TIMESERIES_LIMITED | 0 | 0x0803 | "Table creation limited by license" | -2147481597 | +| TSDB_CODE_GRANT_DB_LIMITED | 0 | 0x0804 | "DB creation limited by license" | -2147481596 | +| TSDB_CODE_GRANT_USER_LIMITED | 0 | 0x0805 | "User creation limited by license" | -2147481595 | +| TSDB_CODE_GRANT_CONN_LIMITED | 0 | 0x0806 | "Conn creation limited by license" | -2147481594 | +| TSDB_CODE_GRANT_STREAM_LIMITED | 0 | 0x0807 | "Stream creation limited by license" | -2147481593 | +| TSDB_CODE_GRANT_SPEED_LIMITED | 0 | 0x0808 | "Write speed limited by license" | -2147481592 | +| TSDB_CODE_GRANT_STORAGE_LIMITED | 0 | 0x0809 | "Storage capacity limited by license" | -2147481591 | +| TSDB_CODE_GRANT_QUERYTIME_LIMITED | 0 | 0x080A | "Query time limited by license" | -2147481590 | +| TSDB_CODE_GRANT_CPU_LIMITED | 0 | 0x080B | "CPU cores limited by license" | -2147481589 | +| TSDB_CODE_SYN_INVALID_CONFIG | 0 | 0x0900 | "Invalid Sync Configuration" | -2147481344 | +| TSDB_CODE_SYN_NOT_ENABLED | 0 | 0x0901 | "Sync module not enabled" | -2147481343 | +| TSDB_CODE_WAL_APP_ERROR | 0 | 0x1000 | "Unexpected generic error in wal" | -2147479552 | diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index fbb82ee140..bcf80d8fa2 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -41,9 +41,9 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 -| | 类型 | Bytes | 说明 | +| # | **类型** | **Bytes** | **说明** | | ---- | :-------: | ------ | ------------------------------------------------------------ | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18 版本开始，已经去除了这一时间范围限制） | +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18.0 版本开始，已经去除了这一时间范围限制） | | 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | | 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | @@ -53,6 +53,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM | 8 | TINYINT | 1 | 单字节整型，范围 [-127, 127], -128 用于 NULL | | 9 | BOOL | 1 | 布尔型，{true, false} | | 10 | NCHAR | 自定义 | 记录包含多字节字符在内的字符串，如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用，字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小，类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符，会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度，将会报错。 | + **Tips**: 1. TDengine 对 SQL 语句中的英文字符不区分大小写，自动转化为小写执行。因此用户大小写敏感的字符串及密码，需要使用单引号将字符串引起来。 @@ -63,11 +64,11 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM - **创建数据库** ```mysql - CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1]; + CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1]; ``` - 说明： + 说明： - 1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； + 1) KEEP是该数据库的数据保留多长天数，缺省是3650天(10年)，数据库会自动删除超过时限的数据； 2) UPDATE 标志数据库支持更新相同时间戳数据； @@ -75,7 +76,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 4) 一条SQL 语句的最大长度为65480个字符； - 5) 数据库还有更多与存储相关的配置参数，请参见系统管理。 + 5) 数据库还有更多与存储相关的配置参数，请参见 [服务端配置](https://www.taosdata.com/cn/documentation/taos-sql#management) 章节。 - **显示系统当前参数** @@ -167,22 +168,22 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name TAGS (tag_value1, ...); ``` - 以指定的超级表为模板，指定 tags 的值来创建数据表。 + 以指定的超级表为模板，指定 TAGS 的值来创建数据表。 -- **以超级表为模板创建数据表，并指定具体的 tags 列** +- **以超级表为模板创建数据表，并指定具体的 TAGS 列** ```mysql CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...); ``` - 以指定的超级表为模板，指定一部分 tags 列的值来创建数据表。（没被指定的 tags 列会设为空值。） - 说明：从 2.0.17 版本开始支持这种方式。在之前的版本中，不允许指定 tags 列，而必须显式给出所有 tags 列的取值。 + 以指定的超级表为模板，指定一部分 TAGS 列的值来创建数据表（没被指定的 TAGS 列会设为空值）。 + 说明：从 2.0.17.0 版本开始支持这种方式。在之前的版本中，不允许指定 TAGS 列，而必须显式给出所有 TAGS 列的取值。 - **批量创建数据表** ```mysql CREATE TABLE [IF NOT EXISTS] tb_name1 USING stb_name TAGS (tag_value1, ...) tb_name2 USING stb_name TAGS (tag_value2, ...) ...; ``` - 以更快的速度批量创建大量数据表。（服务器端 2.0.14 及以上版本） + 以更快的速度批量创建大量数据表（服务器端 2.0.14 及以上版本）。 说明： @@ -220,6 +221,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql SET MAX_BINARY_DISPLAY_WIDTH ; ``` + 如显示的内容后面以...结尾时，表示该内容已被截断，可通过本命令修改显示字符宽度以显示完整的内容。 - **获取表的结构信息** @@ -236,14 +238,14 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM 1) 列的最大个数为1024，最小个数为2； - 2) 列名最大长度为64； + 2) 列名最大长度为64。 - **表删除列** ```mysql ALTER TABLE tb_name DROP COLUMN field_name; ``` - 如果表是通过[超级表](../super-table/)创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构 + 如果表是通过超级表创建，更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表，可以直接修改表结构。 ## 超级表STable管理 @@ -254,7 +256,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql CREATE STABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` - 创建 STable，与创建表的 SQL 语法相似，但需指定 TAGS 字段的名称和类型 + 创建 STable，与创建表的 SQL 语法相似，但需要指定 TAGS 字段的名称和类型 说明： @@ -276,7 +278,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM - **显示当前数据库下的所有超级表信息** ```mysql - SHOW STABLES [LIKE tb_name_wildcar]; + SHOW STABLES [LIKE tb_name_wildcard]; ``` 查看数据库内全部 STable，及其相关信息，包括 STable 的名称、创建时间、列数量、标签（TAG）数量、通过该 STable 建表的数量。 @@ -341,7 +343,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql INSERT INTO tb_name VALUES (field_value, ...); ``` - 向表tb_name中插入一条记录 + 向表tb_name中插入一条记录。 - **插入一条记录，数据对应到指定的列** ```mysql @@ -353,42 +355,51 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM ```mysql INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 向表tb_name中插入多条记录 + 向表tb_name中插入多条记录。 **注意**：在使用“插入多条记录”方式写入数据时，不能把第一列的时间戳取值都设为now，否则会导致语句中的多条记录使用相同的时间戳，于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。 - **按指定的列插入多条记录** ```mysql INSERT INTO tb_name (field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 向表tb_name中按指定的列插入多条记录 + 向表tb_name中按指定的列插入多条记录。 - **向多个表插入多条记录** ```mysql INSERT INTO tb1_name VALUES (field1_value1, ...) (field1_value2, ...) ... tb2_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 同时向表tb1_name和tb2_name中分别插入多条记录 + 同时向表tb1_name和tb2_name中分别插入多条记录。 - **同时向多个表按列插入多条记录** ```mysql INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ... tb2_name (tb2_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 同时向表tb1_name和tb2_name中按列分别插入多条记录 + 同时向表tb1_name和tb2_name中按列分别插入多条记录。 - 注意：允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 + 注意： + 1) 如果时间戳为now，系统将自动使用客户端当前时间作为该记录的时间戳； + 2) 允许插入的最老记录的时间戳，是相对于当前服务器时间，减去配置的keep值（数据保留的天数），允许插入的最新记录的时间戳，是相对于当前服务器时间，加上配置的days值（数据文件存储数据的时间跨度，单位为天）。keep和days都是可以在创建数据库时指定的，缺省值分别是3650天和10天。 - **插入记录时自动建表** ```mysql INSERT INTO tb_name USING stb_name TAGS (tag_value1, ...) VALUES (field_value1, ...); ``` - 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 tags 取值。 + 如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。 -- **插入记录时自动建表，并指定具体的 tags 列** +- **插入记录时自动建表，并指定具体的 TAGS 列** ```mysql INSERT INTO tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...) VALUES (field_value1, ...); ``` - 在自动建表时，可以只是指定部分 tags 列的取值，未被指定的 tags 列将取为空值。 + 在自动建表时，可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将取为空值。 + +- **同时向多个表按列插入多条记录，自动建表** + ```mysql + INSERT INTO tb1_name (tb1_field1_name, ...) [USING stb1_name TAGS (tag_value1, ...)] VALUES (field1_value1, ...) (field1_value2, ...) ... + tb2_name (tb2_field1_name, ...) [USING stb2_name TAGS (tag_value2, ...)] VALUES (field1_value1, ...) (field1_value2, ...) ...; + ``` + 以自动建表的方式，同时向表tb1_name和tb2_name中按列分别插入多条记录。 **历史记录写入**：可使用IMPORT或者INSERT命令，IMPORT的语法，功能与INSERT完全一样。 @@ -471,7 +482,7 @@ Query OK, 9 row(s) in set (0.002022s) SELECT * FROM d1001; SELECT d1001.* FROM d1001; ``` -在Join查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。 +在JOIN查询中，带前缀的\*和不带前缀\*返回的结果有差别， \*返回全部表的所有列数据（不包含标签），带前缀的通配符，则只返回该表的列数据。 ```mysql taos> SELECT * FROM d1001, d1003 WHERE d1001.ts=d1003.ts; ts | current | voltage | phase | ts | current | voltage | phase | @@ -487,7 +498,7 @@ taos> SELECT d1001.* FROM d1001,d1003 WHERE d1001.ts = d1003.ts; Query OK, 1 row(s) in set (0.020443s) ``` -在使用SQL函数来进行查询过程中，部分SQL函数支持通配符操作。其中的区别在于： +在使用SQL函数来进行查询的过程中，部分SQL函数支持通配符操作。其中的区别在于： ```count(*)```函数只返回一列。```first```、```last```、```last_row```函数则是返回全部列。 ```mysql @@ -522,12 +533,12 @@ Query OK, 2 row(s) in set (0.003112s) ##### 获取标签列的去重取值 -从 2.0.15 版本开始，支持在超级表查询标签列时，指定 distinct 关键字，这样将返回指定标签列的所有不重复取值。 +从 2.0.15 版本开始，支持在超级表查询标签列时，指定 DISTINCT 关键字，这样将返回指定标签列的所有不重复取值。 ```mysql SELECT DISTINCT tag_name FROM stb_name; ``` -注意：目前 distinct 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 +注意：目前 DISTINCT 关键字只支持对超级表的标签列进行去重，而不能用于普通列。 @@ -562,7 +573,7 @@ SELECT * FROM d1001; #### 特殊功能 -部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database() +部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database()： ```mysql taos> SELECT DATABASE(); database() | @@ -570,7 +581,7 @@ taos> SELECT DATABASE(); power | Query OK, 1 row(s) in set (0.000079s) ``` -如果登录的时候没有指定默认数据库，且没有使用```use```命令切换数据，则返回NULL。 +如果登录的时候没有指定默认数据库，且没有使用```USE```命令切换数据，则返回NULL。 ```mysql taos> SELECT DATABASE(); database() | @@ -578,7 +589,7 @@ taos> SELECT DATABASE(); NULL | Query OK, 1 row(s) in set (0.000184s) ``` -获取服务器和客户端版本号: +获取服务器和客户端版本号： ```mysql taos> SELECT CLIENT_VERSION(); client_version() | @@ -622,7 +633,7 @@ SELECT TBNAME, location FROM meters; ```mysql SELECT COUNT(TBNAME) FROM meters; ``` -以上两个查询均只支持在Where条件子句中添加针对标签（TAGS）的过滤条件。例如： +以上两个查询均只支持在WHERE条件子句中添加针对标签（TAGS）的过滤条件。例如： ```mysql taos> SELECT TBNAME, location FROM meters; tbname | location | @@ -648,30 +659,31 @@ Query OK, 1 row(s) in set (0.001091s) - 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 - 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。 -- 通过 ">>" 输出结果可以导出到指定文件。 +- 通过 “>>” 输出结果可以导出到指定文件。 ### 支持的条件过滤操作 -| Operation | Note | Applicable Data Types | -| ----------- | ----------------------------- | ------------------------------------- | -| > | larger than | **`timestamp`** and all numeric types | -| < | smaller than | **`timestamp`** and all numeric types | -| >= | larger than or equal to | **`timestamp`** and all numeric types | -| <= | smaller than or equal to | **`timestamp`** and all numeric types | -| = | equal to | all types | -| <> | not equal to | all types | -| between and | within a certain range | **`timestamp`** and all numeric types | -| % | match with any char sequences | **`binary`** **`nchar`** | -| _ | match with a single char | **`binary`** **`nchar`** | +| **Operation** | **Note** | **Applicable Data Types** | +| --------------- | ----------------------------- | ------------------------------------- | +| > | larger than | **`timestamp`** and all numeric types | +| < | smaller than | **`timestamp`** and all numeric types | +| >= | larger than or equal to | **`timestamp`** and all numeric types | +| <= | smaller than or equal to | **`timestamp`** and all numeric types | +| = | equal to | all types | +| <> | not equal to | all types | +| between and | within a certain range | **`timestamp`** and all numeric types | +| % | match with any char sequences | **`binary`** **`nchar`** | +| _ | match with a single char | **`binary`** **`nchar`** | 1. 同时进行多个字段的范围过滤，需要使用关键词 AND 来连接不同的查询条件，暂不支持 OR 连接的不同列之间的查询过滤条件。 2. 针对单一字段的过滤，如果是时间过滤条件，则一条语句中只支持设定一个；但针对其他的（普通）列或标签列，则可以使用 `OR` 关键字进行组合条件的查询过滤。例如：((value > 20 AND value < 30) OR (value < 12)) 。 3. 从 2.0.17 版本开始，条件过滤开始支持 BETWEEN AND 语法，例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 -### UNION ALL 操作符 + +### UNION ALL 操作符 ```mysql SELECT ... @@ -691,37 +704,38 @@ TDengine 支持 UNION ALL 操作符。也就是说，如果多个 SELECT 子句 ### SQL 示例 -- 对于下面的例子，表tb1用以下语句创建 +- 对于下面的例子，表tb1用以下语句创建： ```mysql CREATE TABLE tb1 (ts TIMESTAMP, col1 INT, col2 FLOAT, col3 BINARY(50)); ``` -- 查询tb1刚过去的一个小时的所有记录 +- 查询tb1刚过去的一个小时的所有记录： ```mysql SELECT * FROM tb1 WHERE ts >= NOW - 1h; ``` -- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序 +- 查询表tb1从2018-06-01 08:00:00.000 到2018-06-02 08:00:00.000时间范围，并且col3的字符串是'nny'结尾的记录，结果按照时间戳降序： ```mysql SELECT * FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND ts <= '2018-06-02 08:00:00.000' AND col3 LIKE '%nny' ORDER BY ts DESC; ``` -- 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始 +- 查询col1与col2的和，并取名complex, 时间大于2018-06-01 08:00:00.000, col2大于1.2，结果输出仅仅10条记录，从第5条开始： ```mysql SELECT (col1 + col2) AS 'complex' FROM tb1 WHERE ts > '2018-06-01 08:00:00.000' AND col2 > 1.2 LIMIT 10 OFFSET 5; ``` -- 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`. +- 查询过去10分钟的记录，col2的值大于3.14，并且将结果输出到文件 `/home/testoutpu.csv`： ```mysql SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv; ``` -## SQL 函数 + +## SQL 函数 ### 聚合函数 @@ -741,7 +755,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 说明： - 1）可以使用星号\*来替代具体的字段，使用星号(\*)返回全部记录数量。 + 1）可以使用星号(\*)来替代具体的字段，使用星号(\*)返回全部记录数量。 2）针对同一表的（不包含NULL值）字段查询结果均相同。 @@ -1012,7 +1026,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 1）*k*值取值范围1≤*k*≤100； - 2）系统同时返回该记录关联的时间戳列。 + 2）系统同时返回该记录关联的时间戳列； + + 3）限制：TOP函数不支持FILL子句。 示例： ```mysql @@ -1048,7 +1064,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 1）*k*值取值范围1≤*k*≤100； - 2）系统同时返回该记录关联的时间戳列。 + 2）系统同时返回该记录关联的时间戳列； + + 3）限制：BOTTOM函数不支持FILL子句。 示例： ```mysql @@ -1124,7 +1142,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 适用于：**表、超级表**。 - 说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。 + 说明：与LAST函数不同，LAST_ROW不支持时间范围限制，强制返回最后一条记录。 + + 限制：LAST_ROW()不能与INTERVAL一起使用。 示例： ```mysql @@ -1233,40 +1253,40 @@ SELECT function_list FROM tb_name [WHERE where_condition] INTERVAL (interval [, offset]) [SLIDING sliding] - [FILL ({NONE | VALUE | PREV | NULL | LINEAR})] + [FILL ({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] SELECT function_list FROM stb_name [WHERE where_condition] INTERVAL (interval [, offset]) [SLIDING sliding] - [FILL ({ VALUE | PREV | NULL | LINEAR})] + [FILL ({ VALUE | PREV | NULL | LINEAR | NEXT})] [GROUP BY tags] ``` - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。 -- WHERE语句可以指定查询的起止时间和其他过滤条件 -- SLIDING语句用于指定聚合时间段的前向增量 +- WHERE语句可以指定查询的起止时间和其他过滤条件。 +- SLIDING语句用于指定聚合时间段的前向增量。 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种： - * 不进行填充：NONE(默认填充模式)。 - * VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。 - * NULL填充：使用NULL填充数据。例如：fill(null)。 - * PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 + 1. 不进行填充：NONE(默认填充模式)。 + 2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 + 3. NULL填充：使用NULL填充数据。例如：FILL(NULL)。 + 4. PREV填充：使用前一个非NULL值填充数据。例如：FILL(PREV)。 + 5. NEXT填充：使用下一个非NULL值填充数据。例如：FILL(NEXT)。 说明： 1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。 2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 - 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。 + 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用GROUP BY语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了GROUP BY语句分组，则返回结果中每个GROUP内不按照时间序列严格单调递增。 时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。 -**示例:** 智能电表的建表语句如下： +**示例**： 智能电表的建表语句如下： ```mysql CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); ``` -针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。 -使用的查询语句如下： +针对智能电表采集的数据，以10分钟为一个阶段，计算过去24小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非NULL值填充。使用的查询语句如下： ```mysql SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters @@ -1287,15 +1307,15 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P ## TAOS SQL其他约定 -**group by的限制** +**GROUP BY的限制** -TAOS SQL支持对标签、tbname进行group by操作，也支持普通列进行group by，前提是：仅限一列且该列的唯一值小于10万个。 +TAOS SQL支持对标签、TBNAME进行GROUP BY操作，也支持普通列进行GROUP BY，前提是：仅限一列且该列的唯一值小于10万个。 -**join操作的限制** +**JOIN操作的限制** TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持两个表之间聚合后的四则运算。 -**is not null与不为空的表达式适用范围** +**IS NOT NULL与不为空的表达式适用范围** -is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 +IS NOT NULL支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 From 13004a574010119332f99c59d8ed1ac591a149c4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 25 May 2021 19:41:36 +0800 Subject: [PATCH 56/56] Feature/sangshuduo/td 4068 taosdemo stmt (#6225) * merge with develop branch. change query/tests/CMakeLists.txt to allow unused function and variable. * refactor data generating. * refactor. * refactor. * refactor. * refactor. * refactor * add prepare stmt function. * refactor get rand timestamp. * fix windows compile error. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 341 ++++++++++++++++++++++-------------- 1 file changed, 214 insertions(+), 127 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6f651c18cf..bae0476b17 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -19,6 +19,7 @@ */ #include +#include #define _GNU_SOURCE #define CURL_STATICLIB @@ -229,13 +230,13 @@ typedef struct SArguments_S { uint32_t num_of_threads; uint64_t insert_interval; int64_t query_times; - uint64_t interlace_rows; - uint64_t num_of_RPR; // num_of_records_per_req + uint32_t interlace_rows; + uint32_t num_of_RPR; // num_of_records_per_req uint64_t max_sql_len; int64_t num_of_tables; int64_t num_of_DPT; int abort; - int disorderRatio; // 0: no disorder, >0: x% + uint32_t disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision uint32_t method_of_delete; char ** arg_list; @@ -258,12 +259,12 @@ typedef struct SSuperTable_S { uint8_t autoCreateTable; // 0: create sub table, 1: auto create sub table char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - uint16_t insertMode; // 0: taosc, 1: rest, 2: stmt + uint16_t iface; // 0: taosc, 1: rest, 2: stmt int64_t childTblLimit; uint64_t childTblOffset; // int multiThreadWriteOneTbl; // 0: no, 1: yes - uint64_t interlaceRows; // + uint32_t interlaceRows; // int disorderRatio; // 0: no disorder, >0: x% int disorderRange; // ms or us by database precision uint64_t maxSqlLen; // @@ -551,6 +552,8 @@ static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, char* sqlstr, threadInfo *pThreadInfo); +static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, + int disorderRatio, int disorderRange); /* ************ Global variables ************ */ @@ -1070,7 +1073,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } printf("# Insertion interval: %"PRIu64"\n", arguments->insert_interval); - printf("# Number of records per req: %"PRIu64"\n", + printf("# Number of records per req: %ud\n", arguments->num_of_RPR); printf("# Max SQL length: %"PRIu64"\n", arguments->max_sql_len); @@ -1362,7 +1365,7 @@ static int printfInsertMeta() { g_Dbs.threadCountByCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); - printf("number of records per req: \033[33m%"PRIu64"\033[0m\n", + printf("number of records per req: \033[33m%ud\033[0m\n", g_args.num_of_RPR); printf("max sql length: \033[33m%"PRIu64"\033[0m\n", g_args.max_sql_len); @@ -1468,9 +1471,9 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].childTblPrefix); printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", - (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); + printf(" iface: \033[33m%s\033[0m\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); @@ -1488,7 +1491,7 @@ static int printfInsertMeta() { printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); } */ - printf(" interlaceRows: \033[33m%"PRIu64"\033[0m\n", + printf(" interlaceRows: \033[33m%ud\033[0m\n", g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { @@ -1566,7 +1569,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); - fprintf(fp, "number of records per req: %"PRIu64"\n", g_args.num_of_RPR); + fprintf(fp, "number of records per req: %ud\n", g_args.num_of_RPR); fprintf(fp, "max sql length: %"PRIu64"\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); @@ -1658,12 +1661,12 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].childTblPrefix); fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); - fprintf(fp, " insertMode: %s\n", - (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt"); + fprintf(fp, " iface: %s\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); - fprintf(fp, " interlace rows: %"PRIu64"\n", + fprintf(fp, " interlace rows: %ud\n", g_Dbs.db[i].superTbls[j].interlaceRows); if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { fprintf(fp, " stable insert interval: %"PRIu64"\n", @@ -1676,7 +1679,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, " multiThreadWriteOneTbl: yes\n"); } */ - fprintf(fp, " interlaceRows: %"PRIu64"\n", + fprintf(fp, " interlaceRows: %ud\n", g_Dbs.db[i].superTbls[j].interlaceRows); fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); @@ -3563,9 +3566,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // rows per table need be less than insert batch if (g_args.interlace_rows > g_args.num_of_RPR) { - printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", + printf("NOTICE: interlace rows value %ud > num_of_records_per_req %ud\n\n", g_args.interlace_rows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", + printf(" interlace rows value will be set to num_of_records_per_req %ud\n\n", g_args.num_of_RPR); prompt(); g_args.interlace_rows = g_args.num_of_RPR; @@ -3880,22 +3883,22 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt - if (insertMode && insertMode->type == cJSON_String - && insertMode->valuestring != NULL) { - if (0 == strcasecmp(insertMode->valuestring, "taosc")) { - g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; - } else if (0 == strcasecmp(insertMode->valuestring, "rest")) { - g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE; - } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) { - g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE; + cJSON *stbIface = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt + if (stbIface && stbIface->type == cJSON_String + && stbIface->valuestring != NULL) { + if (0 == strcasecmp(stbIface->valuestring, "taosc")) { + g_Dbs.db[i].superTbls[j].iface= TAOSC_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "rest")) { + g_Dbs.db[i].superTbls[j].iface= REST_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { + g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; } else { errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n", - __func__, __LINE__, insertMode->valuestring); + __func__, __LINE__, stbIface->valuestring); goto PARSE_OVER; } - } else if (!insertMode) { - g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE; + } else if (!stbIface) { + g_Dbs.db[i].superTbls[j].iface = TAOSC_IFACE; } else { errorPrint("%s", "failed to read json, insert_mode not found\n"); goto PARSE_OVER; @@ -4032,9 +4035,10 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint; // rows per table need be less than insert batch if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { - printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n", - i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); - printf(" interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n", + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %ud > num_of_records_per_req %ud\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, + g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_req %ud\n\n", g_args.num_of_RPR); prompt(); g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; @@ -4864,11 +4868,11 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); if (superTblInfo) { - if (superTblInfo->insertMode == TAOSC_IFACE) { + if (superTblInfo->iface == TAOSC_IFACE) { affectedRows = queryDbExec( pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false); - } else if (superTblInfo->insertMode == REST_IFACE) { + } else if (superTblInfo->iface == REST_IFACE) { if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, pThreadInfo->buffer, NULL /* not set result file */)) { affectedRows = -1; @@ -4877,7 +4881,7 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) } else { affectedRows = k; } - } else if (superTblInfo->insertMode == STMT_IFACE) { + } else if (superTblInfo->iface == STMT_IFACE) { debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt); if (0 != taos_stmt_execute(pThreadInfo->stmt)) { errorPrint("%s() LN%d, failied to execute insert statement\n", @@ -4888,7 +4892,7 @@ static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k) affectedRows = k; } else { errorPrint("%s() LN%d: unknown insert mode: %d\n", - __func__, __LINE__, superTblInfo->insertMode); + __func__, __LINE__, superTblInfo->iface); affectedRows = 0; } } else { @@ -4924,7 +4928,7 @@ static void getTableName(char *pTblName, } static int64_t generateDataTailWithoutStb( - uint64_t batch, char* buffer, + uint32_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, uint64_t startFrom, int64_t startTime, /* int64_t *pSamplePos, */int64_t *dataLen) { @@ -4932,7 +4936,7 @@ static int64_t generateDataTailWithoutStb( uint64_t len = 0; char *pstr = buffer; - verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); + verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch); int64_t k = 0; for (k = 0; k < batch;) { @@ -4944,22 +4948,11 @@ static int64_t generateDataTailWithoutStb( char **data_type = g_args.datatype; int lenOfBinary = g_args.len_of_binary; - int64_t randTail = DEFAULT_TIMESTAMP_STEP * k; - - if (g_args.disorderRatio != 0) { - int rand_num = taosRandom() % 100; - if (rand_num < g_args.disorderRatio) { - randTail = (randTail + - (taosRandom() % g_args.disorderRange + 1)) * (-1); - - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } else { - randTail = DEFAULT_TIMESTAMP_STEP * k; - } - retLen = generateData(data, data_type, - startTime + randTail, + startTime + getTSRandTail( + (int64_t)DEFAULT_TIMESTAMP_STEP, k, + g_args.disorderRatio, + g_args.disorderRange), lenOfBinary); if (len > remainderBufLen) @@ -4984,9 +4977,25 @@ static int64_t generateDataTailWithoutStb( return k; } -static int64_t generateStbDataTail( +static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, + int disorderRatio, int disorderRange) +{ + int64_t randTail = timeStampStep * seq; + if (disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if(rand_num < disorderRatio) { + randTail = (randTail + + (taosRandom() % disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %"PRId64"\n", randTail); + } + } + + return randTail; +} + +static int32_t generateStbDataTail( SSuperTable* superTblInfo, - uint64_t batch, char* buffer, + uint32_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows, uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) { @@ -4994,37 +5003,35 @@ static int64_t generateStbDataTail( char *pstr = buffer; - verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch); + bool tsRand; + if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; + } else { + tsRand = false; + } + verbosePrint("%s() LN%d batch=%ud\n", __func__, __LINE__, batch); - int64_t k = 0; + int32_t k = 0; for (k = 0; k < batch;) { char data[MAX_DATA_SIZE]; memset(data, 0, MAX_DATA_SIZE); int64_t retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, - "sample", strlen("sample"))) { - retLen = getRowDataFromSample( + if (tsRand) { + retLen = generateStbRowData(superTblInfo, data, + startTime + getTSRandTail( + superTblInfo->timeStampStep, k, + superTblInfo->disorderRatio, + superTblInfo->disorderRange) + ); + } else { + retLen = getRowDataFromSample( data, remainderBufLen, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, - "rand", strlen("rand"))) { - int64_t randTail = superTblInfo->timeStampStep * k; - if (superTblInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if(rand_num < superTblInfo->disorderRatio) { - randTail = (randTail + - (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); - debugPrint("rand data generated, back %"PRId64"\n", randTail); - } - } - - int64_t d = startTime + randTail; - retLen = generateStbRowData(superTblInfo, data, d); } if (retLen > remainderBufLen) { @@ -5036,7 +5043,7 @@ static int64_t generateStbDataTail( len += retLen; remainderBufLen -= retLen; - verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n", + verbosePrint("%s() LN%d len=%"PRIu64" k=%ud \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); startFrom ++; @@ -5133,9 +5140,11 @@ static int generateStbSQLHead( return len; } -static int64_t generateStbInterlaceData( +static int32_t generateStbInterlaceData( SSuperTable *superTblInfo, - char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes, + char *tableName, uint32_t batchPerTbl, + uint64_t i, + uint32_t batchPerTblTimes, uint64_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, @@ -5162,7 +5171,7 @@ static int64_t generateStbInterlaceData( int64_t dataLen = 0; - verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n", + verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%ud batchPerTbl = %ud\n", pThreadInfo->threadID, __func__, __LINE__, i, batchPerTblTimes, batchPerTbl); @@ -5170,7 +5179,7 @@ static int64_t generateStbInterlaceData( startTime = taosGetTimestamp(pThreadInfo->time_precision); } - int64_t k = generateStbDataTail( + int32_t k = generateStbDataTail( superTblInfo, batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, startTime, @@ -5180,7 +5189,7 @@ static int64_t generateStbInterlaceData( pstr += dataLen; *pRemainderBufLen -= dataLen; } else { - debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + debugPrint("%s() LN%d, generated data tail: %ud, not equal batch per table: %ud\n", __func__, __LINE__, k, batchPerTbl); pstr -= headLen; pstr[0] = '\0'; @@ -5191,7 +5200,7 @@ static int64_t generateStbInterlaceData( } static int64_t generateInterlaceDataWithoutStb( - char *tableName, uint64_t batchPerTbl, + char *tableName, uint32_t batchPerTbl, uint64_t tableSeq, char *dbName, char *buffer, int64_t insertRows, @@ -5223,7 +5232,7 @@ static int64_t generateInterlaceDataWithoutStb( pstr += dataLen; *pRemainderBufLen -= dataLen; } else { - debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n", + debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %ud\n", __func__, __LINE__, k, batchPerTbl); pstr -= headLen; pstr[0] = '\0'; @@ -5233,7 +5242,71 @@ static int64_t generateInterlaceDataWithoutStb( return k; } -static int64_t generateStbProgressiveData( +static int32_t prepareStbStmt(SSuperTable *stbInfo, + TAOS_STMT *stmt, + char *tableName, uint32_t batch, uint64_t insertRows, + int64_t startTime, char *buffer) +{ + uint32_t k; + int ret; + char *pstr = buffer; + pstr += sprintf(pstr, "INSERT INTO %s values(?", tableName); + + for (int i = 0; i < stbInfo->columnCount; i++) { + pstr += sprintf(pstr, ",?"); + } + pstr += sprintf(pstr, ")"); + + ret = taos_stmt_prepare(stmt, buffer, 0); + if (ret != 0){ + errorPrint("failed to execute taos_stmt_prepare. return 0x%x. reason: %s\n", + ret, taos_errstr(NULL)); + return ret; + } + + void *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); + if (bindArray == NULL) { + errorPrint("Failed to allocate %d bind params\n", batch); + return -1; + } + + bool tsRand; + if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { + tsRand = true; + } else { + tsRand = false; + } + for (k = 0; k < batch; k++) { + /* columnCount + 1 (ts) */ + for (int i = 0; i <= stbInfo->columnCount; i ++) { + TAOS_BIND *bind = (TAOS_BIND *)bindArray + (sizeof(TAOS_BIND) * i); + if (i == 0) { + bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + int64_t ts; + if (tsRand) { + ts = startTime + getTSRandTail( + stbInfo->timeStampStep, k, + stbInfo->disorderRatio, + stbInfo->disorderRange); + } else { + ts = startTime + stbInfo->timeStampStep * k; + } + bind->buffer = &ts; + + } else { + + } + } + // if msg > 3MB, break + } + + taos_stmt_bind_param(stmt, bindArray); + taos_stmt_add_batch(stmt); + + return k; +} + +static int32_t generateStbProgressiveData( SSuperTable *superTblInfo, char *tableName, int64_t tableSeq, @@ -5267,12 +5340,17 @@ static int64_t generateStbProgressiveData( pSamplePos, &dataLen); } +static int64_t prepareStmtWithoutStb(char *tableName) +{ + return -1; +} + static int64_t generateProgressiveDataWithoutStb( char *tableName, - int64_t tableSeq, + /* int64_t tableSeq, */ threadInfo *pThreadInfo, char *buffer, int64_t insertRows, - uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, + uint64_t startFrom, int64_t startTime, /*int64_t *pSamplePos, */ int64_t *pRemainderBufLen) { assert(buffer != NULL); @@ -5313,7 +5391,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__); int64_t insertRows; - uint64_t interlaceRows; + uint32_t interlaceRows; uint64_t maxSqlLen; int64_t nTimeStampStep; uint64_t insert_interval; @@ -5351,8 +5429,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (interlaceRows > g_args.num_of_RPR) interlaceRows = g_args.num_of_RPR; - uint64_t batchPerTbl = interlaceRows; - uint64_t batchPerTblTimes; + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { batchPerTblTimes = @@ -5401,9 +5479,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pstr += len; remainderBufLen -= len; - uint64_t recOfBatch = 0; + uint32_t recOfBatch = 0; - for (uint64_t i = 0; i < batchPerTblTimes; i ++) { + for (uint32_t i = 0; i < batchPerTblTimes; i ++) { char tableName[TSDB_TABLE_NAME_LEN]; getTableName(tableName, pThreadInfo, tableSeq); @@ -5416,11 +5494,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { uint64_t oldRemainderLen = remainderBufLen; - int64_t generated; + int32_t generated; if (superTblInfo) { generated = generateStbInterlaceData( superTblInfo, - tableName, batchPerTbl, i, batchPerTblTimes, + tableName, batchPerTbl, i, + batchPerTblTimes, tableSeq, pThreadInfo, pstr, insertRows, @@ -5435,10 +5514,10 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { &remainderBufLen); } - debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", + debugPrint("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); if (generated < 0) { - errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n", + errorPrint("[%d] %s() LN%d, generated records is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); goto free_of_interlace; } else if (generated == 0) { @@ -5450,7 +5529,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pstr += (oldRemainderLen - remainderBufLen); // startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; - verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n", + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); @@ -5466,7 +5545,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (generatedRecPerTbl >= insertRows) break; - int remainRows = insertRows - generatedRecPerTbl; + int64_t remainRows = insertRows - generatedRecPerTbl; if ((remainRows > 0) && (batchPerTbl > remainRows)) batchPerTbl = remainRows; @@ -5482,7 +5561,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { break; } - verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n", + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch, pThreadInfo->totalInsertRows); verbosePrint("[%d] %s() LN%d, buffer=%s\n", @@ -5491,7 +5570,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", + errorPrint("[%d] %s() LN%d try inserting records of batch is %d\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch); errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); @@ -5513,7 +5592,7 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->totalDelay += delay; if (recOfBatch != affectedRows) { - errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n", + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", pThreadInfo->threadID, __func__, __LINE__, recOfBatch, affectedRows, pThreadInfo->buffer); goto free_of_interlace; @@ -5574,12 +5653,6 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { uint64_t startTs = taosGetTimestampMs(); uint64_t endTs; -/* int insert_interval = - superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; - uint64_t st = 0; - uint64_t et = 0xffffffff; - */ - pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; @@ -5606,20 +5679,33 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pstr += len; remainderBufLen -= len; - int64_t generated; + int32_t generated; if (superTblInfo) { - generated = generateStbProgressiveData( - superTblInfo, - tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows, - i, start_time, - &(pThreadInfo->samplePos), - &remainderBufLen); + if (superTblInfo->iface == STMT_IFACE) { + generated = prepareStbStmt(superTblInfo, + pThreadInfo->stmt, + tableName, g_args.num_of_RPR, + insertRows, start_time, pstr); + } else { + generated = generateStbProgressiveData( + superTblInfo, + tableName, tableSeq, pThreadInfo->db_name, pstr, + insertRows, i, start_time, + &(pThreadInfo->samplePos), + &remainderBufLen); + } } else { - generated = generateProgressiveDataWithoutStb( - tableName, tableSeq, pThreadInfo, pstr, insertRows, - i, start_time, - &(pThreadInfo->samplePos), - &remainderBufLen); + if (g_args.iface == STMT_IFACE) { + generated = prepareStmtWithoutStb(tableName); + } else { + generated = generateProgressiveDataWithoutStb( + tableName, + /* tableSeq, */ + pThreadInfo, pstr, insertRows, + i, start_time, + /* &(pThreadInfo->samplePos), */ + &remainderBufLen); + } } if (generated > 0) i += generated; @@ -5687,7 +5773,7 @@ static void* syncWrite(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - int interlaceRows; + uint32_t interlaceRows; if (superTblInfo) { if ((superTblInfo->interlaceRows == 0) @@ -5818,7 +5904,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, char* precision,SSuperTable* superTblInfo) { //TAOS* taos; - //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + //if (0 == strncasecmp(superTblInfo->iface, "taosc", 5)) { // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); // if (NULL == taos) { // printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); @@ -5963,8 +6049,9 @@ static void startMultiThreadInsertData(int threads, char* db_name, } if ((superTblInfo) - && (superTblInfo->insertMode == REST_IFACE)) { - if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { + && (superTblInfo->iface == REST_IFACE)) { + if (convertHostToServAddr( + g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { exit(-1); } } @@ -5989,8 +6076,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, pThreadInfo->minDelay = UINT64_MAX; if ((NULL == superTblInfo) || - (superTblInfo->insertMode != REST_IFACE)) { - //pThreadInfo->taos = taos; + (superTblInfo->iface != REST_IFACE)) { + //t_info->taos = taos; pThreadInfo->taos = taos_connect( g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); @@ -6003,7 +6090,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, exit(-1); } - if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) { + if ((superTblInfo) && (superTblInfo->iface == STMT_IFACE)) { pThreadInfo->stmt = taos_stmt_init(pThreadInfo->taos); if (NULL == pThreadInfo->stmt) { errorPrint( @@ -7252,7 +7339,7 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].insertMode = g_args.iface; + g_Dbs.db[0].superTbls[0].iface = g_args.iface; tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;