From 0f78a600cbc77054ae4563020150cbdc8afc2c14 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 16 Sep 2022 10:22:11 +0800 Subject: [PATCH 01/20] test:add test case for param ttl --- tests/system-test/1-insert/table_param_ttl.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index 5ef6548789..f5fc2d627b 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -67,9 +67,21 @@ class TDTestCase: tdSql.checkRows(self.tbnum - int(self.tbnum/2)) tdSql.execute('drop database db') + def ttl_check_insert(self): + tdSql.prepare() + tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + for i in range(self.tbnum): + tdSql.execute(f'insert into {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) + tdSql.query(f'show tables') + tdSql.checkRows(0) + tdSql.execute('drop database db') def run(self): self.ttl_check_ntb() self.ttl_check_ctb() + self.ttl_check_insert() def stop(self): tdSql.close() From 30b2bd98b8b88c298bf65f6a2b65adeb57c54671 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Sep 2022 10:39:51 +0800 Subject: [PATCH 02/20] fix: STrans struct dbname2 naming ambiguity --- source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 64 ++++++++++++------------- utils/test/c/sdbDump.c | 4 +- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index cd08ee4ad5..a9723abacc 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -165,8 +165,8 @@ typedef struct { SEpSet lastEpset; tmsg_t lastMsgType; tmsg_t originRpcType; - char dbname1[TSDB_TABLE_FNAME_LEN]; - char dbname2[TSDB_TABLE_FNAME_LEN]; + char dbname[TSDB_TABLE_FNAME_LEN]; + char stbname[TSDB_TABLE_FNAME_LEN]; int32_t startFunc; int32_t stopFunc; int32_t paramLen; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index faf656a251..36d056a941 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -71,7 +71,7 @@ int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); -void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2); +void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); void mndTransSetSerial(STrans *pTrans); void mndTransSetOper(STrans *pTrans, EOperType oper); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index dd7a9e71ea..515d5987b9 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -671,7 +671,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); goto _OVER; } - mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); + mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); // hack way mDebug("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); // create stb for stream diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 030f4542b9..8d42cdcfd6 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -127,8 +127,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT8(pRaw, dataPos, 0, _OVER) SDB_SET_INT16(pRaw, dataPos, pTrans->originRpcType, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_TABLE_FNAME_LEN, _OVER) - SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pTrans->redoActionPos, _OVER) int32_t redoActionNum = taosArrayGetSize(pTrans->redoActions); @@ -289,8 +289,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->oper = oper; SDB_GET_INT16(pRaw, dataPos, &pTrans->originRpcType, _OVER) SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_TABLE_FNAME_LEN, _OVER) - SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pTrans->stbname, TSDB_TABLE_FNAME_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pTrans->redoActionPos, _OVER) SDB_GET_INT32(pRaw, dataPos, &redoActionNum, _OVER) SDB_GET_INT32(pRaw, dataPos, &undoActionNum, _OVER) @@ -706,7 +706,7 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c if (pIter == NULL) break; if (pTrans->oper == oper) { - if (strcasecmp(dbname, pTrans->dbname1) == 0) { + if (strcasecmp(dbname, pTrans->dbname) == 0) { mInfo("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper); if (pTrans->pRpcArray == NULL) { pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); @@ -725,12 +725,12 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c return code; } -void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) { - if (dbname1 != NULL) { - tstrncpy(pTrans->dbname1, dbname1, TSDB_TABLE_FNAME_LEN); +void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) { + if (dbname != NULL) { + tstrncpy(pTrans->dbname, dbname, TSDB_TABLE_FNAME_LEN); } - if (dbname2 != NULL) { - tstrncpy(pTrans->dbname2, dbname2, TSDB_TABLE_FNAME_LEN); + if (stbname != NULL) { + tstrncpy(pTrans->stbname, stbname, TSDB_TABLE_FNAME_LEN); } } @@ -759,9 +759,9 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { return 0; } -static bool mndCheckDbConflict(const char *db, STrans *pTrans) { - if (db[0] == 0) return false; - if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true; +static bool mndCheckDbConflict(const char *conflict, STrans *pTrans) { + if (conflict[0] == 0) return false; + if (strcasecmp(conflict, pTrans->dbname) == 0 || strcasecmp(conflict, pTrans->stbname) == 0) return true; return false; } @@ -780,28 +780,28 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pNew->conflict == TRN_CONFLICT_DB) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { - if (mndCheckDbConflict(pNew->dbname1, pTrans)) conflict = true; - if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; + if (mndCheckDbConflict(pNew->dbname, pTrans)) conflict = true; + if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; } } if (pNew->conflict == TRN_CONFLICT_DB_INSIDE) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_DB) { - if (mndCheckDbConflict(pNew->dbname1, pTrans)) conflict = true; - if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; + if (mndCheckDbConflict(pNew->dbname, pTrans)) conflict = true; + if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; } if (pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { - if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; // for stb + if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; // for stb } } if (conflict) { - mError("trans:%d, db1:%s db2:%s type:%d, can't execute since conflict with trans:%d db1:%s db2:%s type:%d", - pNew->id, pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2, + mError("trans:%d, db:%s stb:%s type:%d, can't execute since conflict with trans:%d db:%s stb:%s type:%d", + pNew->id, pNew->dbname, pNew->stbname, pNew->conflict, pTrans->id, pTrans->dbname, pTrans->stbname, pTrans->conflict); } else { - mDebug("trans:%d, db1:%s db2:%s type:%d, not conflict with trans:%d db1:%s db2:%s type:%d", pNew->id, - pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2, + mDebug("trans:%d, db:%s stb:%s type:%d, not conflict with trans:%d db:%s stb:%s type:%d", pNew->id, + pNew->dbname, pNew->stbname, pNew->conflict, pTrans->id, pTrans->dbname, pTrans->stbname, pTrans->conflict); } sdbRelease(pMnode->pSdb, pTrans); @@ -812,7 +812,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { - if (strlen(pTrans->dbname1) == 0 && strlen(pTrans->dbname2) == 0) { + if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare conflict db not set", pTrans->id); return -1; @@ -913,12 +913,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { mInfo("trans:%d, origin msgtype:%s", pTrans->id, TMSG_INFO(pTrans->originRpcType)); - SDbObj *pDb = mndAcquireDb(pMnode, pTrans->dbname1); + SDbObj *pDb = mndAcquireDb(pMnode, pTrans->dbname); if (pDb != NULL) { for (int32_t j = 0; j < 12; j++) { bool ready = mndIsDbReady(pMnode, pDb); if (!ready) { - mInfo("trans:%d, db:%s not ready yet, wait %d times", pTrans->id, pTrans->dbname1, j); + mInfo("trans:%d, db:%s not ready yet, wait %d times", pTrans->id, pTrans->dbname, j); taosMsleep(1000); } else { break; @@ -929,7 +929,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } else if (pTrans->originRpcType == TDMT_MND_CREATE_STB) { void *pCont = NULL; int32_t contLen = 0; - if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname1, pTrans->dbname2, &pCont, &contLen) != 0) { + if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname, pTrans->stbname, &pCont, &contLen) != 0) { mndTransSetRpcRsp(pTrans, pCont, contLen); } } @@ -1599,15 +1599,15 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)stage, false); - char dbname1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname1, mndGetDbStr(pTrans->dbname1), pShow->pMeta->pSchemas[cols].bytes); + char dbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbname, mndGetDbStr(pTrans->dbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)dbname1, false); + colDataAppend(pColInfo, numOfRows, (const char *)dbname, false); - char dbname2[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(dbname2, mndGetDbStr(pTrans->dbname2), pShow->pMeta->pSchemas[cols].bytes); + char stbname[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(stbname, mndGetDbStr(pTrans->stbname), pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)dbname2, false); + colDataAppend(pColInfo, numOfRows, (const char *)stbname, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false); diff --git a/utils/test/c/sdbDump.c b/utils/test/c/sdbDump.c index b90b781e44..b056fe58a3 100644 --- a/utils/test/c/sdbDump.c +++ b/utils/test/c/sdbDump.c @@ -294,8 +294,8 @@ void dumpTrans(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "conflict", pObj->conflict); tjsonAddIntegerToObject(item, "exec", pObj->exec); tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); - tjsonAddStringToObject(item, "dbname1", pObj->dbname1); - tjsonAddStringToObject(item, "dbname2", pObj->dbname2); + tjsonAddStringToObject(item, "dbname", pObj->dbname); + tjsonAddStringToObject(item, "stbname", pObj->stbname); tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions)); tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions)); tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions)); From 186b3e6086b8ce257d8a51b2372ebac9673e9a35 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 19 Sep 2022 10:41:49 +0800 Subject: [PATCH 03/20] fix: STrans struct dbname2 naming ambiguity --- source/common/src/systable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 496806d877..25abd65b40 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -227,8 +227,8 @@ static const SSysDbTableSchema transSchema[] = { {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "db1", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "db2", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "stable", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "failed_times", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "last_action_info", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, From a6db31006f98a1c8c882f7864000a7909930c452 Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Mon, 19 Sep 2022 11:23:07 +0800 Subject: [PATCH 04/20] Update table_param_ttl.py --- tests/system-test/1-insert/table_param_ttl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index f5fc2d627b..dcf8c76b1e 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -69,13 +69,13 @@ class TDTestCase: def ttl_check_insert(self): tdSql.prepare() - tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table db.{self.stbname} (ts timestamp,c0 int) tags(t0 int)') for i in range(self.tbnum): - tdSql.execute(f'insert into {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') - tdSql.query(f'show tables') + tdSql.execute(f'insert into db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(0) tdSql.execute('drop database db') def run(self): From d5e900ecdb7845b8e87921487f7d80affe1ecf03 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 19 Sep 2022 11:27:26 +0800 Subject: [PATCH 05/20] update test case --- tests/system-test/1-insert/table_param_ttl.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index f5fc2d627b..21c591b07e 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -31,51 +31,51 @@ class TDTestCase: tdSql.prepare() for i in range(self.tbnum): - tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}') - tdSql.query(f'show tables') + tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(0) for i in range(self.tbnum): - tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}') + tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}') for i in range(int(self.tbnum/2)): - tdSql.execute(f'alter table {self.ntbname}_{i} ttl {self.modify_ttl}') + tdSql.execute(f'alter table db.{self.ntbname}_{i} ttl {self.modify_ttl}') sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum - int(self.tbnum/2)) tdSql.execute('drop database db') def ttl_check_ctb(self): tdSql.prepare() - tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table db.{self.stbname} (ts timestamp,c0 int) tags(t0 int)') for i in range(self.tbnum): - tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}') - tdSql.query(f'show tables') + tdSql.execute(f'create table db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param}') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(0) for i in range(self.tbnum): - tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.default_ttl}') - tdSql.query(f'show tables') + tdSql.execute(f'create table db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.default_ttl}') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) for i in range(int(self.tbnum/2)): - tdSql.execute(f'alter table {self.stbname}_{i} ttl {self.modify_ttl}') + tdSql.execute(f'alter table db.{self.stbname}_{i} ttl {self.modify_ttl}') sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum - int(self.tbnum/2)) tdSql.execute('drop database db') def ttl_check_insert(self): tdSql.prepare() - tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table db.{self.stbname} (ts timestamp,c0 int) tags(t0 int)') for i in range(self.tbnum): - tdSql.execute(f'insert into {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') - tdSql.query(f'show tables') + tdSql.execute(f'insert into db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)') + tdSql.query(f'show db.tables') tdSql.checkRows(self.tbnum) sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) - tdSql.query(f'show tables') + tdSql.query(f'show db.tables') tdSql.checkRows(0) tdSql.execute('drop database db') def run(self): From d0bfa65cc4fecc5be9d12f1db255839cbebd9952 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 19 Sep 2022 11:54:30 +0800 Subject: [PATCH 06/20] docs: add insert select description --- docs/en/12-taos-sql/05-insert.md | 6 +++++- docs/zh/12-taos-sql/05-insert.md | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index e7d56fb3c7..97bfc38c7e 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -16,6 +16,8 @@ INSERT INTO [(field1_name, ...)] VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path ...]; + +INSERT INTO tb_name [(field1_name, ...)] query ``` **Timestamps** @@ -37,7 +39,7 @@ INSERT INTO 4. The FILE clause inserts tags or data from a comma-separates values (CSV) file. Do not include headers in your CSV files. -5. A single INSERT statement can write data to multiple tables. +5. A single INSERT ... VALUES statement or INSERT ... FILE statement can write data to multiple tables. 6. The INSERT statement is fully parsed before being executed, so that if any element of the statement fails, the entire statement will fail. For example, the following statement will not create a table because the latter part of the statement is invalid: @@ -47,6 +49,8 @@ INSERT INTO 7. However, an INSERT statement that writes data to multiple subtables can succeed for some tables and fail for others. This situation is caused because vnodes perform write operations independently of each other. One vnode failing to write data does not affect the ability of other vnodes to write successfully. +8. Data from TDengine can be inserted into a specified table using the INSERT ... query statement. Arbitrary query statements are supported. This syntax can only be used for subtables and normal tables, and does not support automatic table creation. + ## Insert a Record Single row or multiple rows specified with VALUES can be inserted into a specific table. A single row is inserted using the below statement. diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index 59af9c55ed..c865bc8022 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -17,6 +17,8 @@ INSERT INTO [(field1_name, ...)] VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path ...]; + +INSERT INTO tb_name [(field1_name, ...)] query ``` **关于时间戳** @@ -38,7 +40,7 @@ INSERT INTO 4. FILE 语法表示数据来自于 CSV 文件(英文逗号分隔、英文单引号括住每个值),CSV 文件无需表头。 -5. 无论使用哪种语法,均可以在一条 INSERT 语句中同时向多个表插入数据。 +5. INSERT ... VALUES 语句和 INSERT ... FILE 语句均可以在一条 INSERT 语句中同时向多个表插入数据。 6. INSERT 语句是完整解析后再执行的,对如下语句,不会再出现数据错误但建表成功的情况: @@ -48,6 +50,8 @@ INSERT INTO 7. 对于向多个子表插入数据的情况,依然会有部分数据写入失败,部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上,客户端将 INSERT 语句完整解析后,将数据发往各个涉及的 VNODE 上,每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因(比如网络问题或磁盘故障)导致写入失败,并不会影响其他 VNODE 节点的写入。 +8. 可以使用 INSERT ... query 语句将 TDengine 中的数据插入到指定表中。query 可以是任意的查询语句。此语法只能用于子表和普通表,且不支持自动建表。 + ## 插入一条记录 指定已经创建好的数据子表的表名,并通过 VALUES 关键字提供一行或多行数据,即可向数据库写入这些数据。例如,执行如下语句可以写入一行记录: From 838ab9bddb5ba83d39275f0bb01dd630adcf5d2e Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 19 Sep 2022 12:45:36 +0800 Subject: [PATCH 07/20] add test case into ci --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b45b24258..604add0eb3 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -32,7 +32,7 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 1-insert/mutil_stage.py - +python3 ./test.py -f 1-insert/table_param_ttl.py -R python3 ./test.py -f 1-insert/update_data_muti_rows.py python3 ./test.py -f 1-insert/db_tb_name_check.py From 23a243230a0d24fdf411079f9a9cc375b9cbfbc1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 19 Sep 2022 13:11:05 +0800 Subject: [PATCH 08/20] docs: add insert select description --- docs/en/12-taos-sql/05-insert.md | 6 +++--- docs/zh/12-taos-sql/05-insert.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 97bfc38c7e..da21896866 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -17,7 +17,7 @@ INSERT INTO VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path ...]; -INSERT INTO tb_name [(field1_name, ...)] query +INSERT INTO tb_name [(field1_name, ...)] subquery ``` **Timestamps** @@ -39,7 +39,7 @@ INSERT INTO tb_name [(field1_name, ...)] query 4. The FILE clause inserts tags or data from a comma-separates values (CSV) file. Do not include headers in your CSV files. -5. A single INSERT ... VALUES statement or INSERT ... FILE statement can write data to multiple tables. +5. A single `INSERT ... VALUES` statement and `INSERT ... FILE` statement can write data to multiple tables. 6. The INSERT statement is fully parsed before being executed, so that if any element of the statement fails, the entire statement will fail. For example, the following statement will not create a table because the latter part of the statement is invalid: @@ -49,7 +49,7 @@ INSERT INTO tb_name [(field1_name, ...)] query 7. However, an INSERT statement that writes data to multiple subtables can succeed for some tables and fail for others. This situation is caused because vnodes perform write operations independently of each other. One vnode failing to write data does not affect the ability of other vnodes to write successfully. -8. Data from TDengine can be inserted into a specified table using the INSERT ... query statement. Arbitrary query statements are supported. This syntax can only be used for subtables and normal tables, and does not support automatic table creation. +8. Data from TDengine can be inserted into a specified table using the `INSERT ... subquery` statement. Arbitrary query statements are supported. This syntax can only be used for subtables and normal tables, and does not support automatic table creation. ## Insert a Record diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index c865bc8022..5e64827a8f 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -18,7 +18,7 @@ INSERT INTO VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path ...]; -INSERT INTO tb_name [(field1_name, ...)] query +INSERT INTO tb_name [(field1_name, ...)] subquery ``` **关于时间戳** @@ -40,7 +40,7 @@ INSERT INTO tb_name [(field1_name, ...)] query 4. FILE 语法表示数据来自于 CSV 文件(英文逗号分隔、英文单引号括住每个值),CSV 文件无需表头。 -5. INSERT ... VALUES 语句和 INSERT ... FILE 语句均可以在一条 INSERT 语句中同时向多个表插入数据。 +5. `INSERT ... VALUES` 语句和 `INSERT ... FILE` 语句均可以在一条 INSERT 语句中同时向多个表插入数据。 6. INSERT 语句是完整解析后再执行的,对如下语句,不会再出现数据错误但建表成功的情况: @@ -50,7 +50,7 @@ INSERT INTO tb_name [(field1_name, ...)] query 7. 对于向多个子表插入数据的情况,依然会有部分数据写入失败,部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上,客户端将 INSERT 语句完整解析后,将数据发往各个涉及的 VNODE 上,每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因(比如网络问题或磁盘故障)导致写入失败,并不会影响其他 VNODE 节点的写入。 -8. 可以使用 INSERT ... query 语句将 TDengine 中的数据插入到指定表中。query 可以是任意的查询语句。此语法只能用于子表和普通表,且不支持自动建表。 +8. 可以使用 `INSERT ... subquery` 语句将 TDengine 中的数据插入到指定表中。subquery 可以是任意的查询语句。此语法只能用于子表和普通表,且不支持自动建表。 ## 插入一条记录 From 8c48dc4d581ba0a03f58574d5a1faef937b9a89d Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Mon, 19 Sep 2022 13:55:35 +0800 Subject: [PATCH 09/20] doc: update quick start docs for new edition --- docs/en/05-get-started/01-docker.md | 6 +++++- docs/en/05-get-started/03-package.md | 6 +++++- docs/en/05-get-started/index.md | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index 6191492b37..be61dfeb8e 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -3,7 +3,11 @@ sidebar_label: Docker title: Quick Install on Docker --- -This document describes how to install TDengine in a Docker container and perform queries and inserts. To get started with TDengine in a non-containerized environment, see [Quick Install](../../get-started/package). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). +This document describes how to install TDengine in a Docker container and perform queries and inserts. + +- To get started with TDengine in a non-containerized environment, see [Quick Install from Package](../../get-started/package). +- For a fully managed solution, see the [TDengine Cloud documentation](/cloud/). +- If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). ## Run TDengine diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 292fa82183..da06aca148 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -7,7 +7,11 @@ import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; import PkgListV3 from "/components/PkgListV3"; -For information about installing TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). +This document describes how to install TDengine on Linux and Windows and perform queries and inserts. + +- To get started with TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). +- For a fully managed solution, see the [TDengine Cloud documentation](/cloud/). +- If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter). diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index cf475a8cd7..09875362f4 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -3,9 +3,9 @@ title: Get Started description: This article describes how to install TDengine and test its performance. --- -The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. In addition to connectors for multiple languages, TDengine also provides a [RESTful interface](/reference/rest-api) through [taosAdapter](/reference/taosadapter). +You can install and run TDengine on Linux and Windows machines as well as Docker containers. You can also deploy TDengine as a managed service with TDengine Cloud. -You can install and run TDengine on Linux and Windows machines as well as Docker containers. +The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. In addition to connectors for multiple languages, TDengine also provides a [RESTful interface](/reference/rest-api) through [taosAdapter](/reference/taosadapter). ```mdx-code-block import DocCardList from '@theme/DocCardList'; From e5e21bf6357cdba1e69d75bb0dafb8038b3d0f7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Sep 2022 14:08:30 +0800 Subject: [PATCH 10/20] fix(query): add null ptr check. --- source/libs/executor/src/executil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d75f0580e1..b88589740d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -989,7 +989,8 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod if (pNode->output) { (*numOfOutputCols) += 1; - } else { + } else if (info != NULL) { + // select distinct tbname from stb where tbname='abc'; info->output = false; } } From 70bc3e6984ea378e837ef7c5f5cbaa5574ded7c4 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 19 Sep 2022 14:21:41 +0800 Subject: [PATCH 11/20] enh: plan serialization optimize --- source/libs/nodes/src/nodesMsgFuncs.c | 67 +++++++++++++++------------ 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index ade5be4722..ffe4b0ab14 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2462,33 +2462,54 @@ static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { - PHY_INTERVAL_CODE_WINDOW = 1, - PHY_INTERVAL_CODE_INTERVAL, - PHY_INTERVAL_CODE_OFFSET, - PHY_INTERVAL_CODE_SLIDING, - PHY_INTERVAL_CODE_INTERVAL_UNIT, - PHY_INTERVAL_CODE_SLIDING_UNIT -}; +enum { PHY_INTERVAL_CODE_WINDOW = 1, PHY_INTERVAL_CODE_INLINE_ATTRS }; + +static int32_t physiIntervalNodeInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; + + int32_t code = tlvEncodeValueI64(pEncoder, pNode->interval); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI64(pEncoder, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI64(pEncoder, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI8(pEncoder, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueI8(pEncoder, pNode->slidingUnit); + } + + return code; +} static int32_t physiIntervalNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_INTERVAL, pNode->interval); + code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_INLINE_ATTRS, physiIntervalNodeInlineToMsg, pNode); + } + + return code; +} + +static int32_t msgToPhysiIntervalNodeInline(STlvDecoder* pDecoder, void* pObj) { + SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; + + int32_t code = tlvDecodeValueI64(pDecoder, &pNode->interval); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueI64(pDecoder, &pNode->offset); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_OFFSET, pNode->offset); + code = tlvDecodeValueI64(pDecoder, &pNode->sliding); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_SLIDING, pNode->sliding); + code = tlvDecodeValueI8(pDecoder, &pNode->intervalUnit); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_INTERVAL_UNIT, pNode->intervalUnit); - } - if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_SLIDING_UNIT, pNode->slidingUnit); + code = tlvDecodeValueI8(pDecoder, &pNode->slidingUnit); } return code; @@ -2504,20 +2525,8 @@ static int32_t msgToPhysiIntervalNode(STlvDecoder* pDecoder, void* pObj) { case PHY_INTERVAL_CODE_WINDOW: code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); break; - case PHY_INTERVAL_CODE_INTERVAL: - code = tlvDecodeI64(pTlv, &pNode->interval); - break; - case PHY_INTERVAL_CODE_OFFSET: - code = tlvDecodeI64(pTlv, &pNode->offset); - break; - case PHY_INTERVAL_CODE_SLIDING: - code = tlvDecodeI64(pTlv, &pNode->sliding); - break; - case PHY_INTERVAL_CODE_INTERVAL_UNIT: - code = tlvDecodeI8(pTlv, &pNode->intervalUnit); - break; - case PHY_INTERVAL_CODE_SLIDING_UNIT: - code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + case PHY_INTERVAL_CODE_INLINE_ATTRS: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiIntervalNodeInline, pNode); break; default: break; From 01adee2f53306f12913936798aa8938173af2084 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Sep 2022 14:36:55 +0800 Subject: [PATCH 12/20] fix(query): add an assert --- source/common/src/tdatablock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 16b8e55cf7..8a8e6a83a5 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1446,6 +1446,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; int32_t nRows = payloadSize / rowSize; + ASSERT(nRows >= 1); // the true value must be less than the value of nRows int32_t additional = 0; From d48a56326019f9444a84efc1dad280944dd7e774 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Sep 2022 15:42:03 +0800 Subject: [PATCH 13/20] fix(query): assign index back. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 45fe29f0fa..5f03a82bc0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -320,6 +320,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) { pIter->pSttBlk = NULL; if (index != -1) { + pIter->iSttBlk = index; pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); } } From 74f75893eafe238233d04c4a8b3a154f44395832 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 19 Sep 2022 15:56:14 +0800 Subject: [PATCH 14/20] enh: physical plan serialization supports cross-platform --- source/libs/nodes/src/nodesMsgFuncs.c | 130 ++++++++++++++++++---- source/libs/planner/test/planTestUtil.cpp | 3 +- 2 files changed, 111 insertions(+), 22 deletions(-) diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index ffe4b0ab14..fce06d6b5a 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -17,6 +17,14 @@ #include "plannodes.h" #include "tdatablock.h" +#define htonll(x) \ + (((int64_t)x & 0x00000000000000ff) << 7 * 8) | (((int64_t)x & 0x000000000000ff00) << 5 * 8) | \ + (((int64_t)x & 0x0000000000ff0000) << 3 * 8) | (((int64_t)x & 0x00000000ff000000) << 1 * 8) | \ + (((int64_t)x & 0x000000ff00000000) >> 1 * 8) | (((int64_t)x & 0x0000ff0000000000) >> 3 * 8) | \ + (((int64_t)x & 0x00ff000000000000) >> 5 * 8) | (((int64_t)x & 0xff00000000000000) >> 7 * 8) + +#define ntohll(x) htonll(x) + #define NODES_MSG_DEFAULT_LEN 1024 #define TLV_TYPE_ARRAY_ELEM 0 @@ -86,8 +94,8 @@ static int32_t tlvEncodeImpl(STlvEncoder* pEncoder, int16_t type, const void* pV pEncoder->allocSize = pEncoder->allocSize * 2; } STlv* pTlv = (STlv*)(pEncoder->pBuf + pEncoder->offset); - pTlv->type = type; - pTlv->len = len; + pTlv->type = htons(type); + pTlv->len = htonl(len); memcpy(pTlv->value, pValue, len); pEncoder->offset += tlvLen; ++(pEncoder->tlvCount); @@ -117,26 +125,32 @@ static int32_t tlvEncodeValueI8(STlvEncoder* pEncoder, int8_t value) { } static int32_t tlvEncodeI16(STlvEncoder* pEncoder, int16_t type, int16_t value) { + value = htons(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI16(STlvEncoder* pEncoder, int16_t value) { + value = htons(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeI32(STlvEncoder* pEncoder, int16_t type, int32_t value) { + value = htonl(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI32(STlvEncoder* pEncoder, int32_t value) { + value = htonl(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeI64(STlvEncoder* pEncoder, int16_t type, int64_t value) { + value = htonll(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueI64(STlvEncoder* pEncoder, int64_t value) { + value = htonll(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } @@ -149,34 +163,44 @@ static int32_t tlvEncodeValueU8(STlvEncoder* pEncoder, uint8_t value) { } static int32_t tlvEncodeU16(STlvEncoder* pEncoder, int16_t type, uint16_t value) { + value = htons(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueU16(STlvEncoder* pEncoder, uint16_t value) { + value = htons(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeU64(STlvEncoder* pEncoder, int16_t type, uint64_t value) { + value = htonll(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueU64(STlvEncoder* pEncoder, uint64_t value) { + value = htonll(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } static int32_t tlvEncodeDouble(STlvEncoder* pEncoder, int16_t type, double value) { - return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); + int64_t temp = *(int64_t*)&value; + temp = htonll(temp); + return tlvEncodeImpl(pEncoder, type, &temp, sizeof(temp)); } static int32_t tlvEncodeValueDouble(STlvEncoder* pEncoder, double value) { - return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); + int64_t temp = *(int64_t*)&value; + temp = htonll(temp); + return tlvEncodeValueImpl(pEncoder, &temp, sizeof(temp)); } static int32_t tlvEncodeEnum(STlvEncoder* pEncoder, int16_t type, int32_t value) { + value = htonl(value); return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); } static int32_t tlvEncodeValueEnum(STlvEncoder* pEncoder, int32_t value) { + value = htonl(value); return tlvEncodeValueImpl(pEncoder, &value, sizeof(value)); } @@ -197,7 +221,7 @@ static int32_t tlvEncodeCStr(STlvEncoder* pEncoder, int16_t type, const char* pV static int32_t tlvEncodeValueCStr(STlvEncoder* pEncoder, const char* pValue) { int16_t len = strlen(pValue); - int32_t code = tlvEncodeValueImpl(pEncoder, &len, sizeof(len)); + int32_t code = tlvEncodeValueI16(pEncoder, len); if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueImpl(pEncoder, pValue, len); } @@ -218,8 +242,8 @@ static int32_t tlvEncodeObj(STlvEncoder* pEncoder, int16_t type, FToMsg func, co int32_t code = func(pObj, pEncoder); if (TSDB_CODE_SUCCESS == code) { STlv* pTlv = (STlv*)(pEncoder->pBuf + start); - pTlv->type = type; - pTlv->len = pEncoder->offset - start - sizeof(STlv); + pTlv->type = htons(type); + pTlv->len = htonl(pEncoder->offset - start - sizeof(STlv)); } ++(pEncoder->tlvCount); return code; @@ -236,8 +260,8 @@ static int32_t tlvEncodeObjArray(STlvEncoder* pEncoder, int16_t type, FToMsg fun } if (TSDB_CODE_SUCCESS == code) { STlv* pTlv = (STlv*)(pEncoder->pBuf + start); - pTlv->type = type; - pTlv->len = pEncoder->offset - start - sizeof(STlv); + pTlv->type = htons(type); + pTlv->len = htonl(pEncoder->offset - start - sizeof(STlv)); } } return code; @@ -259,6 +283,8 @@ static int32_t tlvGetNextTlv(STlvDecoder* pDecoder, STlv** pTlv) { } *pTlv = (STlv*)(pDecoder->pBuf + pDecoder->offset); + (*pTlv)->type = ntohs((*pTlv)->type); + (*pTlv)->len = ntohl((*pTlv)->len); if ((*pTlv)->len + pDecoder->offset > pDecoder->bufSize) { return TSDB_CODE_FAILED; } @@ -291,22 +317,52 @@ static int32_t tlvDecodeValueI8(STlvDecoder* pDecoder, int8_t* pValue) { return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); } -static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; +} static int32_t tlvDecodeValueI16(STlvDecoder* pDecoder, int16_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; } -static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohl(*pValue); + } + return code; +} static int32_t tlvDecodeValueI32(STlvDecoder* pDecoder, int32_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohl(*pValue); + } + return code; } -static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; +} static int32_t tlvDecodeValueI64(STlvDecoder* pDecoder, int64_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; } static int32_t tlvDecodeU8(STlv* pTlv, uint8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } @@ -315,22 +371,54 @@ static int32_t tlvDecodeValueU8(STlvDecoder* pDecoder, uint8_t* pValue) { return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); } -static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; +} static int32_t tlvDecodeValueU16(STlvDecoder* pDecoder, uint16_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohs(*pValue); + } + return code; } -static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { + int32_t code = tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; +} static int32_t tlvDecodeValueU64(STlvDecoder* pDecoder, uint64_t* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int32_t code = tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + if (TSDB_CODE_SUCCESS == code) { + *pValue = ntohll(*pValue); + } + return code; } -static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } +static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { + int64_t temp = 0; + int32_t code = tlvDecodeI64(pTlv, &temp); + if (TSDB_CODE_SUCCESS == code) { + *pValue = *(double*)&temp; + } + return code; +} static int32_t tlvDecodeValueDouble(STlvDecoder* pDecoder, double* pValue) { - return tlvDecodeValueImpl(pDecoder, pValue, sizeof(*pValue)); + int64_t temp = 0; + int32_t code = tlvDecodeValueI64(pDecoder, &temp); + if (TSDB_CODE_SUCCESS == code) { + *pValue = *(double*)&temp; + } + return code; } static int32_t convertIntegerType(int32_t value, void* pValue, int16_t len) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index bf19c7a222..2b8e3d9864 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -473,10 +473,11 @@ class PlannerTestBaseImpl { cout << "nodesNodeToMsg: " << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + string copyStr(pStr, len); SNode* pNode = NULL; char* pNewStr = NULL; int32_t newlen = 0; - DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode) + DO_WITH_THROW(nodesMsgToNode, copyStr.c_str(), len, &pNode) DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen) if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) { cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl; From d1d55f4e7865c974bd161611cb898f32984b779d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 19 Sep 2022 16:50:00 +0800 Subject: [PATCH 15/20] enh: physical plan serialization supports cross-platform --- source/libs/nodes/src/nodesMsgFuncs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index fce06d6b5a..2c47ddea8b 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -17,6 +17,8 @@ #include "plannodes.h" #include "tdatablock.h" +#ifndef htonll + #define htonll(x) \ (((int64_t)x & 0x00000000000000ff) << 7 * 8) | (((int64_t)x & 0x000000000000ff00) << 5 * 8) | \ (((int64_t)x & 0x0000000000ff0000) << 3 * 8) | (((int64_t)x & 0x00000000ff000000) << 1 * 8) | \ @@ -25,6 +27,8 @@ #define ntohll(x) htonll(x) +#endif + #define NODES_MSG_DEFAULT_LEN 1024 #define TLV_TYPE_ARRAY_ELEM 0 From 5dddf1812b355e49b4911e6ba91a6ea63ba093f6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 19 Sep 2022 16:53:18 +0800 Subject: [PATCH 16/20] fix: copy rpgno with non-leaf pages --- source/libs/tdb/src/db/tdbBtree.c | 4 ++++ source/libs/tdb/src/db/tdbPager.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6ecd37680..c5204ef59e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -841,6 +841,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx // copy content to the parent page tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); tdbPageCopy(pNews[0], pParent, 1); + + if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { + ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; + } } for (int i = 0; i < 3; i++) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 2cc62d3d6a..543ffc55b6 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -260,7 +260,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; - // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -353,7 +353,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; - // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } From 2a494b267157f4a4b12d362390fe157d3131af72 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 19 Sep 2022 16:56:35 +0800 Subject: [PATCH 17/20] fix(tsc): fix taosdump failure --- source/client/src/clientHb.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7ce80553a0..75ccd44977 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -878,12 +878,18 @@ int hbMgrInit() { clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); TdThreadMutexAttr attr = {0}; - taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + int ret = taosThreadMutexAttrInit(&attr); assert(ret == 0); - taosThreadMutexInit(&clientHbMgr.lock, &attr); - taosThreadMutexAttrDestroy(&attr); + ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + assert(ret == 0); + + ret = taosThreadMutexInit(&clientHbMgr.lock, &attr); + assert(ret == 0); + + ret = taosThreadMutexAttrDestroy(&attr); + assert(ret == 0); // init handle funcs hbMgrInitHandle(); From b08288df489f2e4d9d6dbd9ca09e688dc98ba3f2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 19 Sep 2022 17:07:22 +0800 Subject: [PATCH 18/20] fix(query): fix invalid read in doModeAdd TD-19124 --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9b502eded7..a23f58a732 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5297,12 +5297,12 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } static void doModeAdd(SModeInfo* pInfo, char* data) { - int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { int32_t size = sizeof(SModeItem) + pInfo->colBytes; SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); - memcpy(pItem->data, data, pInfo->colBytes); + memcpy(pItem->data, data, hashKeyBytes); pItem->count += 1; taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); From 1dcc46a5d4721d256004868047ea7f3335ae6d93 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Sep 2022 17:19:19 +0800 Subject: [PATCH 19/20] feat: update taostools 318246c for3.0 (#16915) * feat: update taos-tools 9f4c01e for 3.0 * feat: update taos-tools e7270c9 for 3.0 * feat: update taos-tools 7d7b3ce for 3.0 * feat: update taos-tools 125c77a for 3.0 * feat: update taos-tools 318246c for 3.0 * feat: update taos-tools 509ec72 for 3.0 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 6a3439ada9..5cc580a9c6 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 125c77a + GIT_TAG 509ec72 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From a5c92370d50e0e367bcf7c25c2721c1c79ceb3a8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 19 Sep 2022 19:01:15 +0800 Subject: [PATCH 20/20] fix: func declarition for arm32 (#16930) --- source/client/src/TMQConnector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index fcf6957df9..26bf55055f 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -212,7 +212,7 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN tmq_commit_async(tmq, res, commit_cb, consumer); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { jniError("jobj:%p, tmq is closed", jobj); @@ -222,7 +222,7 @@ JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqUnsubscribeImp( return tmq_unsubscribe((tmq_t *)tmq); } -JNIEXPORT int JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConsumerCloseImp(JNIEnv *env, jobject jobj, jlong jtmq) { tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) {