diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 376dea895c..c5ba653ad2 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -751,7 +751,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { } static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) { + SHashObj *schemaHashCheck, ESchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { @@ -759,15 +759,11 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); SML_CHECK_NULL(kv); SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info)); - } - - for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j); - SML_CHECK_NULL(kv); - if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) { + if (taosHashGet(schemaHashCheck, kv->key, kv->keyLen) != NULL) { SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN); } } + END: RETURN } @@ -998,14 +994,14 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa END: RETURN } -static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, +static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ ESchemaAction action = SCHEMA_ACTION_NULL; SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; int32_t lino = 0; - SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true)); + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, hashTmpCheck, &action, true)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); @@ -1029,14 +1025,14 @@ END: RETURN } -static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn, +static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn, SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){ ESchemaAction action = SCHEMA_ACTION_NULL; SArray *pColumns = NULL; SArray *pTags = NULL; int32_t code = 0; int32_t lino = 0; - SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false)); + SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, hashTmpCheck, &action, false)); if (action != SCHEMA_ACTION_NULL) { SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE)); @@ -1079,7 +1075,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } int32_t code = 0; int32_t lino = 0; - SHashObj *hashTmp = NULL; + SHashObj *colHashTmp = NULL; + SHashObj *tagHashTmp = NULL; STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; @@ -1119,17 +1116,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK); } - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - SML_CHECK_NULL(hashTmp); - SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); - SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); - taosHashClear(hashTmp); - SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); - SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta)); + colHashTmp = taosHashInit(pTableMeta->tableInfo.numOfColumns, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + tagHashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SML_CHECK_NULL(colHashTmp); + SML_CHECK_NULL(tagHashTmp); + SML_CHECK_CODE(smlBuildTempHash(tagHashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags)); + SML_CHECK_CODE(smlBuildTempHash(colHashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns)); + + SML_CHECK_CODE(smlModifyTag(info, colHashTmp, tagHashTmp, &conn, sTableData, &pName, &pTableMeta)); + SML_CHECK_CODE(smlModifyCols(info, tagHashTmp, colHashTmp, &conn, sTableData, &pName, &pTableMeta)); needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; + taosHashCleanup(colHashTmp); + taosHashCleanup(tagHashTmp); + colHashTmp = NULL; + tagHashTmp = NULL; } else { uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code)); goto END; @@ -1153,7 +1154,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { END: taosHashCancelIterate(info->superTables, tmp); - taosHashCleanup(hashTmp); + taosHashCleanup(colHashTmp); + taosHashCleanup(tagHashTmp); taosMemoryFreeClear(pTableMeta); (void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code, @@ -1924,3 +1926,4 @@ TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t * return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } + diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8ebbe62857..4a90e33bfa 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -152,13 +152,13 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start for (int32_t i = 0; i < numOfRows; i += 20) { char sql[1024] = {0}; (void)sprintf(sql, - "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, - i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, - i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, + i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, + i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { (void)printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -170,14 +170,14 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start for (int32_t i = 0; i < numOfRows; i += 20) { char sql[1024*50] = {0}; (void)sprintf(sql, - "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " - "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " - "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", - index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, - pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, - pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, - i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, - pVarchar, startTs + 19, i + 19, pVarchar); + "insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, " + "%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, " + "'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')", + index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4, + pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9, + pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14, + i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18, + pVarchar, startTs + 19, i + 19, pVarchar); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { (void)printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -309,8 +309,8 @@ int main(int argc, char** argv) { if (code != 0) { return code; } - - + + } numOfThreads = TMAX(numOfThreads, 1); @@ -1441,7 +1441,7 @@ TEST(clientCase, sub_tb_test) { int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign); if (code != 0) { (void)printf("error occurs:%s\n", tmq_err2str(code)); - (void)tmq_consumer_close(tmq); + (void)tmq_consumer_close(tmq); taos_close(pConn); (void)fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); return; @@ -1523,29 +1523,29 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "drop database if exists db1"); + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_timezone"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "create database db1"); + pRes = taos_query(pConn, "create database db_timezone"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)"); while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) { taosMsleep(2000); - pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)"); + pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)"); } ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); char sql[256] = {0}; - (void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:00', 1)"); + (void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:00', 1)"); pRes = taos_query(pConn, sql); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 17:00:00'"); + pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 17:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1571,7 +1571,7 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 01:00:00'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 01:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1588,7 +1588,7 @@ TEST(clientCase, timezone_Test) { taos_free_result(pRes); char sql[256] = {0}; - (void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:01', 1)"); + (void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:01', 1)"); pRes = taos_query(pConn, sql); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); @@ -1604,7 +1604,7 @@ TEST(clientCase, timezone_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 09:00:00'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 09:00:00'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; @@ -1620,7 +1620,7 @@ TEST(clientCase, timezone_Test) { taos_free_result(pRes); { - TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-17 01:00:01'"); + TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-17 01:00:01'"); ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS); TAOS_ROW pRow = NULL; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d46968a22d..05b9826ae8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1259,6 +1259,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SDbObj *pDb = NULL; SMCreateStbReq createReq = {0}; bool isAlter = false; + SHashObj *pHash = NULL; if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { code = TSDB_CODE_INVALID_MSG; @@ -1319,6 +1320,33 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } + pHash = taosHashInit(createReq.numOfColumns + createReq.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_NO_LOCK); + if (pHash == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + for (int32_t i = 0; i < createReq.numOfColumns; ++i) { + SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i); + if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) { + if (code == TSDB_CODE_DUP_KEY) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + } + goto _OVER; + } + } + + for (int32_t i = 0; i < createReq.numOfTags; ++i) { + SField *pField = taosArrayGet(createReq.pTags, i); + if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) { + if (code == TSDB_CODE_DUP_KEY) { + code = TSDB_CODE_TSC_DUP_COL_NAMES; + } + goto _OVER; + } + } + pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { code = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -1383,6 +1411,10 @@ _OVER: mndReleaseDb(pMnode, pDb); tFreeSMCreateStbReq(&createReq); + if (pHash != NULL) { + taosHashCleanup(pHash); + } + TAOS_RETURN(code); } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index e92231907f..438b4fd2e4 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -25,6 +25,7 @@ class MndTestStb : public ::testing::Test { void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); void* BuildDropDbReq(const char* dbname, int32_t* pContLen); void* BuildCreateStbReq(const char* stbname, int32_t* pContLen); + void* BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen); void* BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen); void* BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen); void* BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, @@ -137,6 +138,71 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { return pHead; } +void* MndTestStb::BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen) { + SMCreateStbReq createReq = {0}; + createReq.numOfColumns = 2; + createReq.numOfTags = 4; + createReq.igExists = 0; + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + strcpy(createReq.name, stbname); + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(field.name, "ts"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 12; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "col1"); + taosArrayPush(createReq.pColumns, &field); + } + + { + SField field = {0}; + field.bytes = 2; + field.type = TSDB_DATA_TYPE_TINYINT; + strcpy(field.name, "tag1"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 8; + field.type = TSDB_DATA_TYPE_BIGINT; + strcpy(field.name, "tag2"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 16; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "tag3"); + taosArrayPush(createReq.pTags, &field); + } + + { + SField field = {0}; + field.bytes = 16; + field.type = TSDB_DATA_TYPE_BINARY; + strcpy(field.name, "tag3"); + taosArrayPush(createReq.pTags, &field); + } + + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); + void* pHead = rpcMallocCont(tlen); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); + *pContLen = tlen; + return pHead; +} + void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { SMAlterStbReq req = {0}; strcpy(req.name, stbname); @@ -896,3 +962,26 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { rpcFreeCont(pRsp->pCont); } } + +TEST_F(MndTestStb, 09_Create_Duplicate_Stb) { + const char* dbname = "1.d2"; + const char* stbname = "1.d2.stb"; + + { + int32_t contLen = 0; + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + rpcFreeCont(pRsp->pCont); + } + + { + int32_t contLen = 0; + void* pReq = BuildCreateStbDuplicateReq(stbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_TSC_DUP_COL_NAMES); + rpcFreeCont(pRsp->pCont); + } +} \ No newline at end of file diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 42d2e95d24..e7c0617d88 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -50,10 +50,14 @@ %left AND. %left UNION ALL MINUS EXCEPT INTERSECT. %left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT. +%left NK_LT NK_GT NK_LE NK_GE NK_EQ NK_NE LIKE MATCH NMATCH REGEXP CONTAINS BETWEEN IS IN. %left NK_PLUS NK_MINUS. %left NK_STAR NK_SLASH NK_REM. %left NK_CONCAT. +%right NOT. +%left NK_ARROW. + %right INNER LEFT RIGHT FULL OUTER SEMI ANTI ASOF WINDOW JOIN ON WINDOW_OFFSET JLIMIT. /************************************************ create/alter account *****************************************/ @@ -1285,9 +1289,9 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= cols_func(B) NK_LP cols_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= - CAST(B) NK_LP expr_or_subquery(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } + CAST(B) NK_LP common_expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } function_expression(A) ::= - CAST(B) NK_LP expr_or_subquery(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } + CAST(B) NK_LP common_expression(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } function_expression(A) ::= POSITION(B) NK_LP expr_or_subquery(C) IN expr_or_subquery(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createPositionFunctionNode(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); } function_expression(A) ::= diff --git a/tests/army/cmdline/data/pwd.txt b/tests/army/cmdline/data/pwd.txt new file mode 100644 index 0000000000..771ffcffe6 --- /dev/null +++ b/tests/army/cmdline/data/pwd.txt @@ -0,0 +1 @@ +taosdata \ No newline at end of file diff --git a/tests/army/cmdline/data/source.sql b/tests/army/cmdline/data/source.sql new file mode 100644 index 0000000000..e546d17ace --- /dev/null +++ b/tests/army/cmdline/data/source.sql @@ -0,0 +1,37 @@ +drop database if exists db; +create database db ; +create table db.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table db.d0 using db.meters tags(1, 'china'); +insert into db.d0 values(1700000000001,1,1,1); +insert into db.d0 values(1700000000002,2,2,2); +insert into db.d0 values(1700000000003,3,3,4); +insert into db.d0 values(1700000000004,4,4,4); +insert into db.d0 values(1700000000005,5,5,5); +show db.tables; +set max_binary_display_width 100; +explain select * from db.meters; +show dnodes; +desc db.d0; +select * from db.meters limit 2; +drop database if exists dbus; +create database dbus precision 'us'; +create table dbus.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table dbus.d0 using dbus.meters tags(1, 'china'); +insert into dbus.d0 values(1700000000000001,1,1,1); +insert into dbus.d0 values(1700000000000002,2,2,2); +insert into dbus.d0 values(1700000000000003,3,3,3); +insert into dbus.d0 values(1700000000000004,4,4,4); +insert into dbus.d0 values(1700000000000005,5,5,5); +select * from dbus.meters; +select * from dbus.meters\G; +drop database if exists duns; +create database duns precision 'ns'; +create table duns.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24)); +create table duns.d0 using duns.meters tags(1, 'china'); +insert into duns.d0 values(1700000000000000001,1,1,1); +insert into duns.d0 values(1700000000000000002,2,2,2); +insert into duns.d0 values(1700000000000000003,3,3,4); +insert into duns.d0 values(1700000000000000004,4,4,4); +insert into duns.d0 values(1700000000000000005,5,5,5); +select * from duns.meters; +select * from duns.meters\G; \ No newline at end of file diff --git a/tests/army/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py index 7f97a6892a..4b99360676 100644 --- a/tests/army/cmdline/fullopt.py +++ b/tests/army/cmdline/fullopt.py @@ -30,7 +30,7 @@ class TDTestCase(TBase): updatecfgDict = { 'queryMaxConcurrentTables': '2K', 'streamMax': '1M', - 'totalMemoryKB': '1G', + 'totalMemoryKB': '32000000', 'streamMax': '1P', 'streamBufferSize':'1T', 'slowLogScope':"query" @@ -47,10 +47,6 @@ class TDTestCase(TBase): # taosBenchmark run etool.benchMark(command = f"-d {self.db} -t {self.childtable_count} -n {self.insert_rows} -v 2 -y") - def checkQueryOK(self, rets): - if rets[-2][:9] != "Query OK,": - tdLog.exit(f"check taos -s return unexpect: {rets}") - def doTaos(self): tdLog.info(f"check taos command options...") @@ -71,10 +67,10 @@ class TDTestCase(TBase): ] # exec for option in options: - rets = etool.runBinFile("taos", f"-s \"alter local '{option}'\";") - self.checkQueryOK(rets) + rlist = self.taos(f"-s \"alter local '{option}'\"") + self.checkListString(rlist, "Query OK,") # error - etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\";") + etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\"") # help rets = etool.runBinFile("taos", "--help") @@ -158,6 +154,8 @@ class TDTestCase(TBase): sc.dnodeStop(idx) etool.exeBinFile("taos", f'-n server', wait=False) time.sleep(3) + rlist = self.taos("-n client") + self.checkListString(rlist, "total succ: 100/100") eos.exe("pkill -9 taos") # call enter password diff --git a/tests/army/cmdline/json/taosCli.json b/tests/army/cmdline/json/taosCli.json index 2dcd28afdb..5bdecef72b 100644 --- a/tests/army/cmdline/json/taosCli.json +++ b/tests/army/cmdline/json/taosCli.json @@ -23,32 +23,45 @@ "childtable_prefix":"d", "data_source":"rand", "insert_mode":"taosc", - "childtable_count": 1, - "insert_rows":10, + "childtable_count": 2, + "insert_rows":100, "timestamp_step":1000, "start_timestamp":"2022-10-01 00:00:00.000", "columns":[ - { "type": "int", "count": 1000, "max": 2000, "min": 0 } + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} ], "tags":[ - { - "type":"binary", - "name":"location", - "max":64, - "min":1, - "values":[ - "San Francisco", - "Los Angles", - "San Diego", - "San Jose", - "Palo Alto", - "Campbell", - "Mountain View", - "Sunnyvale", - "Santa Clara", - "Cupertino" - ] - } + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} ] } ] diff --git a/tests/army/cmdline/json/taosCliDesc.json b/tests/army/cmdline/json/taosCliDesc.json new file mode 100644 index 0000000000..2dcd28afdb --- /dev/null +++ b/tests/army/cmdline/json/taosCliDesc.json @@ -0,0 +1,57 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 1, + "insert_rows":10, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "int", "count": 1000, "max": 2000, "min": 0 } + ], + "tags":[ + { + "type":"binary", + "name":"location", + "max":64, + "min":1, + "values":[ + "San Francisco", + "Los Angles", + "San Diego", + "San Jose", + "Palo Alto", + "Campbell", + "Mountain View", + "Sunnyvale", + "Santa Clara", + "Cupertino" + ] + } + ] + } + ] + } + ] +} diff --git a/tests/army/cmdline/taosCli.py b/tests/army/cmdline/taosCli.py index d446a25c86..ba52ba97be 100644 --- a/tests/army/cmdline/taosCli.py +++ b/tests/army/cmdline/taosCli.py @@ -42,7 +42,7 @@ class TDTestCase(TBase): tdLog.info(f"check describe show full.") # insert - json = "cmdline/json/taosCli.json" + json = "cmdline/json/taosCliDesc.json" db, stb, childCount, insertRows = self.insertBenchJson(json) # describe sql = f"describe {db}.{stb};" @@ -53,6 +53,167 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(2 + 1000) + + def checkResultWithMode(self, db, stb, arg): + result = "Query OK, 10 row(s)" + mode = arg[0] + rowh = arg[1] + rowv = arg[2] + idx = arg[3] + idxv = arg[4] + + # use db + if mode != "-R": + rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') + self.checkListString(rlist, "Database changed") + + # hori + cmd = f'{mode} -s "select * from {db}.{stb} limit 10' + rlist = self.taos(cmd + '"') + # line count + self.checkSame(len(rlist), rowh) + # last line + self.checkSame(rlist[idx][:len(result)], result) + + # vec + rlist = self.taos(cmd + '\G"') + # line count + self.checkSame(len(rlist), rowv) + self.checkSame(rlist[idxv], "*************************** 10.row ***************************") + # last line + self.checkSame(rlist[idx][:len(result)], result) + + # -B have some problem need todo + self.taos(f'{mode} -B -s "select * from {db}.{stb} where ts < 1"') + + # get empty result + rlist = self.taos(f'{mode} -r -s "select * from {db}.{stb} where ts < 1"') + self.checkListString(rlist, "Query OK, 0 row(s) in set") + + def checkBasic(self): + tdLog.info(f"check describe show full.") + + # insert + json = "cmdline/json/taosCli.json" + db, stb, childCount, insertRows = self.insertBenchJson(json) + + # native restful websock test + args = [ + ["", 18, 346, -2, 310], + ["-R", 22, 350, -3, 313], + ["-T 40 -E http://localhost:6041", 21, 349, -3, 312] + ] + for arg in args: + self.checkResultWithMode(db, stb, arg) + + + def checkDumpInOutMode(self, source, arg, db, insertRows): + mode = arg[0] + self.taos(f'{mode} -s "source {source}" ') + self.taos(f'{mode} -s "select * from {db}.d0>>d0.csv" ') + + # use db + rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ') + # update sql + rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ') + self.checkListString(rlist, "Query O") + + # only native support csv import + if mode == "": + self.taos(f'{mode} -s "delete from {db}.d0" ') + self.taos(f'{mode} -s "insert into {db}.d0 file d0.csv" ') + + sql = f"select count(*) from {db}.d0" + self.taos(f'{mode} -B -s "{sql}" ') + tdSql.checkAgg(sql, insertRows) + sql = f"select first(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 1) + sql = f"select last(voltage) from {db}.d0" + tdSql.checkFirstValue(sql, 5) + + def checkDumpInOut(self): + args = [ + ["", 18], + ["-R ", 22], + ["-E http://localhost:6041", 21] + ] + + source = "cmdline/data/source.sql" + db = "db" + insertRows = 5 + for arg in args: + # insert + self.checkDumpInOutMode(source, arg, db, insertRows) + + def checkVersion(self): + rlist1 = self.taos("-V") + rlist2 = self.taos("--version") + + self.checkSame(rlist1, rlist2) + self.checkSame(len(rlist1), 5) + + if len(rlist1[2]) < 42: + tdLog.exit("git commit id length is invalid: " + rlist1[2]) + + + def checkHelp(self): + # help + rlist1 = self.taos("--help") + rlist2 = self.taos("-?") + self.checkSame(rlist1, rlist2) + + # check return + strings = [ + "--auth=AUTH", + "--database=DATABASE", + "--version", + " --help" + ] + for string in strings: + self.checkListString(rlist1, string) + + def checkCommand(self): + # check coredump + + # o logpath + char = 'a' + lname =f'-o "/root/log/{char * 1000}/" -s "quit;"' + queryOK = "Query OK" + + # invalid input check + args = [ + [lname, "failed to create log at"], + ['-uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"', queryOK], + ['-o "./current/log/files/" -s"show databases;"', queryOK], + ['-a ""', "Invalid auth"], + ['-s "quit;"', "Welcome to the TDengine Command Line Interface"], + ['-a "abc"', "[0x80000357]"], + ['-h "" -s "show dnodes;"', "Invalid host"], + ['-u "" -s "show dnodes;"', "Invalid user"], + ['-P "" -s "show dnodes;"', "Invalid port"], + ['-u "AA" -s "show dnodes;"', "failed to connect to server"], + ['-p"abc" -s "show dnodes;"', "[0x80000357]"], + ['-d "abc" -s "show dnodes;"', "[0x80000388]"], + ['-N 0 -s "show dnodes;"', "Invalid pktNum"], + ['-N 10 -s "show dnodes;"', queryOK], + ['-w 0 -s "show dnodes;"', "Invalid displayWidth"], + ['-w 10 -s "show dnodes;"', queryOK], + ['-W 10 -s "show dnodes;"', None], + ['-l 0 -s "show dnodes;"', "Invalid pktLen"], + ['-l 10 -s "show dnodes;"', queryOK], + ['-C', "buildinfo"], + ['-B -s "show dnodes;"', queryOK], + ['-s "help;"', "Timestamp expression Format"], + ['-s ""', "Invalid commands"], + ['-t', "2: service ok"], + ['-uroot -p < cmdline/data/pwd.txt -s "show dnodes;"', queryOK], + ] + + for arg in args: + rlist = self.taos(arg[0]) + if arg[1] != None: + self.checkListString(rlist, arg[1]) + # run def run(self): tdLog.debug(f"start to excute {__file__}") @@ -60,6 +221,21 @@ class TDTestCase(TBase): # check show whole self.checkDescribe() + # check basic + self.checkBasic() + + # version + self.checkVersion() + + # help + self.checkHelp() + + # check command + self.checkCommand() + + # check data in/out + self.checkDumpInOut() + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index d8dcb29e44..4427ddcea5 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -19,6 +19,7 @@ import random import copy import json +import frame.etool import frame.eutil from frame.log import * from frame.sql import * @@ -231,6 +232,14 @@ class TBase: tdLog.info("sql1 same result with sql2.") + # check same value + def checkSame(self, real, expect, show = True): + if real == expect: + if show: + tdLog.info(f"check same succ. real={real} expect={expect}.") + else: + tdLog.exit(f"check same failed. real={real} expect={expect}.") + # # get db information # @@ -270,6 +279,16 @@ class TBase: print(dics) return dics +# +# run bin file +# + # taos + def taos(self, command, show = True, checkRun = False): + return frame.etool.runBinFile("taos", command, show, checkRun) + + def taosdump(self, command, show = True, checkRun = True, retFail = True): + return frame.etool.runBinFile("taosdump", command, show, checkRun, retFail) + # # util @@ -312,6 +331,17 @@ class TBase: tdLog.exit(f"list is empty {tips}") + # check list have str + def checkListString(self, vlist, s): + for i in range(len(vlist)): + if vlist[i].find(s) != -1: + # found + tdLog.info(f'found "{s}" on index {i} , line={vlist[i]}') + return + + # not found + tdLog.exit(f'faild, not found "{s}" on list:{vlist}') + # # str util # @@ -328,7 +358,7 @@ class TBase: # taosBenchmark # - # run taosBenchmark and check insert Result + # insert def insertBenchJson(self, jsonFile, options="", checkStep=False): # exe insert cmd = f"{options} -f {jsonFile}" @@ -395,4 +425,18 @@ class TBase: if vgroups != None: tdSql.checkData(0, 0, vgroups) - return db, stb,child_count, insert_rows \ No newline at end of file + return db, stb, child_count, insert_rows + + + # tmq + def tmqBenchJson(self, jsonFile, options="", checkStep=False): + # exe insert + command = f"{options} -f {jsonFile}" + rlist = frame.etool.runBinFile("taosBenchmark", command, checkRun = True) + + # + # check insert result + # + print(rlist) + + return rlist \ No newline at end of file diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 802b62e052..9994fa7497 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -52,8 +52,11 @@ def isArm64Cpu(): # # wait util execute file finished -def exe(file): - return os.system(file) +def exe(command, show = False): + code = os.system(command) + if show: + print(f"eos.exe retcode={code} command:{command}") + return code # execute file and return immediately def exeNoWait(file): @@ -64,24 +67,63 @@ def exeNoWait(file): return exe(cmd) # run return output and error -def run(command, timeout = 10): - process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - process.wait(timeout) +def run(command, show = True): + # out to file + id = time.clock_gettime_ns(time.CLOCK_REALTIME) % 100000 + out = f"out_{id}.txt" + err = f"err_{id}.txt" + + code = exe(command + f" 1>{out} 2>{err}", show) - output = process.stdout.read().decode(encoding="gbk") - error = process.stderr.read().decode(encoding="gbk") + # read from file + output = readFileContext(out) + error = readFileContext(err) - return output, error + # del + if os.path.exists(out): + os.remove(out) + if os.path.exists(err): + os.remove(err) + + return output, error, code # return list after run -def runRetList(command, timeout=10): - output,error = run(command, timeout) - return output.splitlines() +def runRetList(command, show = True, checkRun = False, retFail = False): + output, error, code = run(command, show) + if checkRun and code != 0: + print(f"eos.runRetList checkRun return code failed. code={code} error={error}") + assert code == 0 + + rList = output.splitlines() + if retFail and error != "": + rList += error.splitlines() + return rList # # file # def delFile(file): - return exe(f"rm -rf {file}") \ No newline at end of file + return exe(f"rm -rf {file}") + +def readFileContext(filename): + file = open(filename) + context = file.read() + file.close() + return context + +def writeFileContext(filename, context): + file = open(filename, "w") + file.write(context) + file.close() + +def appendFileContext(filename, context): + global resultContext + resultContext += context + try: + file = open(filename, "a") + wsize = file.write(context) + file.close() + except: + print(f"appand file error context={context} .") \ No newline at end of file diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 9065a3dede..4ad3efb036 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -51,7 +51,7 @@ def benchMark(command = "", json = "") : # run if command != "": - frame.eos.exe(bmFile + " " + command) + status = frame.eos.run(bmFile + " " + command) if json != "": cmd = f"{bmFile} -f {json}" print(cmd) @@ -66,7 +66,7 @@ def curFile(fullPath, filename): # run build/bin file -def runBinFile(fname, command, show=True): +def runBinFile(fname, command, show = True, checkRun = False, retFail = False ): binFile = frame.epath.binFile(fname) if frame.eos.isWin(): binFile += ".exe" @@ -74,7 +74,7 @@ def runBinFile(fname, command, show=True): cmd = f"{binFile} {command}" if show: tdLog.info(cmd) - return frame.eos.runRetList(cmd) + return frame.eos.runRetList(cmd, show, checkRun, retFail) # exe build/bin file def exeBinFile(fname, command, wait=True, show=True): diff --git a/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json b/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json new file mode 100644 index 0000000000..62ab67de2f --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/tmqBasicInsert.json @@ -0,0 +1,71 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count": 2, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes", + "vgroups": 4 + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 10, + "insert_rows": 100, + "timestamp_step": 1, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/benchmark/basic/json/tmqBasicPara.json b/tests/army/tools/benchmark/basic/json/tmqBasicPara.json new file mode 100644 index 0000000000..c5b9c5b4f2 --- /dev/null +++ b/tests/army/tools/benchmark/basic/json/tmqBasicPara.json @@ -0,0 +1,29 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "confirm_parameter_prompt": "no", + "result_file": "./tmq_result_para.txt", + "tmq_info": { + "concurrent": 2, + "poll_delay": 3000, + "create_mode": "parallel", + "group_mode": "independent", + "client.id": "clientId", + "auto.offset.reset": "earliest", + "enable.auto.commit": "true", + "auto.commit.interval.ms": 1000, + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "false", + "rows_file": "./rows_file_para", + "expect_rows": 100, + "topic_list": [ + {"name": "topic_benchmark_d0", "sql": "select * from test.d0;"} + ] + } +} diff --git a/tests/army/tools/benchmark/basic/json/tmq.json b/tests/army/tools/benchmark/basic/json/tmqBasicSequ.json similarity index 67% rename from tests/army/tools/benchmark/basic/json/tmq.json rename to tests/army/tools/benchmark/basic/json/tmqBasicSequ.json index cf0eb1b297..d508fe71eb 100644 --- a/tests/army/tools/benchmark/basic/json/tmq.json +++ b/tests/army/tools/benchmark/basic/json/tmqBasicSequ.json @@ -5,12 +5,13 @@ "port": 6030, "user": "root", "password": "taosdata", - "databases": "db", + "databases": "test", "confirm_parameter_prompt": "no", - "result_file": "./tmq_result1.txt", + "result_file": "./tmq_result_sequ.txt", "tmq_info": { - "concurrent": 2, + "concurrent": 4, "poll_delay": 3000, + "group_mode": "independent", "group.id": "grpId_0", "client.id": "clientId", "auto.offset.reset": "earliest", @@ -19,10 +20,10 @@ "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "false", - "rows_file": "./consumed_rows1", - "expect_rows": 50, + "rows_file": "./rows_file_sequ", + "expect_rows": 1000, "topic_list": [ - {"name": "tmq_topic_0", "sql": "select c0 from db.stb;"} + {"name": "topic_benchmark_meters", "sql": "select * from test.meters;"} ] } } diff --git a/tests/army/tools/benchmark/basic/default_tmq_json.py b/tests/army/tools/benchmark/basic/tmqBasic.py similarity index 51% rename from tests/army/tools/benchmark/basic/default_tmq_json.py rename to tests/army/tools/benchmark/basic/tmqBasic.py index e59b72d1f3..66d2290a1f 100644 --- a/tests/army/tools/benchmark/basic/default_tmq_json.py +++ b/tests/army/tools/benchmark/basic/tmqBasic.py @@ -25,37 +25,22 @@ from frame import * class TDTestCase(TBase): def caseDescription(self): """ - [TD-11510] taosBenchmark test cases + taosBenchmark tmp->Basic test cases """ - - def run(self): - tdSql.execute("drop topic if exists topic_0") - binPath = etool.benchMarkFile() - cmd = "%s -f ./tools/benchmark/basic/json/default.json" % binPath - tdLog.info("%s" % cmd) - os.system("%s" % cmd) - tdSql.execute("reset query cache") + + # insert data + json = "tools/benchmark/basic/json/tmqBasicInsert.json" + db, stb, child_count, insert_rows = self.insertBenchJson(json, checkStep = True) - tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000") + # tmq Sequ + json = "tools/benchmark/basic/json/tmqBasicSequ.json" + self.tmqBenchJson(json) - cmd = "%s -f ./tools/benchmark/basic/json/tmq.json " % binPath - tdLog.info("%s" % cmd) - os.system("%s" % cmd) - sleep(15) - - # try: - # for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"): - # fields = line.split() - - # pid = fields[0] - - # os.kill(int(pid), signal.SIGINT) - # time.sleep(3) - # print("taosBenchmark be killed on purpose") - # except: - # tdLog.exit("failed to kill taosBenchmark") + # tmq Parallel + json = "tools/benchmark/basic/json/tmqBasicPara.json" + self.tmqBenchJson(json) def stop(self): tdSql.close() diff --git a/tests/army/tools/taosdump/native/json/insertFullType.json b/tests/army/tools/taosdump/native/json/insertFullType.json new file mode 100644 index 0000000000..a121670c37 --- /dev/null +++ b/tests/army/tools/taosdump/native/json/insertFullType.json @@ -0,0 +1,66 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"test", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 2, + "insert_rows":100, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/taosdump/native/json/insertOther.json b/tests/army/tools/taosdump/native/json/insertOther.json new file mode 100644 index 0000000000..bd6bb12bec --- /dev/null +++ b/tests/army/tools/taosdump/native/json/insertOther.json @@ -0,0 +1,70 @@ +{ + "filetype":"insert", + "cfgdir":"/etc/taos", + "host":"127.0.0.1", + "port":6030, + "user":"root", + "password":"taosdata", + "thread_count":1, + "create_table_thread_count":1, + "confirm_parameter_prompt":"no", + "prepare_rand":100, + "num_of_records_per_req":100, + "databases": [ + { + "dbinfo":{ + "name":"testother", + "drop":"yes" + }, + "super_tables":[ + { + "name":"meters", + "child_table_exists":"no", + "childtable_prefix":"d", + "data_source":"rand", + "insert_mode":"taosc", + "childtable_count": 2, + "insert_rows":100, + "timestamp_step":1000, + "start_timestamp":"2022-10-01 00:00:00.000", + "columns":[ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "si", "max": 100, "min": -50 }, + { "type": "int", "name": "ic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "bi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "bin", "len": 4}, + { "type": "nchar", "name": "nch", "len": 8}, + { "type": "varbinary", "name": "vab", "len": 8}, + { "type": "varchar", "name": "vac", "len": 8}, + { "type": "geometry", "name": "geo", "len": 32} + ], + "tags":[ + { "type": "bool", "name": "tbc"}, + { "type": "float", "name": "tfc", "max": 1, "min": 0 }, + { "type": "double", "name": "tdc", "max": 10, "min": 0 }, + { "type": "tinyint", "name": "tti", "max": 100, "min": -100 }, + { "type": "smallint", "name": "tsi", "max": 100, "min": -50 }, + { "type": "int", "name": "tic", "max": 1000, "min": -1000 }, + { "type": "bigint", "name": "tbi", "max": 100, "min": -1000 }, + { "type": "utinyint", "name": "tuti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "tusi", "max": 100, "min": 0 }, + { "type": "uint", "name": "tui", "max": 1000, "min": 0 }, + { "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 }, + { "type": "binary", "name": "tbin", "len": 4}, + { "type": "nchar", "name": "tnch", "len": 8}, + { "type": "varbinary", "name": "tvab", "len": 8}, + { "type": "varchar", "name": "tvac", "len": 8}, + { "type": "geometry", "name": "tgeo", "len": 32} + ] + } + ] + } + ] +} diff --git a/tests/army/tools/taosdump/native/taosdumpCommandline.py b/tests/army/tools/taosdump/native/taosdumpCommandline.py index 0e7c3dd29f..c43ea0e9e7 100644 --- a/tests/army/tools/taosdump/native/taosdumpCommandline.py +++ b/tests/army/tools/taosdump/native/taosdumpCommandline.py @@ -14,6 +14,7 @@ import os import json import frame +import frame.eos import frame.etool from frame.log import * from frame.cases import * @@ -28,23 +29,19 @@ class TDTestCase(TBase): test taosdump support commandline arguments """ - def exec(self, command): - tdLog.info(command) - return os.system(command) - def clearPath(self, path): os.system("rm -rf %s/*" % path) def findPrograme(self): # taosdump - taosdump = etool.taosDumpFile() + taosdump = frame.etool.taosDumpFile() if taosdump == "": tdLog.exit("taosdump not found!") else: tdLog.info("taosdump found in %s" % taosdump) # taosBenchmark - benchmark = etool.benchMarkFile() + benchmark = frame.etool.benchMarkFile() if benchmark == "": tdLog.exit("benchmark not found!") else: @@ -60,7 +57,7 @@ class TDTestCase(TBase): return taosdump, benchmark,tmpdir - def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval=False): + def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval = True): # # check insert result # @@ -73,9 +70,9 @@ class TDTestCase(TBase): else: db = newdb - stb = data["databases"][0]["super_tables"][0]["name"] - child_count = data["databases"][0]["super_tables"][0]["childtable_count"] - insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] + stb = data["databases"][0]["super_tables"][0]["name"] + child_count = data["databases"][0]["super_tables"][0]["childtable_count"] + insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"] timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"] tdLog.info(f"get json: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} \n") @@ -91,15 +88,9 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(0) - def testBenchmarkJson(self, benchmark, jsonFile, options="", checkInterval=False): - # exe insert - cmd = f"{benchmark} {options} -f {jsonFile}" - self.exec(cmd) - self.checkCorrectWithJson(jsonFile) - - def insertData(self, benchmark, json, db): + def insertData(self, json): # insert super table - self.testBenchmarkJson(benchmark, json) + db, stb, child_count, insert_rows = self.insertBenchJson(json) # normal table sqls = [ @@ -112,14 +103,8 @@ class TDTestCase(TBase): ] for sql in sqls: tdSql.execute(sql) - - def dumpOut(self, taosdump, db , outdir): - # dump out - self.exec(f"{taosdump} -D {db} -o {outdir}") - - def dumpIn(self, taosdump, db, newdb, indir): - # dump in - self.exec(f'{taosdump} -W "{db}={newdb}" -i {indir}') + + return db, stb, child_count, insert_rows def checkSame(self, db, newdb, stb, aggfun): # sum pk db @@ -136,31 +121,75 @@ class TDTestCase(TBase): else: tdLog.exit(f"{aggfun} source db:{sum1} import db:{sum2} not equal.") - def verifyResult(self, db, newdb, json): # compare with insert json self.checkCorrectWithJson(json, newdb) # compare sum(pk) stb = "meters" - self.checkSame(db, newdb, stb, "sum(pk)") - self.checkSame(db, newdb, stb, "sum(usi)") + self.checkSame(db, newdb, stb, "sum(fc)") + self.checkSame(db, newdb, stb, "sum(ti)") + self.checkSame(db, newdb, stb, "sum(si)") self.checkSame(db, newdb, stb, "sum(ic)") + self.checkSame(db, newdb, stb, "avg(bi)") + self.checkSame(db, newdb, stb, "sum(uti)") + self.checkSame(db, newdb, stb, "sum(usi)") + self.checkSame(db, newdb, stb, "sum(ui)") + self.checkSame(db, newdb, stb, "avg(ubi)") # check normal table self.checkSame(db, newdb, "ntb", "sum(c1)") - - # basic commandline - def basicCommandLine(self, taosdump, tmpdir): - # -h -P -u -p -o - self.exec(taosdump + f" -h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}") + # with Native Rest and WebSocket + def dumpInOutMode(self, mode, db, json, tmpdir): + # dump out self.clearPath(tmpdir) + self.taosdump(f"{mode} -D {db} -o {tmpdir}") + + # dump in + newdb = "new" + db + self.taosdump(f"{mode} -W '{db}={newdb}' -i {tmpdir}") + + # check same + self.verifyResult(db, newdb, json) + + + # basic commandline + def basicCommandLine(self, tmpdir): + #command and check result + checkItems = [ + [f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped"]], + [f"-r result -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]], + [f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], + [f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]], + [f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"]], + [f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]], + [f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]], + [f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]], + [f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"]], + [f"--help", ["Report bugs to"]], + [f"-?", ["Report bugs to"]], + [f"-V", ["version:"]], + [f"--usage", ["taosdump [OPTION...] -o outpath"]] + ] + + # executes + for item in checkItems: + self.clearPath(tmpdir) + command = item[0] + results = item[1] + rlist = self.taosdump(command) + for result in results: + self.checkListString(rlist, result) + # clear tmp + # check except def checkExcept(self, command): try: - code = self.exec(command) + code = frame.eos.exe(command, show = True) if code == 0: tdLog.exit(f"Failed, not report error cmd:{command}") else: @@ -170,7 +199,7 @@ class TDTestCase(TBase): # except commandline - def exceptCommandLine(self, taosdump, tmpdir): + def exceptCommandLine(self, taosdump, db, stb, tmpdir): # -o self.checkExcept(taosdump + " -o= ") self.checkExcept(taosdump + " -o") @@ -178,34 +207,47 @@ class TDTestCase(TBase): self.checkExcept(taosdump + " -A -o ") self.checkExcept(taosdump + " -A -o ./noexistpath/") self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}") + self.checkExcept(taosdump + f" -d unknown -o {tmpdir}") + self.checkExcept(taosdump + f" -P invalidport") + self.checkExcept(taosdump + f" -D") + self.checkExcept(taosdump + f" -P 65536") + self.checkExcept(taosdump + f" -t 2 -k 2 -z 1 -C https://not-exist.com:80/cloud -D test -o {tmpdir}") + self.checkExcept(taosdump + f" -P 65536") # run def run(self): - # database - db = "pridb" - newdb = "npridb" # find taosdump, benchmark, tmpdir = self.findPrograme() - json = "./tools/taosdump/ws/json/primaryKey.json" + json = "./tools/taosdump/native/json/insertFullType.json" # insert data with taosBenchmark - self.insertData(benchmark, json, db) + db, stb, childCount, insertRows = self.insertData(json) + + # dumpInOut + modes = ["", "-R" , "--cloud=http://localhost:6041"] + for mode in modes: + self.dumpInOutMode(mode, db , json, tmpdir) + + tdLog.info("1. native rest ws dumpIn Out .......................... [Passed]") # basic commandline - self.basicCommandLine(taosdump, tmpdir) + self.basicCommandLine(tmpdir) + tdLog.info("2. basic command line .................................. [Passed]") # except commandline - self.exceptCommandLine(taosdump, tmpdir) + self.exceptCommandLine(taosdump, db, stb, tmpdir) + tdLog.info("3. except command line ................................. [Passed]") - # dump out - #self.dumpOut(taosdump, db, tmpdir) - - # dump in - #self.dumpIn(taosdump, db, newdb, tmpdir) - - # verify db - #self.verifyResult(db, newdb, json) + # + # varbinary and geometry for native + # + json = "./tools/taosdump/native/json/insertOther.json" + # insert + db, stb, childCount, insertRows = self.insertData(json) + # dump in/out + self.dumpInOutMode("", db , json, tmpdir) + tdLog.info("4. native varbinary geometry ........................... [Passed]") def stop(self): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 890e152554..1c0f550f5c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -70,7 +70,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py -,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py +,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py -B ,,n,army,python3 ./test.py -f whole/checkErrorCode.py # @@ -88,7 +88,6 @@ ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/custom_col_tag.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py -,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py @@ -140,6 +139,8 @@ ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R + +,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmqBasic.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py ,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/cloud/cloud-test.py @@ -206,7 +207,7 @@ ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeBool.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpRetry.py -B ,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeTinyInt.py -B -,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py +,,n,army,python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py -B # # system test diff --git a/tests/system-test/2-query/test_ts4467.py b/tests/system-test/2-query/test_ts4467.py index 4e09a9fbbe..52ea3aec70 100644 --- a/tests/system-test/2-query/test_ts4467.py +++ b/tests/system-test/2-query/test_ts4467.py @@ -55,6 +55,19 @@ class TDTestCase: tdLog.debug("res2: %s" % str(res2)) assert(len(res1) == len(res2) and res1[0][0] == res2[4][0]) + # test ts-5613 + sql = "select cast(2<>3 as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,1) + + sql = "select cast(2 not in(3) as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,1) + + sql = "select cast(2 is NULL as int) from t" + tdSql.query(sql) + tdSql.checkData(0,0,0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 87cdc546d2..f8ea42917c 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -112,6 +112,7 @@ SWords shellCommands[] = { {"create view as select", 0, 0, NULL}, {"compact database ", 0, 0, NULL}, #endif + {"desc ;", 0, 0, NULL}, {"describe ;", 0, 0, NULL}, {"delete from where ", 0, 0, NULL}, {"drop database ;", 0, 0, NULL}, diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ebec4ca06c..a3e542a768 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -254,8 +254,7 @@ void shellRunSingleCommandImp(char *command) { } if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { - fprintf(stdout, "Database changed.\r\n\r\n"); - fflush(stdout); + printf("Database changed.\r\n\r\n"); // call back auto tab module callbackAutoTab(command, pSql, true); diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 0350d934a7..a8c1193ab8 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -122,6 +122,7 @@ void shellCheckServerStatus() { } while (1); } #ifdef WEBSOCKET +char dsn[1024] = "ws://localhost:6041"; void shellCheckConnectMode() { if (shell.args.dsn) { shell.args.cloud = true; @@ -144,21 +145,22 @@ void shellCheckConnectMode() { shell.args.restful = false; return; } - - if (shell.args.restful) { - if (!shell.args.host) { - shell.args.host = "localhost"; - } - if (!shell.args.port) { - shell.args.port = 6041; - } - shell.args.dsn = taosMemoryCalloc(1, 1024); - snprintf(shell.args.dsn, 1024, "ws://%s:%d", - shell.args.host, shell.args.port); - } - shell.args.cloud = false; - return; } + + if (shell.args.restful) { + if (!shell.args.host) { + shell.args.host = "localhost"; + } + if (!shell.args.port) { + shell.args.port = 6041; + } + shell.args.dsn = dsn; + snprintf(shell.args.dsn, 1024, "ws://%s:%d", + shell.args.host, shell.args.port); + } + shell.args.cloud = false; + return; + } #endif diff --git a/tools/taos-tools/inc/bench.h b/tools/taos-tools/inc/bench.h index 078bc1ac20..97c82ac2fa 100644 --- a/tools/taos-tools/inc/bench.h +++ b/tools/taos-tools/inc/bench.h @@ -617,10 +617,8 @@ typedef struct SDataBase_S { int durMinute; // duration minutes BArray *cfgs; BArray *superTbls; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 int32_t vgroups; BArray *vgArray; -#endif // TD_VER_COMPATIBLE_3_0_0_0 bool flush; } SDataBase; @@ -838,9 +836,7 @@ typedef struct SThreadInfo_S { BArray* delayList; uint64_t *query_delay_list; double avg_delay; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 SVGroup *vg; -#endif int posOfTblCreatingBatch; int posOfTblCreatingInterval; diff --git a/tools/taos-tools/src/CMakeLists.txt b/tools/taos-tools/src/CMakeLists.txt index da05133f89..1f0899db5c 100644 --- a/tools/taos-tools/src/CMakeLists.txt +++ b/tools/taos-tools/src/CMakeLists.txt @@ -78,7 +78,6 @@ ADD_DEFINITIONS(-DBUILD_INFO="${BUILD_INFO}") MESSAGE(STATUS "build:${BUILD_INFO}") MESSAGE("") - # # build proj # @@ -88,15 +87,8 @@ INCLUDE_DIRECTORIES(/usr/local/taos/include) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/avro/lang/c/src) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/toolscJson/src) -if(NOT DEFINED TD_VER_COMPATIBLE) - SET(TD_VER_COMPATIBLE "3.0.0.0") -ENDIF() - -IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread) - INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex) - ADD_DEFINITIONS(-DTD_VER_COMPATIBLE_3_0_0_0) -ENDIF () +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread) +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex) MESSAGE("Current system is: ${CMAKE_SYSTEM_NAME}") @@ -128,22 +120,14 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin ADD_DEPENDENCIES(deps-snappy apache-avro) ADD_DEPENDENCIES(taosdump deps-jansson) ADD_DEPENDENCIES(taosdump deps-snappy) - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF() + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ELSE () INCLUDE_DIRECTORIES(/usr/local/include) ADD_DEFINITIONS(-DDARWIN) LINK_DIRECTORIES(/usr/local/lib) SET(OS_ID "Darwin") - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ELSE() - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF() + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) ENDIF () # websocket @@ -337,11 +321,7 @@ ELSE () SET(CMAKE_C_STANDARD 11) SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8") SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8") - IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0") - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) - ELSE () - ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c) - ENDIF () + ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c) ADD_EXECUTABLE(taosdump taosdump.c dumpUtil.c wsdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c) ADD_DEPENDENCIES(apache-avro tools-zlib) diff --git a/tools/taos-tools/src/benchCommandOpt.c b/tools/taos-tools/src/benchCommandOpt.c index 8c18b54eac..9bc57bff01 100644 --- a/tools/taos-tools/src/benchCommandOpt.c +++ b/tools/taos-tools/src/benchCommandOpt.c @@ -245,9 +245,8 @@ void initArgument() { g_arguments->trying_interval = 0; g_arguments->iface = TAOSC_IFACE; g_arguments->rest_server_ver_major = -1; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 g_arguments->inputted_vgroups = -1; -#endif + g_arguments->mistMode = false; initDatabase(); diff --git a/tools/taos-tools/src/benchData.c b/tools/taos-tools/src/benchData.c index f5fdccf538..9b58ad06ca 100644 --- a/tools/taos-tools/src/benchData.c +++ b/tools/taos-tools/src/benchData.c @@ -812,17 +812,6 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf, continue; } } - if (field->type == TSDB_DATA_TYPE_TIMESTAMP && !tag) { - n = snprintf(sampleDataBuf + pos, bufLen - pos, "now,"); - if (n < 0 || n >= bufLen - pos) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - return -1; - } else { - pos += n; - continue; - } - } switch (field->type) { case TSDB_DATA_TYPE_BOOL: { bool boolTmp = tmpBool(field); diff --git a/tools/taos-tools/src/benchJsonOpt.c b/tools/taos-tools/src/benchJsonOpt.c index 3e41908668..4791385741 100644 --- a/tools/taos-tools/src/benchJsonOpt.c +++ b/tools/taos-tools/src/benchJsonOpt.c @@ -2254,7 +2254,6 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) { return 0; } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 static int getMetaFromTmqJsonFile(tools_cJSON *json) { int32_t code = -1; @@ -2422,7 +2421,6 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { TMQ_PARSE_OVER: return code; } -#endif int readJsonConfig(char * file) { int32_t code = -1; diff --git a/tools/taos-tools/src/benchSubscribe.c b/tools/taos-tools/src/benchSubscribe.c deleted file mode 100644 index 7f0d7e1579..0000000000 --- a/tools/taos-tools/src/benchSubscribe.c +++ /dev/null @@ -1,458 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the MIT license as published by the Free Software - * Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - */ - -#include -#include "benchLog.h" - -extern int g_majorVersionOfClient; - -static void stable_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, - int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("failed to subscribe result, code:%d, reason:%s\n", - code, taos_errstr(res)); - return; - } - - if (param) fetchResult(res, ((threadInfo *)param)->filePath); - // tao_unsubscribe() will free result. -} - -static void specified_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param, - int code) { - if (res == NULL || taos_errno(res) != 0) { - errorPrint("failed to subscribe result, code:%d, reason:%s\n", - code, taos_errstr(res)); - return; - } - - if (param) fetchResult(res, ((threadInfo *)param)->filePath); - // tao_unsubscribe() will free result. -} - -static TAOS_SUB *subscribeImpl(QUERY_CLASS class, threadInfo *pThreadInfo, - char *sql, char *topic, bool restart, - uint64_t interval) { - TAOS_SUB *tsub = NULL; - - if (taos_select_db(pThreadInfo->conn->taos, g_queryInfo.dbName)) { - errorPrint("failed to select database(%s)\n", g_queryInfo.dbName); - return NULL; - } - if ((SPECIFIED_CLASS == class) && - (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) { - tsub = taos_subscribe( - pThreadInfo->conn->taos, restart, - topic, sql, specified_sub_callback, - (void *)pThreadInfo, - (int)g_queryInfo.specifiedQueryInfo.subscribeInterval); - } else if ((STABLE_CLASS == class) && - (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) { - tsub = taos_subscribe(pThreadInfo->conn->taos, restart, topic, sql, - stable_sub_callback, (void *)pThreadInfo, - (int)g_queryInfo.superQueryInfo.subscribeInterval); - } else { - tsub = taos_subscribe(pThreadInfo->conn->taos, - restart, topic, sql, NULL, - NULL, (int)interval); - } - - if (tsub == NULL) { - errorPrint("failed to create subscription. topic:%s, sql:%s\n", - topic, sql); - return NULL; - } - - return tsub; -} - -static void *specifiedSubscribe(void *sarg) { - int32_t *code = benchCalloc(1, sizeof(int32_t), false); - *code = -1; - threadInfo *pThreadInfo = (threadInfo *)sarg; -#ifdef LINUX - prctl(PR_SET_NAME, "specSub"); -#endif - sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - "taosbenchmark-subscribe-%" PRIu64 "-%d", pThreadInfo->querySeq, - pThreadInfo->threadID); - SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, - pThreadInfo->querySeq); - if (sql->result[0] != - '\0') { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, - "%s-%d", sql->result, pThreadInfo->threadID); - } - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, sql->command, - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) { - goto free_of_specified_subscribe; - } - - // start loop to consume result - - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0; - while ((g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq] == -1) || - (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] < - g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq])) { - infoPrint( - "consumed[%d]: %d, endAfterConsum[%" PRId64 "]: %d\n", - pThreadInfo->threadID, - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID], - pThreadInfo->querySeq, - g_queryInfo.specifiedQueryInfo - .endAfterConsume[pThreadInfo->querySeq]); - - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = - taos_consume( - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]); - if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) { - if (sql->result[0] != 0) { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, - "%s-%d", sql->result, pThreadInfo->threadID); - } - fetchResult( - g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID], - pThreadInfo->filePath); - - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID]++; - if ((g_queryInfo.specifiedQueryInfo - .resubAfterConsume[pThreadInfo->querySeq] != -1) && - (g_queryInfo.specifiedQueryInfo - .consumed[pThreadInfo->threadID] >= - g_queryInfo.specifiedQueryInfo - .resubAfterConsume[pThreadInfo->querySeq])) { - infoPrint( - "keepProgress:%d, resub specified query: %" PRIu64 - "\n", - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress, - pThreadInfo->querySeq); - g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = - 0; - taos_unsubscribe( - g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); - g_queryInfo.specifiedQueryInfo - .tsub[pThreadInfo->threadID] = subscribeImpl( - SPECIFIED_CLASS, pThreadInfo, sql->command, - g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID], - g_queryInfo.specifiedQueryInfo.subscribeRestart, - g_queryInfo.specifiedQueryInfo.subscribeInterval); - if (NULL == g_queryInfo.specifiedQueryInfo - .tsub[pThreadInfo->threadID]) { - goto free_of_specified_subscribe; - } - } - } - } - *code = 0; - taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]); -free_of_specified_subscribe: - return code; -} - -static void *superSubscribe(void *sarg) { - int32_t *code = benchCalloc(1, sizeof(int32_t), false); - *code = -1; - threadInfo *pThreadInfo = (threadInfo *)sarg; - TAOS_SUB * tsub[MAX_QUERY_SQL_COUNT] = {0}; - uint64_t tsubSeq; - char * subSqlStr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); -#ifdef LINUX - prctl(PR_SET_NAME, "superSub"); -#endif - if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) { - errorPrint( - "The table number(%" PRId64 - ") of the thread is more than max query sql count: %d\n", - pThreadInfo->ntables, MAX_QUERY_SQL_COUNT); - goto free_of_super_subscribe; - } - - char topic[32] = {0}; - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - sprintf(topic, "taosbenchmark-subscribe-%" PRIu64 "-%" PRIu64 "", i, - pThreadInfo->querySeq); - memset(subSqlStr, 0, TSDB_MAX_ALLOWED_SQL_LEN); - replaceChildTblName( - g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlStr, - (int)i); - if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) { - snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d", - g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - tsub[tsubSeq] = - subscribeImpl(STABLE_CLASS, pThreadInfo, subSqlStr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[tsubSeq]) { - goto free_of_super_subscribe; - } - } - - // start loop to consume result - int consumed[MAX_QUERY_SQL_COUNT]; - for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) { - consumed[i] = 0; - } - TAOS_RES *res = NULL; - - uint64_t st = 0, et = 0; - - while ( - (g_queryInfo.superQueryInfo.endAfterConsume == -1) || - (g_queryInfo.superQueryInfo.endAfterConsume > - consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) { - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } - - st = toolsGetTimestampMs(); - perfPrint( - "st: %" PRIu64 " et: %" PRIu64 " st-et: %" PRIu64 "\n", - st, et, (st - et)); - res = taos_consume(tsub[tsubSeq]); - et = toolsGetTimestampMs(); - perfPrint( - "st: %" PRIu64 " et: %" PRIu64 " delta: %" PRIu64 "\n", - st, et, (et - st)); - - if (res) { - if (g_queryInfo.superQueryInfo - .result[pThreadInfo->querySeq][0] != 0) { - snprintf(pThreadInfo->filePath, - MAX_PATH_LEN, - "%s-%d", - g_queryInfo.superQueryInfo - .result[pThreadInfo->querySeq], - pThreadInfo->threadID); - } - fetchResult(res, pThreadInfo->filePath); - consumed[tsubSeq]++; - - if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) && - (consumed[tsubSeq] >= - g_queryInfo.superQueryInfo.resubAfterConsume)) { - taos_unsubscribe( - tsub[tsubSeq], - g_queryInfo.superQueryInfo.subscribeKeepProgress); - consumed[tsubSeq] = 0; - tsub[tsubSeq] = subscribeImpl( - STABLE_CLASS, pThreadInfo, subSqlStr, topic, - g_queryInfo.superQueryInfo.subscribeRestart, - g_queryInfo.superQueryInfo.subscribeInterval); - if (NULL == tsub[tsubSeq]) { - goto free_of_super_subscribe; - } - } - } - } - } - taos_free_result(res); - - for (uint64_t i = pThreadInfo->start_table_from; - i <= pThreadInfo->end_table_to; i++) { - tsubSeq = i - pThreadInfo->start_table_from; - taos_unsubscribe(tsub[tsubSeq], 0); - } - *code = 0; -free_of_super_subscribe: - - tmfree(subSqlStr); - return code; -} - -int subscribeTestProcess() { - prompt(0); - - if (REST_IFACE == g_queryInfo.iface) { - encodeAuthBase64(); - } - if (0 != g_queryInfo.superQueryInfo.sqlCount) { - SBenchConn* conn = initBenchConn(); - if (conn == NULL) { - return -1; - } - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (3 == g_majorVersionOfClient) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(TBNAME) FROM %s.%s", - g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName); - } - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == strlen((char *)(row[0]))) { - errorPrint("stable %s have no child table\n", - g_queryInfo.superQueryInfo.stbName); - return -1; - } - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp); - } - infoPrint("%s's childTblCount: %" PRId64 "\n", - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblCount); - taos_free_result(res); - g_queryInfo.superQueryInfo.childTblName = - benchCalloc(g_queryInfo.superQueryInfo.childTblCount, - sizeof(char *), false); - if (getAllChildNameOfSuperTable( - conn->taos, g_queryInfo.dbName, - g_queryInfo.superQueryInfo.stbName, - g_queryInfo.superQueryInfo.childTblName, - g_queryInfo.superQueryInfo.childTblCount)) { - closeBenchConn(conn); - return -1; - } - closeBenchConn(conn); - } - - pthread_t * pids = NULL; - threadInfo *infos = NULL; - - pthread_t * pidsOfStable = NULL; - threadInfo *infosOfStable = NULL; - - //==== create threads for query for specified table - int sqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; - if (sqlCount > 0) { - pids = benchCalloc(1, sqlCount - *g_queryInfo.specifiedQueryInfo.concurrent - *sizeof(pthread_t), false); - infos = benchCalloc(1, sqlCount - *g_queryInfo.specifiedQueryInfo.concurrent - *sizeof(threadInfo), false); - for (int i = 0; i < sqlCount; i++) { - for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; - j++) { - uint64_t seq = - i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *pThreadInfo = infos + seq; - pThreadInfo->threadID = (int)seq; - pThreadInfo->querySeq = i; - pThreadInfo->conn = initBenchConn(); - pthread_create(pids + seq, NULL, specifiedSubscribe, - pThreadInfo); - } - } - - for (int i = 0; i < sqlCount; i++) { - for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; - j++) { - uint64_t seq = - i * g_queryInfo.specifiedQueryInfo.concurrent + j; - threadInfo *pThreadInfo = infos + seq; - void *result; - pthread_join(pids[seq], &result); - if (*(int32_t *)result) { - g_fail = true; - } - closeBenchConn(pThreadInfo->conn); - tmfree(result); - } - } - } - - //==== create threads for super table query - if (g_queryInfo.superQueryInfo.sqlCount > 0 && - g_queryInfo.superQueryInfo.threadCnt > 0) { - pidsOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(pthread_t), false); - - infosOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount * - g_queryInfo.superQueryInfo.threadCnt * - sizeof(threadInfo), false); - - int64_t ntables = g_queryInfo.superQueryInfo.childTblCount; - int threads = g_queryInfo.superQueryInfo.threadCnt; - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - - int64_t b = 0; - if (threads != 0) { - b = ntables % threads; - } - - for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - uint64_t tableFrom = 0; - for (int j = 0; j < threads; j++) { - uint64_t seq = i * threads + j; - threadInfo *pThreadInfo = infosOfStable + seq; - pThreadInfo->threadID = (int)seq; - pThreadInfo->querySeq = i; - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = j < b ? a + 1 : a; - pThreadInfo->end_table_to = - j < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->conn = initBenchConn(); - pthread_create(pidsOfStable + seq, NULL, superSubscribe, - pThreadInfo); - } - } - - g_queryInfo.superQueryInfo.threadCnt = threads; - - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - for (int j = 0; j < threads; j++) { - uint64_t seq = (uint64_t)i * threads + j; - threadInfo *pThreadInfo = infosOfStable + seq; - void * result; - pthread_join(pidsOfStable[seq], &result); - if (*(int32_t *)result) { - g_fail = true; - } - closeBenchConn(pThreadInfo->conn); - tmfree(result); - } - } - } - - tmfree((char *)pids); - tmfree((char *)infos); - - tmfree((char *)pidsOfStable); - tmfree((char *)infosOfStable); - // - if (g_fail) { - return -1; - } - return 0; -} diff --git a/tools/taos-tools/src/benchSys.c b/tools/taos-tools/src/benchSys.c index 9e29701515..d7e44c045f 100644 --- a/tools/taos-tools/src/benchSys.c +++ b/tools/taos-tools/src/benchSys.c @@ -77,9 +77,7 @@ void benchPrintHelp() { printf("%s%s%s%s\r\n", indent, "-W,", indent, BENCH_DSN); printf("%s%s%s%s\r\n", indent, "-D,", indent, BENCH_TIMEOUT); #endif -#ifdef TD_VER_COMPATIBLE_3_0_0_0 printf("%s%s%s%s\r\n", indent, "-v,", indent, BENCH_VGROUPS); -#endif printf("%s%s%s%s\r\n", indent, "-V,", indent, BENCH_VERSION); printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); } @@ -199,9 +197,7 @@ static struct argp_option bench_options[] = { #endif {"keep-trying", 'k', "NUMBER", 0, BENCH_KEEPTRYING}, {"trying-interval", 'z', "NUMBER", 0, BENCH_TRYING_INTERVAL}, -#ifdef TD_VER_COMPATIBLE_3_0_0_0 {"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS}, -#endif {"version", 'V', 0, 0, BENCH_VERSION}, {"nodrop", 'Q', 0, 0, BENCH_NODROP}, {0} @@ -642,14 +638,12 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) { g_arguments->timeout = atoi(arg); break; #endif -#ifdef TD_VER_COMPATIBLE_3_0_0_0 case 'v': if (!toolsIsStringNumber(arg)) { errorPrintReqArg2(CUS_PROMPT"Benchmark", "v"); } g_arguments->inputted_vgroups = atoi(arg); break; -#endif case 'Q': database->drop = false; g_argFlag |= ARG_OPT_NODROP; diff --git a/tools/taos-tools/src/taosdump.c b/tools/taos-tools/src/taosdump.c index d27812b67a..35fae99c6a 100644 --- a/tools/taos-tools/src/taosdump.c +++ b/tools/taos-tools/src/taosdump.c @@ -5831,26 +5831,6 @@ static int64_t dumpInAvroDataImpl( #endif } } // tbName -#ifndef TD_VER_COMPATIBLE_3_0_0_0 - else { - // 2.6 need call taos_stmt_set_tbname every loop - const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3; - char *escapedTbName = calloc(1, escapedTbNameLen); - snprintf(escapedTbName, escapedTbNameLen, "%s%s%s", - g_escapeChar, tbName, g_escapeChar); - - if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) { - errorPrint("Failed to execute taos_stmt_set_tbname(%s)." - "reason: %s\n", - escapedTbName, taos_stmt_errstr(stmt)); - free(escapedTbName); - freeTbNameIfLooseMode(tbName); - tbName = NULL; - continue; - } - free(escapedTbName); - } -#endif debugPrint("%s() LN%d, count: %"PRId64"\n", __func__, __LINE__, count);