xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/internal

This commit is contained in:
Simon Guan 2025-02-27 10:04:34 +08:00
parent 960ab5d64e 7494432770
commit 7f10288df1
34 changed files with 983 additions and 729 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
source/client/src/clientSml.c
View File

@ -751,7 +751,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
}
static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) {
SHashObj *schemaHashCheck, ESchemaAction *action, bool isTag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
@ -759,15 +759,11 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
SML_CHECK_NULL(kv);
SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info));
}
for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j);
SML_CHECK_NULL(kv);
if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) {
if (taosHashGet(schemaHashCheck, kv->key, kv->keyLen) != NULL) {
SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN);
}
}
END:
RETURN
}
@ -998,14 +994,14 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa
END:
RETURN
}
static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn,
static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn,
SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){
ESchemaAction action = SCHEMA_ACTION_NULL;
SArray *pColumns = NULL;
SArray *pTags = NULL;
int32_t code = 0;
int32_t lino = 0;
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true));
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, hashTmpCheck, &action, true));
if (action != SCHEMA_ACTION_NULL) {
SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE));
@ -1029,14 +1025,14 @@ END:
RETURN
}
static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn,
static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn,
SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){
ESchemaAction action = SCHEMA_ACTION_NULL;
SArray *pColumns = NULL;
SArray *pTags = NULL;
int32_t code = 0;
int32_t lino = 0;
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false));
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, hashTmpCheck, &action, false));
if (action != SCHEMA_ACTION_NULL) {
SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE));
@ -1079,7 +1075,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
int32_t code = 0;
int32_t lino = 0;
SHashObj *hashTmp = NULL;
SHashObj *colHashTmp = NULL;
SHashObj *tagHashTmp = NULL;
STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
@ -1119,17 +1116,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK);
}
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
SML_CHECK_NULL(hashTmp);
SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags));
SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta));
taosHashClear(hashTmp);
SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns));
SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta));
colHashTmp = taosHashInit(pTableMeta->tableInfo.numOfColumns, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
tagHashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
SML_CHECK_NULL(colHashTmp);
SML_CHECK_NULL(tagHashTmp);
SML_CHECK_CODE(smlBuildTempHash(tagHashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags));
SML_CHECK_CODE(smlBuildTempHash(colHashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns));
SML_CHECK_CODE(smlModifyTag(info, colHashTmp, tagHashTmp, &conn, sTableData, &pName, &pTableMeta));
SML_CHECK_CODE(smlModifyCols(info, tagHashTmp, colHashTmp, &conn, sTableData, &pName, &pTableMeta));
needCheckMeta = true;
taosHashCleanup(hashTmp);
hashTmp = NULL;
taosHashCleanup(colHashTmp);
taosHashCleanup(tagHashTmp);
colHashTmp = NULL;
tagHashTmp = NULL;
} else {
uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code));
goto END;
@ -1153,7 +1154,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
END:
taosHashCancelIterate(info->superTables, tmp);
taosHashCleanup(hashTmp);
taosHashCleanup(colHashTmp);
taosHashCleanup(tagHashTmp);
taosMemoryFreeClear(pTableMeta);
(void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error
uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code,
@ -1924,3 +1926,4 @@ TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
TSDB_DEFAULT_TABLE_TTL, 0);
}

20
source/client/test/clientTests.cpp
View File

@ -1523,29 +1523,29 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "drop database if exists db1");
TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_timezone");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "create database db1");
pRes = taos_query(pConn, "create database db_timezone");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)");
pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)");
while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) {
taosMsleep(2000);
pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)");
pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)");
}
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
char sql[256] = {0};
(void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:00', 1)");
(void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:00', 1)");
pRes = taos_query(pConn, sql);
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 17:00:00'");
pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 17:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1571,7 +1571,7 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 01:00:00'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 01:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1588,7 +1588,7 @@ TEST(clientCase, timezone_Test) {
taos_free_result(pRes);
char sql[256] = {0};
(void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:01', 1)");
(void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:01', 1)");
pRes = taos_query(pConn, sql);
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
@ -1604,7 +1604,7 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 09:00:00'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 09:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1620,7 +1620,7 @@ TEST(clientCase, timezone_Test) {
taos_free_result(pRes);
{
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-17 01:00:01'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-17 01:00:01'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;

32
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1259,6 +1259,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL;
SMCreateStbReq createReq = {0};
bool isAlter = false;
SHashObj *pHash = NULL;
if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
@ -1319,6 +1320,33 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
goto _OVER;
}
pHash = taosHashInit(createReq.numOfColumns + createReq.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
false, HASH_NO_LOCK);
if (pHash == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
for (int32_t i = 0; i < createReq.numOfColumns; ++i) {
SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i);
if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) {
if (code == TSDB_CODE_DUP_KEY) {
code = TSDB_CODE_TSC_DUP_COL_NAMES;
}
goto _OVER;
}
}
for (int32_t i = 0; i < createReq.numOfTags; ++i) {
SField *pField = taosArrayGet(createReq.pTags, i);
if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) {
if (code == TSDB_CODE_DUP_KEY) {
code = TSDB_CODE_TSC_DUP_COL_NAMES;
}
goto _OVER;
}
}
pDb = mndAcquireDbByStb(pMnode, createReq.name);
if (pDb == NULL) {
code = TSDB_CODE_MND_DB_NOT_SELECTED;
@ -1383,6 +1411,10 @@ _OVER:
mndReleaseDb(pMnode, pDb);
tFreeSMCreateStbReq(&createReq);
if (pHash != NULL) {
taosHashCleanup(pHash);
}
TAOS_RETURN(code);
}

89
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -25,6 +25,7 @@ class MndTestStb : public ::testing::Test {
void* BuildCreateDbReq(const char* dbname, int32_t* pContLen);
void* BuildDropDbReq(const char* dbname, int32_t* pContLen);
void* BuildCreateStbReq(const char* stbname, int32_t* pContLen);
void* BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen);
void* BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen);
void* BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen);
void* BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname,
@ -137,6 +138,71 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) {
return pHead;
}
void* MndTestStb::BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen) {
SMCreateStbReq createReq = {0};
createReq.numOfColumns = 2;
createReq.numOfTags = 4;
createReq.igExists = 0;
createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
strcpy(createReq.name, stbname);
{
SField field = {0};
field.bytes = 8;
field.type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(field.name, "ts");
taosArrayPush(createReq.pColumns, &field);
}
{
SField field = {0};
field.bytes = 12;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "col1");
taosArrayPush(createReq.pColumns, &field);
}
{
SField field = {0};
field.bytes = 2;
field.type = TSDB_DATA_TYPE_TINYINT;
strcpy(field.name, "tag1");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 8;
field.type = TSDB_DATA_TYPE_BIGINT;
strcpy(field.name, "tag2");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 16;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "tag3");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 16;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "tag3");
taosArrayPush(createReq.pTags, &field);
}
int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
void* pHead = rpcMallocCont(tlen);
tSerializeSMCreateStbReq(pHead, tlen, &createReq);
tFreeSMCreateStbReq(&createReq);
*pContLen = tlen;
return pHead;
}
void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) {
SMAlterStbReq req = {0};
strcpy(req.name, stbname);
@ -896,3 +962,26 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
rpcFreeCont(pRsp->pCont);
}
}
TEST_F(MndTestStb, 09_Create_Duplicate_Stb) {
const char* dbname = "1.d2";
const char* stbname = "1.d2.stb";
{
int32_t contLen = 0;
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
int32_t contLen = 0;
void* pReq = BuildCreateStbDuplicateReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_TSC_DUP_COL_NAMES);
rpcFreeCont(pRsp->pCont);
}
}

8
source/libs/parser/inc/sql.y
View File

@ -50,10 +50,14 @@
%left AND.
%left UNION ALL MINUS EXCEPT INTERSECT.
%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT.
%left NK_LT NK_GT NK_LE NK_GE NK_EQ NK_NE LIKE MATCH NMATCH REGEXP CONTAINS BETWEEN IS IN.
%left NK_PLUS NK_MINUS.
%left NK_STAR NK_SLASH NK_REM.
%left NK_CONCAT.
%right NOT.
%left NK_ARROW.
%right INNER LEFT RIGHT FULL OUTER SEMI ANTI ASOF WINDOW JOIN ON WINDOW_OFFSET JLIMIT.
/************************************************ create/alter account *****************************************/
@ -1285,9 +1289,9 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= cols_func(B) NK_LP cols_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::=
CAST(B) NK_LP expr_or_subquery(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
CAST(B) NK_LP common_expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
function_expression(A) ::=
CAST(B) NK_LP expr_or_subquery(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
CAST(B) NK_LP common_expression(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
function_expression(A) ::=
POSITION(B) NK_LP expr_or_subquery(C) IN expr_or_subquery(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createPositionFunctionNode(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); }
function_expression(A) ::=

1
tests/army/cmdline/data/pwd.txt Normal file
View File

@ -0,0 +1 @@
taosdata

37
tests/army/cmdline/data/source.sql Normal file
View File

@ -0,0 +1,37 @@
drop database if exists db;
create database db ;
create table db.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table db.d0 using db.meters tags(1, 'china');
insert into db.d0 values(1700000000001,1,1,1);
insert into db.d0 values(1700000000002,2,2,2);
insert into db.d0 values(1700000000003,3,3,4);
insert into db.d0 values(1700000000004,4,4,4);
insert into db.d0 values(1700000000005,5,5,5);
show db.tables;
set max_binary_display_width 100;
explain select * from db.meters;
show dnodes;
desc db.d0;
select * from db.meters limit 2;
drop database if exists dbus;
create database dbus precision 'us';
create table dbus.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table dbus.d0 using dbus.meters tags(1, 'china');
insert into dbus.d0 values(1700000000000001,1,1,1);
insert into dbus.d0 values(1700000000000002,2,2,2);
insert into dbus.d0 values(1700000000000003,3,3,3);
insert into dbus.d0 values(1700000000000004,4,4,4);
insert into dbus.d0 values(1700000000000005,5,5,5);
select * from dbus.meters;
select * from dbus.meters\G;
drop database if exists duns;
create database duns precision 'ns';
create table duns.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table duns.d0 using duns.meters tags(1, 'china');
insert into duns.d0 values(1700000000000000001,1,1,1);
insert into duns.d0 values(1700000000000000002,2,2,2);
insert into duns.d0 values(1700000000000000003,3,3,4);
insert into duns.d0 values(1700000000000000004,4,4,4);
insert into duns.d0 values(1700000000000000005,5,5,5);
select * from duns.meters;
select * from duns.meters\G;

14
tests/army/cmdline/fullopt.py
View File

@ -30,7 +30,7 @@ class TDTestCase(TBase):
updatecfgDict = {
'queryMaxConcurrentTables': '2K',
'streamMax': '1M',
'totalMemoryKB': '1G',
'totalMemoryKB': '32000000',
'streamMax': '1P',
'streamBufferSize':'1T',
'slowLogScope':"query"
@ -47,10 +47,6 @@ class TDTestCase(TBase):
# taosBenchmark run
etool.benchMark(command = f"-d {self.db} -t {self.childtable_count} -n {self.insert_rows} -v 2 -y")
def checkQueryOK(self, rets):
if rets[-2][:9] != "Query OK,":
tdLog.exit(f"check taos -s return unexpect: {rets}")
def doTaos(self):
tdLog.info(f"check taos command options...")
@ -71,10 +67,10 @@ class TDTestCase(TBase):
]
# exec
for option in options:
rets = etool.runBinFile("taos", f"-s \"alter local '{option}'\";")
self.checkQueryOK(rets)
rlist = self.taos(f"-s \"alter local '{option}'\"")
self.checkListString(rlist, "Query OK,")
# error
etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\";")
etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\"")
# help
rets = etool.runBinFile("taos", "--help")
@ -158,6 +154,8 @@ class TDTestCase(TBase):
sc.dnodeStop(idx)
etool.exeBinFile("taos", f'-n server', wait=False)
time.sleep(3)
rlist = self.taos("-n client")
self.checkListString(rlist, "total succ: 100/100")
eos.exe("pkill -9 taos")
# call enter password

55
tests/army/cmdline/json/taosCli.json
View File

@ -23,32 +23,45 @@
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 1,
"insert_rows":10,
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "int", "count": 1000, "max": 2000, "min": 0 }
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{
"type":"binary",
"name":"location",
"max":64,
"min":1,
"values":[
"San Francisco",
"Los Angles",
"San Diego",
"San Jose",
"Palo Alto",
"Campbell",
"Mountain View",
"Sunnyvale",
"Santa Clara",
"Cupertino"
]
}
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]

57
tests/army/cmdline/json/taosCliDesc.json Normal file
View File

@ -0,0 +1,57 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 1,
"insert_rows":10,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "int", "count": 1000, "max": 2000, "min": 0 }
],
"tags":[
{
"type":"binary",
"name":"location",
"max":64,
"min":1,
"values":[
"San Francisco",
"Los Angles",
"San Diego",
"San Jose",
"Palo Alto",
"Campbell",
"Mountain View",
"Sunnyvale",
"Santa Clara",
"Cupertino"
]
}
]
}
]
}
]
}

178
tests/army/cmdline/taosCli.py
View File

@ -42,7 +42,7 @@ class TDTestCase(TBase):
tdLog.info(f"check describe show full.")
# insert
json = "cmdline/json/taosCli.json"
json = "cmdline/json/taosCliDesc.json"
db, stb, childCount, insertRows = self.insertBenchJson(json)
# describe
sql = f"describe {db}.{stb};"
@ -53,6 +53,167 @@ class TDTestCase(TBase):
tdSql.query(sql)
tdSql.checkRows(2 + 1000)
def checkResultWithMode(self, db, stb, arg):
result = "Query OK, 10 row(s)"
mode = arg[0]
rowh = arg[1]
rowv = arg[2]
idx = arg[3]
idxv = arg[4]
# use db
if mode != "-R":
rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ')
self.checkListString(rlist, "Database changed")
# hori
cmd = f'{mode} -s "select * from {db}.{stb} limit 10'
rlist = self.taos(cmd + '"')
# line count
self.checkSame(len(rlist), rowh)
# last line
self.checkSame(rlist[idx][:len(result)], result)
# vec
rlist = self.taos(cmd + '\G"')
# line count
self.checkSame(len(rlist), rowv)
self.checkSame(rlist[idxv], "*************************** 10.row ***************************")
# last line
self.checkSame(rlist[idx][:len(result)], result)
# -B have some problem need todo
self.taos(f'{mode} -B -s "select * from {db}.{stb} where ts < 1"')
# get empty result
rlist = self.taos(f'{mode} -r -s "select * from {db}.{stb} where ts < 1"')
self.checkListString(rlist, "Query OK, 0 row(s) in set")
def checkBasic(self):
tdLog.info(f"check describe show full.")
# insert
json = "cmdline/json/taosCli.json"
db, stb, childCount, insertRows = self.insertBenchJson(json)
# native restful websock test
args = [
["", 18, 346, -2, 310],
["-R", 22, 350, -3, 313],
["-T 40 -E http://localhost:6041", 21, 349, -3, 312]
]
for arg in args:
self.checkResultWithMode(db, stb, arg)
def checkDumpInOutMode(self, source, arg, db, insertRows):
mode = arg[0]
self.taos(f'{mode} -s "source {source}" ')
self.taos(f'{mode} -s "select * from {db}.d0>>d0.csv" ')
# use db
rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ')
# update sql
rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ')
self.checkListString(rlist, "Query O")
# only native support csv import
if mode == "":
self.taos(f'{mode} -s "delete from {db}.d0" ')
self.taos(f'{mode} -s "insert into {db}.d0 file d0.csv" ')
sql = f"select count(*) from {db}.d0"
self.taos(f'{mode} -B -s "{sql}" ')
tdSql.checkAgg(sql, insertRows)
sql = f"select first(voltage) from {db}.d0"
tdSql.checkFirstValue(sql, 1)
sql = f"select last(voltage) from {db}.d0"
tdSql.checkFirstValue(sql, 5)
def checkDumpInOut(self):
args = [
["", 18],
["-R ", 22],
["-E http://localhost:6041", 21]
]
source = "cmdline/data/source.sql"
db = "db"
insertRows = 5
for arg in args:
# insert
self.checkDumpInOutMode(source, arg, db, insertRows)
def checkVersion(self):
rlist1 = self.taos("-V")
rlist2 = self.taos("--version")
self.checkSame(rlist1, rlist2)
self.checkSame(len(rlist1), 5)
if len(rlist1[2]) < 42:
tdLog.exit("git commit id length is invalid: " + rlist1[2])
def checkHelp(self):
# help
rlist1 = self.taos("--help")
rlist2 = self.taos("-?")
self.checkSame(rlist1, rlist2)
# check return
strings = [
"--auth=AUTH",
"--database=DATABASE",
"--version",
" --help"
]
for string in strings:
self.checkListString(rlist1, string)
def checkCommand(self):
# check coredump
# o logpath
char = 'a'
lname =f'-o "/root/log/{char * 1000}/" -s "quit;"'
queryOK = "Query OK"
# invalid input check
args = [
[lname, "failed to create log at"],
['-uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"', queryOK],
['-o "./current/log/files/" -s"show databases;"', queryOK],
['-a ""', "Invalid auth"],
['-s "quit;"', "Welcome to the TDengine Command Line Interface"],
['-a "abc"', "[0x80000357]"],
['-h "" -s "show dnodes;"', "Invalid host"],
['-u "" -s "show dnodes;"', "Invalid user"],
['-P "" -s "show dnodes;"', "Invalid port"],
['-u "AA" -s "show dnodes;"', "failed to connect to server"],
['-p"abc" -s "show dnodes;"', "[0x80000357]"],
['-d "abc" -s "show dnodes;"', "[0x80000388]"],
['-N 0 -s "show dnodes;"', "Invalid pktNum"],
['-N 10 -s "show dnodes;"', queryOK],
['-w 0 -s "show dnodes;"', "Invalid displayWidth"],
['-w 10 -s "show dnodes;"', queryOK],
['-W 10 -s "show dnodes;"', None],
['-l 0 -s "show dnodes;"', "Invalid pktLen"],
['-l 10 -s "show dnodes;"', queryOK],
['-C', "buildinfo"],
['-B -s "show dnodes;"', queryOK],
['-s "help;"', "Timestamp expression Format"],
['-s ""', "Invalid commands"],
['-t', "2: service ok"],
['-uroot -p < cmdline/data/pwd.txt -s "show dnodes;"', queryOK],
]
for arg in args:
rlist = self.taos(arg[0])
if arg[1] != None:
self.checkListString(rlist, arg[1])
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
@ -60,6 +221,21 @@ class TDTestCase(TBase):
# check show whole
self.checkDescribe()
# check basic
self.checkBasic()
# version
self.checkVersion()
# help
self.checkHelp()
# check command
self.checkCommand()
# check data in/out
self.checkDumpInOut()
tdLog.success(f"{__file__} successfully executed")

48
tests/army/frame/caseBase.py
View File

@ -19,6 +19,7 @@ import random
import copy
import json
import frame.etool
import frame.eutil
from frame.log import *
from frame.sql import *
@ -231,6 +232,14 @@ class TBase:
tdLog.info("sql1 same result with sql2.")
# check same value
def checkSame(self, real, expect, show = True):
if real == expect:
if show:
tdLog.info(f"check same succ. real={real} expect={expect}.")
else:
tdLog.exit(f"check same failed. real={real} expect={expect}.")
#
# get db information
#
@ -270,6 +279,16 @@ class TBase:
print(dics)
return dics
#
# run bin file
#
# taos
def taos(self, command, show = True, checkRun = False):
return frame.etool.runBinFile("taos", command, show, checkRun)
def taosdump(self, command, show = True, checkRun = True, retFail = True):
return frame.etool.runBinFile("taosdump", command, show, checkRun, retFail)
#
# util
@ -312,6 +331,17 @@ class TBase:
tdLog.exit(f"list is empty {tips}")
# check list have str
def checkListString(self, vlist, s):
for i in range(len(vlist)):
if vlist[i].find(s) != -1:
# found
tdLog.info(f'found "{s}" on index {i} , line={vlist[i]}')
return
# not found
tdLog.exit(f'faild, not found "{s}" on list:{vlist}')
#
# str util
#
@ -328,7 +358,7 @@ class TBase:
# taosBenchmark
#
# run taosBenchmark and check insert Result
# insert
def insertBenchJson(self, jsonFile, options="", checkStep=False):
# exe insert
cmd = f"{options} -f {jsonFile}"
@ -395,4 +425,18 @@ class TBase:
if vgroups != None:
tdSql.checkData(0, 0, vgroups)
return db, stb,child_count, insert_rows
return db, stb, child_count, insert_rows
# tmq
def tmqBenchJson(self, jsonFile, options="", checkStep=False):
# exe insert
command = f"{options} -f {jsonFile}"
rlist = frame.etool.runBinFile("taosBenchmark", command, checkRun = True)
#
# check insert result
#
print(rlist)
return rlist

64
tests/army/frame/eos.py
View File

@ -52,8 +52,11 @@ def isArm64Cpu():
#
# wait util execute file finished
def exe(file):
return os.system(file)
def exe(command, show = False):
code = os.system(command)
if show:
print(f"eos.exe retcode={code} command:{command}")
return code
# execute file and return immediately
def exeNoWait(file):
@ -64,20 +67,38 @@ def exeNoWait(file):
return exe(cmd)
# run return output and error
def run(command, timeout = 10):
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait(timeout)
def run(command, show = True):
# out to file
id = time.clock_gettime_ns(time.CLOCK_REALTIME) % 100000
out = f"out_{id}.txt"
err = f"err_{id}.txt"
output = process.stdout.read().decode(encoding="gbk")
error = process.stderr.read().decode(encoding="gbk")
code = exe(command + f" 1>{out} 2>{err}", show)
return output, error
# read from file
output = readFileContext(out)
error = readFileContext(err)
# del
if os.path.exists(out):
os.remove(out)
if os.path.exists(err):
os.remove(err)
return output, error, code
# return list after run
def runRetList(command, timeout=10):
output,error = run(command, timeout)
return output.splitlines()
def runRetList(command, show = True, checkRun = False, retFail = False):
output, error, code = run(command, show)
if checkRun and code != 0:
print(f"eos.runRetList checkRun return code failed. code={code} error={error}")
assert code == 0
rList = output.splitlines()
if retFail and error != "":
rList += error.splitlines()
return rList
#
# file
@ -85,3 +106,24 @@ def runRetList(command, timeout=10):
def delFile(file):
return exe(f"rm -rf {file}")
def readFileContext(filename):
file = open(filename)
context = file.read()
file.close()
return context
def writeFileContext(filename, context):
file = open(filename, "w")
file.write(context)
file.close()
def appendFileContext(filename, context):
global resultContext
resultContext += context
try:
file = open(filename, "a")
wsize = file.write(context)
file.close()
except:
print(f"appand file error context={context} .")

6
tests/army/frame/etool.py
View File

@ -51,7 +51,7 @@ def benchMark(command = "", json = "") :
# run
if command != "":
frame.eos.exe(bmFile + " " + command)
status = frame.eos.run(bmFile + " " + command)
if json != "":
cmd = f"{bmFile} -f {json}"
print(cmd)
@ -66,7 +66,7 @@ def curFile(fullPath, filename):
# run build/bin file
def runBinFile(fname, command, show=True):
def runBinFile(fname, command, show = True, checkRun = False, retFail = False ):
binFile = frame.epath.binFile(fname)
if frame.eos.isWin():
binFile += ".exe"
@ -74,7 +74,7 @@ def runBinFile(fname, command, show=True):
cmd = f"{binFile} {command}"
if show:
tdLog.info(cmd)
return frame.eos.runRetList(cmd)
return frame.eos.runRetList(cmd, show, checkRun, retFail)
# exe build/bin file
def exeBinFile(fname, command, wait=True, show=True):

71
tests/army/tools/benchmark/basic/json/tmqBasicInsert.json Normal file
View File

@ -0,0 +1,71 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count": 2,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes",
"vgroups": 4
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 10,
"insert_rows": 100,
"timestamp_step": 1,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]
}
]
}

29
tests/army/tools/benchmark/basic/json/tmqBasicPara.json Normal file
View File

@ -0,0 +1,29 @@
{
"filetype": "subscribe",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"databases": "test",
"confirm_parameter_prompt": "no",
"result_file": "./tmq_result_para.txt",
"tmq_info": {
"concurrent": 2,
"poll_delay": 3000,
"create_mode": "parallel",
"group_mode": "independent",
"client.id": "clientId",
"auto.offset.reset": "earliest",
"enable.auto.commit": "true",
"auto.commit.interval.ms": 1000,
"enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "false",
"rows_file": "./rows_file_para",
"expect_rows": 100,
"topic_list": [
{"name": "topic_benchmark_d0", "sql": "select * from test.d0;"}
]
}
}

13
tests/army/tools/benchmark/basic/json/tmq.json → tests/army/tools/benchmark/basic/json/tmqBasicSequ.json
View File

@ -5,12 +5,13 @@
"port": 6030,
"user": "root",
"password": "taosdata",
"databases": "db",
"databases": "test",
"confirm_parameter_prompt": "no",
"result_file": "./tmq_result1.txt",
"result_file": "./tmq_result_sequ.txt",
"tmq_info": {
"concurrent": 2,
"concurrent": 4,
"poll_delay": 3000,
"group_mode": "independent",
"group.id": "grpId_0",
"client.id": "clientId",
"auto.offset.reset": "earliest",
@ -19,10 +20,10 @@
"enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "false",
"rows_file": "./consumed_rows1",
"expect_rows": 50,
"rows_file": "./rows_file_sequ",
"expect_rows": 1000,
"topic_list": [
{"name": "tmq_topic_0", "sql": "select c0 from db.stb;"}
{"name": "topic_benchmark_meters", "sql": "select * from test.meters;"}
]
}
}

35
tests/army/tools/benchmark/basic/default_tmq_json.py → tests/army/tools/benchmark/basic/tmqBasic.py
View File

@ -25,37 +25,22 @@ from frame import *
class TDTestCase(TBase):
def caseDescription(self):
"""
[TD-11510] taosBenchmark test cases
taosBenchmark tmp->Basic test cases
"""
def run(self):
tdSql.execute("drop topic if exists topic_0")
binPath = etool.benchMarkFile()
cmd = "%s -f ./tools/benchmark/basic/json/default.json" % binPath
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("reset query cache")
tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000")
# insert data
json = "tools/benchmark/basic/json/tmqBasicInsert.json"
db, stb, child_count, insert_rows = self.insertBenchJson(json, checkStep = True)
cmd = "%s -f ./tools/benchmark/basic/json/tmq.json " % binPath
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
sleep(15)
# tmq Sequ
json = "tools/benchmark/basic/json/tmqBasicSequ.json"
self.tmqBenchJson(json)
# try:
# for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"):
# fields = line.split()
# pid = fields[0]
# os.kill(int(pid), signal.SIGINT)
# time.sleep(3)
# print("taosBenchmark be killed on purpose")
# except:
# tdLog.exit("failed to kill taosBenchmark")
# tmq Parallel
json = "tools/benchmark/basic/json/tmqBasicPara.json"
self.tmqBenchJson(json)
def stop(self):
tdSql.close()

66
tests/army/tools/taosdump/native/json/insertFullType.json Normal file
View File

@ -0,0 +1,66 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8}
]
}
]
}
]
}

70
tests/army/tools/taosdump/native/json/insertOther.json Normal file
View File

@ -0,0 +1,70 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"testother",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]
}
]
}

132
tests/army/tools/taosdump/native/taosdumpCommandline.py
View File

@ -14,6 +14,7 @@
import os
import json
import frame
import frame.eos
import frame.etool
from frame.log import *
from frame.cases import *
@ -28,23 +29,19 @@ class TDTestCase(TBase):
test taosdump support commandline arguments
"""
def exec(self, command):
tdLog.info(command)
return os.system(command)
def clearPath(self, path):
os.system("rm -rf %s/*" % path)
def findPrograme(self):
# taosdump
taosdump = etool.taosDumpFile()
taosdump = frame.etool.taosDumpFile()
if taosdump == "":
tdLog.exit("taosdump not found!")
else:
tdLog.info("taosdump found in %s" % taosdump)
# taosBenchmark
benchmark = etool.benchMarkFile()
benchmark = frame.etool.benchMarkFile()
if benchmark == "":
tdLog.exit("benchmark not found!")
else:
@ -60,7 +57,7 @@ class TDTestCase(TBase):
return taosdump, benchmark,tmpdir
def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval=False):
def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval = True):
#
# check insert result
#
@ -91,15 +88,9 @@ class TDTestCase(TBase):
tdSql.query(sql)
tdSql.checkRows(0)
def testBenchmarkJson(self, benchmark, jsonFile, options="", checkInterval=False):
# exe insert
cmd = f"{benchmark} {options} -f {jsonFile}"
self.exec(cmd)
self.checkCorrectWithJson(jsonFile)
def insertData(self, benchmark, json, db):
def insertData(self, json):
# insert super table
self.testBenchmarkJson(benchmark, json)
db, stb, child_count, insert_rows = self.insertBenchJson(json)
# normal table
sqls = [
@ -113,13 +104,7 @@ class TDTestCase(TBase):
for sql in sqls:
tdSql.execute(sql)
def dumpOut(self, taosdump, db , outdir):
# dump out
self.exec(f"{taosdump} -D {db} -o {outdir}")
def dumpIn(self, taosdump, db, newdb, indir):
# dump in
self.exec(f'{taosdump} -W "{db}={newdb}" -i {indir}')
return db, stb, child_count, insert_rows
def checkSame(self, db, newdb, stb, aggfun):
# sum pk db
@ -136,31 +121,75 @@ class TDTestCase(TBase):
else:
tdLog.exit(f"{aggfun} source db:{sum1} import db:{sum2} not equal.")
def verifyResult(self, db, newdb, json):
# compare with insert json
self.checkCorrectWithJson(json, newdb)
# compare sum(pk)
stb = "meters"
self.checkSame(db, newdb, stb, "sum(pk)")
self.checkSame(db, newdb, stb, "sum(usi)")
self.checkSame(db, newdb, stb, "sum(fc)")
self.checkSame(db, newdb, stb, "sum(ti)")
self.checkSame(db, newdb, stb, "sum(si)")
self.checkSame(db, newdb, stb, "sum(ic)")
self.checkSame(db, newdb, stb, "avg(bi)")
self.checkSame(db, newdb, stb, "sum(uti)")
self.checkSame(db, newdb, stb, "sum(usi)")
self.checkSame(db, newdb, stb, "sum(ui)")
self.checkSame(db, newdb, stb, "avg(ubi)")
# check normal table
self.checkSame(db, newdb, "ntb", "sum(c1)")
# with Native Rest and WebSocket
def dumpInOutMode(self, mode, db, json, tmpdir):
# dump out
self.clearPath(tmpdir)
self.taosdump(f"{mode} -D {db} -o {tmpdir}")
# dump in
newdb = "new" + db
self.taosdump(f"{mode} -W '{db}={newdb}' -i {tmpdir}")
# check same
self.verifyResult(db, newdb, json)
# basic commandline
def basicCommandLine(self, taosdump, tmpdir):
# -h -P -u -p -o
self.exec(taosdump + f" -h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}")
def basicCommandLine(self, tmpdir):
#command and check result
checkItems = [
[f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped"]],
[f"-r result -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]],
[f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]],
[f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]],
[f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"]],
[f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]],
[f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]],
[f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"]],
[f"--help", ["Report bugs to"]],
[f"-?", ["Report bugs to"]],
[f"-V", ["version:"]],
[f"--usage", ["taosdump [OPTION...] -o outpath"]]
]
# executes
for item in checkItems:
self.clearPath(tmpdir)
command = item[0]
results = item[1]
rlist = self.taosdump(command)
for result in results:
self.checkListString(rlist, result)
# clear tmp
# check except
def checkExcept(self, command):
try:
code = self.exec(command)
code = frame.eos.exe(command, show = True)
if code == 0:
tdLog.exit(f"Failed, not report error cmd:{command}")
else:
@ -170,7 +199,7 @@ class TDTestCase(TBase):
# except commandline
def exceptCommandLine(self, taosdump, tmpdir):
def exceptCommandLine(self, taosdump, db, stb, tmpdir):
# -o
self.checkExcept(taosdump + " -o= ")
self.checkExcept(taosdump + " -o")
@ -178,34 +207,47 @@ class TDTestCase(TBase):
self.checkExcept(taosdump + " -A -o ")
self.checkExcept(taosdump + " -A -o ./noexistpath/")
self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}")
self.checkExcept(taosdump + f" -d unknown -o {tmpdir}")
self.checkExcept(taosdump + f" -P invalidport")
self.checkExcept(taosdump + f" -D")
self.checkExcept(taosdump + f" -P 65536")
self.checkExcept(taosdump + f" -t 2 -k 2 -z 1 -C https://not-exist.com:80/cloud -D test -o {tmpdir}")
self.checkExcept(taosdump + f" -P 65536")
# run
def run(self):
# database
db = "pridb"
newdb = "npridb"
# find
taosdump, benchmark, tmpdir = self.findPrograme()
json = "./tools/taosdump/ws/json/primaryKey.json"
json = "./tools/taosdump/native/json/insertFullType.json"
# insert data with taosBenchmark
self.insertData(benchmark, json, db)
db, stb, childCount, insertRows = self.insertData(json)
# dumpInOut
modes = ["", "-R" , "--cloud=http://localhost:6041"]
for mode in modes:
self.dumpInOutMode(mode, db , json, tmpdir)
tdLog.info("1. native rest ws dumpIn Out .......................... [Passed]")
# basic commandline
self.basicCommandLine(taosdump, tmpdir)
self.basicCommandLine(tmpdir)
tdLog.info("2. basic command line .................................. [Passed]")
# except commandline
self.exceptCommandLine(taosdump, tmpdir)
self.exceptCommandLine(taosdump, db, stb, tmpdir)
tdLog.info("3. except command line ................................. [Passed]")
# dump out
#self.dumpOut(taosdump, db, tmpdir)
# dump in
#self.dumpIn(taosdump, db, newdb, tmpdir)
# verify db
#self.verifyResult(db, newdb, json)
#
# varbinary and geometry for native
#
json = "./tools/taosdump/native/json/insertOther.json"
# insert
db, stb, childCount, insertRows = self.insertData(json)
# dump in/out
self.dumpInOutMode("", db , json, tmpdir)
tdLog.info("4. native varbinary geometry ........................... [Passed]")
def stop(self):

7
tests/parallel_test/cases.task
View File

@ -70,7 +70,7 @@
,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py
,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py
,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py
,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py -B
,,n,army,python3 ./test.py -f whole/checkErrorCode.py
#
@ -88,7 +88,6 @@
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/custom_col_tag.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py
@ -140,6 +139,8 @@
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmqBasic.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/cloud/cloud-test.py
@ -206,7 +207,7 @@
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeBool.py -B
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpRetry.py -B
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeTinyInt.py -B
,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py
,,n,army,python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py -B
#
# system test

13
tests/system-test/2-query/test_ts4467.py
View File

@ -55,6 +55,19 @@ class TDTestCase:
tdLog.debug("res2: %s" % str(res2))
assert(len(res1) == len(res2) and res1[0][0] == res2[4][0])
# test ts-5613
sql = "select cast(2<>3 as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,1)
sql = "select cast(2 not in(3) as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,1)
sql = "select cast(2 is NULL as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,0)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

1
tools/shell/src/shellAuto.c
View File

@ -112,6 +112,7 @@ SWords shellCommands[] = {
{"create view <anyword> as select", 0, 0, NULL},
{"compact database <db_name>", 0, 0, NULL},
#endif
{"desc <all_table>;", 0, 0, NULL},
{"describe <all_table>;", 0, 0, NULL},
{"delete from <all_table> where ", 0, 0, NULL},
{"drop database <db_name>;", 0, 0, NULL},

3
tools/shell/src/shellEngine.c
View File

@ -254,8 +254,7 @@ void shellRunSingleCommandImp(char *command) {
}
if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) {
fprintf(stdout, "Database changed.\r\n\r\n");
fflush(stdout);
printf("Database changed.\r\n\r\n");
// call back auto tab module
callbackAutoTab(command, pSql, true);

6
tools/shell/src/shellUtil.c
View File

@ -122,6 +122,7 @@ void shellCheckServerStatus() {
} while (1);
}
#ifdef WEBSOCKET
char dsn[1024] = "ws://localhost:6041";
void shellCheckConnectMode() {
if (shell.args.dsn) {
shell.args.cloud = true;
@ -144,6 +145,7 @@ void shellCheckConnectMode() {
shell.args.restful = false;
return;
}
}
if (shell.args.restful) {
if (!shell.args.host) {
@ -152,13 +154,13 @@ void shellCheckConnectMode() {
if (!shell.args.port) {
shell.args.port = 6041;
}
shell.args.dsn = taosMemoryCalloc(1, 1024);
shell.args.dsn = dsn;
snprintf(shell.args.dsn, 1024, "ws://%s:%d",
shell.args.host, shell.args.port);
}
shell.args.cloud = false;
return;
}
}
#endif

4
tools/taos-tools/inc/bench.h
View File

@ -617,10 +617,8 @@ typedef struct SDataBase_S {
int durMinute; // duration minutes
BArray *cfgs;
BArray *superTbls;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
int32_t vgroups;
BArray *vgArray;
#endif // TD_VER_COMPATIBLE_3_0_0_0
bool flush;
} SDataBase;
@ -838,9 +836,7 @@ typedef struct SThreadInfo_S {
BArray* delayList;
uint64_t *query_delay_list;
double avg_delay;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
SVGroup *vg;
#endif
int posOfTblCreatingBatch;
int posOfTblCreatingInterval;

24
tools/taos-tools/src/CMakeLists.txt
View File

@ -78,7 +78,6 @@ ADD_DEFINITIONS(-DBUILD_INFO="${BUILD_INFO}")
MESSAGE(STATUS "build:${BUILD_INFO}")
MESSAGE("")
#
# build proj
#
@ -88,15 +87,8 @@ INCLUDE_DIRECTORIES(/usr/local/taos/include)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/avro/lang/c/src)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/toolscJson/src)
if(NOT DEFINED TD_VER_COMPATIBLE)
SET(TD_VER_COMPATIBLE "3.0.0.0")
ENDIF()
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread)
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex)
ADD_DEFINITIONS(-DTD_VER_COMPATIBLE_3_0_0_0)
ENDIF ()
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread)
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex)
MESSAGE("Current system is: ${CMAKE_SYSTEM_NAME}")
@ -128,22 +120,14 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
ADD_DEPENDENCIES(deps-snappy apache-avro)
ADD_DEPENDENCIES(taosdump deps-jansson)
ADD_DEPENDENCIES(taosdump deps-snappy)
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ELSE ()
INCLUDE_DIRECTORIES(/usr/local/include)
ADD_DEFINITIONS(-DDARWIN)
LINK_DIRECTORIES(/usr/local/lib)
SET(OS_ID "Darwin")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ENDIF ()
# websocket
@ -337,11 +321,7 @@ ELSE ()
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ELSE ()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF ()
ADD_EXECUTABLE(taosdump taosdump.c dumpUtil.c wsdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c)
ADD_DEPENDENCIES(apache-avro tools-zlib)

3
tools/taos-tools/src/benchCommandOpt.c
View File

@ -245,9 +245,8 @@ void initArgument() {
g_arguments->trying_interval = 0;
g_arguments->iface = TAOSC_IFACE;
g_arguments->rest_server_ver_major = -1;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
g_arguments->inputted_vgroups = -1;
#endif
g_arguments->mistMode = false;
initDatabase();

11
tools/taos-tools/src/benchData.c
View File

@ -812,17 +812,6 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
continue;
}
}
if (field->type == TSDB_DATA_TYPE_TIMESTAMP && !tag) {
n = snprintf(sampleDataBuf + pos, bufLen - pos, "now,");
if (n < 0 || n >= bufLen - pos) {
errorPrint("%s() LN%d snprintf overflow\n",
__func__, __LINE__);
return -1;
} else {
pos += n;
continue;
}
}
switch (field->type) {
case TSDB_DATA_TYPE_BOOL: {
bool boolTmp = tmpBool(field);

2
tools/taos-tools/src/benchJsonOpt.c
View File

@ -2254,7 +2254,6 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) {
return 0;
}
#ifdef TD_VER_COMPATIBLE_3_0_0_0
static int getMetaFromTmqJsonFile(tools_cJSON *json) {
int32_t code = -1;
@ -2422,7 +2421,6 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) {
TMQ_PARSE_OVER:
return code;
}
#endif
int readJsonConfig(char * file) {
int32_t code = -1;

458
tools/taos-tools/src/benchSubscribe.c
View File

@ -1,458 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the MIT license as published by the Free Software
* Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*/
#include <bench.h>
#include "benchLog.h"
extern int g_majorVersionOfClient;
static void stable_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param,
int code) {
if (res == NULL || taos_errno(res) != 0) {
errorPrint("failed to subscribe result, code:%d, reason:%s\n",
code, taos_errstr(res));
return;
}
if (param) fetchResult(res, ((threadInfo *)param)->filePath);
// tao_unsubscribe() will free result.
}
static void specified_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param,
int code) {
if (res == NULL || taos_errno(res) != 0) {
errorPrint("failed to subscribe result, code:%d, reason:%s\n",
code, taos_errstr(res));
return;
}
if (param) fetchResult(res, ((threadInfo *)param)->filePath);
// tao_unsubscribe() will free result.
}
static TAOS_SUB *subscribeImpl(QUERY_CLASS class, threadInfo *pThreadInfo,
char *sql, char *topic, bool restart,
uint64_t interval) {
TAOS_SUB *tsub = NULL;
if (taos_select_db(pThreadInfo->conn->taos, g_queryInfo.dbName)) {
errorPrint("failed to select database(%s)\n", g_queryInfo.dbName);
return NULL;
}
if ((SPECIFIED_CLASS == class) &&
(ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
tsub = taos_subscribe(
pThreadInfo->conn->taos, restart,
topic, sql, specified_sub_callback,
(void *)pThreadInfo,
(int)g_queryInfo.specifiedQueryInfo.subscribeInterval);
} else if ((STABLE_CLASS == class) &&
(ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
tsub = taos_subscribe(pThreadInfo->conn->taos, restart, topic, sql,
stable_sub_callback, (void *)pThreadInfo,
(int)g_queryInfo.superQueryInfo.subscribeInterval);
} else {
tsub = taos_subscribe(pThreadInfo->conn->taos,
restart, topic, sql, NULL,
NULL, (int)interval);
}
if (tsub == NULL) {
errorPrint("failed to create subscription. topic:%s, sql:%s\n",
topic, sql);
return NULL;
}
return tsub;
}
static void *specifiedSubscribe(void *sarg) {
int32_t *code = benchCalloc(1, sizeof(int32_t), false);
*code = -1;
threadInfo *pThreadInfo = (threadInfo *)sarg;
#ifdef LINUX
prctl(PR_SET_NAME, "specSub");
#endif
sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
"taosbenchmark-subscribe-%" PRIu64 "-%d", pThreadInfo->querySeq,
pThreadInfo->threadID);
SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls,
pThreadInfo->querySeq);
if (sql->result[0] !=
'\0') {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN,
"%s-%d", sql->result, pThreadInfo->threadID);
}
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
SPECIFIED_CLASS, pThreadInfo, sql->command,
g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeRestart,
g_queryInfo.specifiedQueryInfo.subscribeInterval);
if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
goto free_of_specified_subscribe;
}
// start loop to consume result
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
while ((g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq] == -1) ||
(g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq])) {
infoPrint(
"consumed[%d]: %d, endAfterConsum[%" PRId64 "]: %d\n",
pThreadInfo->threadID,
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID],
pThreadInfo->querySeq,
g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq]);
g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] =
taos_consume(
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
if (sql->result[0] != 0) {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN,
"%s-%d", sql->result, pThreadInfo->threadID);
}
fetchResult(
g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID],
pThreadInfo->filePath);
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID]++;
if ((g_queryInfo.specifiedQueryInfo
.resubAfterConsume[pThreadInfo->querySeq] != -1) &&
(g_queryInfo.specifiedQueryInfo
.consumed[pThreadInfo->threadID] >=
g_queryInfo.specifiedQueryInfo
.resubAfterConsume[pThreadInfo->querySeq])) {
infoPrint(
"keepProgress:%d, resub specified query: %" PRIu64
"\n",
g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
pThreadInfo->querySeq);
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] =
0;
taos_unsubscribe(
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
g_queryInfo.specifiedQueryInfo
.tsub[pThreadInfo->threadID] = subscribeImpl(
SPECIFIED_CLASS, pThreadInfo, sql->command,
g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeRestart,
g_queryInfo.specifiedQueryInfo.subscribeInterval);
if (NULL == g_queryInfo.specifiedQueryInfo
.tsub[pThreadInfo->threadID]) {
goto free_of_specified_subscribe;
}
}
}
}
*code = 0;
taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
free_of_specified_subscribe:
return code;
}
static void *superSubscribe(void *sarg) {
int32_t *code = benchCalloc(1, sizeof(int32_t), false);
*code = -1;
threadInfo *pThreadInfo = (threadInfo *)sarg;
TAOS_SUB * tsub[MAX_QUERY_SQL_COUNT] = {0};
uint64_t tsubSeq;
char * subSqlStr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false);
#ifdef LINUX
prctl(PR_SET_NAME, "superSub");
#endif
if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
errorPrint(
"The table number(%" PRId64
") of the thread is more than max query sql count: %d\n",
pThreadInfo->ntables, MAX_QUERY_SQL_COUNT);
goto free_of_super_subscribe;
}
char topic[32] = {0};
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
sprintf(topic, "taosbenchmark-subscribe-%" PRIu64 "-%" PRIu64 "", i,
pThreadInfo->querySeq);
memset(subSqlStr, 0, TSDB_MAX_ALLOWED_SQL_LEN);
replaceChildTblName(
g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlStr,
(int)i);
if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d",
g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
pThreadInfo->threadID);
}
tsub[tsubSeq] =
subscribeImpl(STABLE_CLASS, pThreadInfo, subSqlStr, topic,
g_queryInfo.superQueryInfo.subscribeRestart,
g_queryInfo.superQueryInfo.subscribeInterval);
if (NULL == tsub[tsubSeq]) {
goto free_of_super_subscribe;
}
}
// start loop to consume result
int consumed[MAX_QUERY_SQL_COUNT];
for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
consumed[i] = 0;
}
TAOS_RES *res = NULL;
uint64_t st = 0, et = 0;
while (
(g_queryInfo.superQueryInfo.endAfterConsume == -1) ||
(g_queryInfo.superQueryInfo.endAfterConsume >
consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) {
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
continue;
}
st = toolsGetTimestampMs();
perfPrint(
"st: %" PRIu64 " et: %" PRIu64 " st-et: %" PRIu64 "\n",
st, et, (st - et));
res = taos_consume(tsub[tsubSeq]);
et = toolsGetTimestampMs();
perfPrint(
"st: %" PRIu64 " et: %" PRIu64 " delta: %" PRIu64 "\n",
st, et, (et - st));
if (res) {
if (g_queryInfo.superQueryInfo
.result[pThreadInfo->querySeq][0] != 0) {
snprintf(pThreadInfo->filePath,
MAX_PATH_LEN,
"%s-%d",
g_queryInfo.superQueryInfo
.result[pThreadInfo->querySeq],
pThreadInfo->threadID);
}
fetchResult(res, pThreadInfo->filePath);
consumed[tsubSeq]++;
if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) &&
(consumed[tsubSeq] >=
g_queryInfo.superQueryInfo.resubAfterConsume)) {
taos_unsubscribe(
tsub[tsubSeq],
g_queryInfo.superQueryInfo.subscribeKeepProgress);
consumed[tsubSeq] = 0;
tsub[tsubSeq] = subscribeImpl(
STABLE_CLASS, pThreadInfo, subSqlStr, topic,
g_queryInfo.superQueryInfo.subscribeRestart,
g_queryInfo.superQueryInfo.subscribeInterval);
if (NULL == tsub[tsubSeq]) {
goto free_of_super_subscribe;
}
}
}
}
}
taos_free_result(res);
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
taos_unsubscribe(tsub[tsubSeq], 0);
}
*code = 0;
free_of_super_subscribe:
tmfree(subSqlStr);
return code;
}
int subscribeTestProcess() {
prompt(0);
if (REST_IFACE == g_queryInfo.iface) {
encodeAuthBase64();
}
if (0 != g_queryInfo.superQueryInfo.sqlCount) {
SBenchConn* conn = initBenchConn();
if (conn == NULL) {
return -1;
}
char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0";
if (3 == g_majorVersionOfClient) {
snprintf(cmd, SHORT_1K_SQL_BUFF_LEN,
"SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)",
g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName);
} else {
snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(TBNAME) FROM %s.%s",
g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName);
}
TAOS_RES *res = taos_query(conn->taos, cmd);
int32_t code = taos_errno(res);
if (code) {
printErrCmdCodeStr(cmd, code, res);
return -1;
}
TAOS_ROW row = NULL;
int num_fields = taos_num_fields(res);
TAOS_FIELD *fields = taos_fetch_fields(res);
while ((row = taos_fetch_row(res)) != NULL) {
if (0 == strlen((char *)(row[0]))) {
errorPrint("stable %s have no child table\n",
g_queryInfo.superQueryInfo.stbName);
return -1;
}
char temp[256] = {0};
taos_print_row(temp, row, fields, num_fields);
g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp);
}
infoPrint("%s's childTblCount: %" PRId64 "\n",
g_queryInfo.superQueryInfo.stbName,
g_queryInfo.superQueryInfo.childTblCount);
taos_free_result(res);
g_queryInfo.superQueryInfo.childTblName =
benchCalloc(g_queryInfo.superQueryInfo.childTblCount,
sizeof(char *), false);
if (getAllChildNameOfSuperTable(
conn->taos, g_queryInfo.dbName,
g_queryInfo.superQueryInfo.stbName,
g_queryInfo.superQueryInfo.childTblName,
g_queryInfo.superQueryInfo.childTblCount)) {
closeBenchConn(conn);
return -1;
}
closeBenchConn(conn);
}
pthread_t * pids = NULL;
threadInfo *infos = NULL;
pthread_t * pidsOfStable = NULL;
threadInfo *infosOfStable = NULL;
//==== create threads for query for specified table
int sqlCount = g_queryInfo.specifiedQueryInfo.sqls->size;
if (sqlCount > 0) {
pids = benchCalloc(1, sqlCount
*g_queryInfo.specifiedQueryInfo.concurrent
*sizeof(pthread_t), false);
infos = benchCalloc(1, sqlCount
*g_queryInfo.specifiedQueryInfo.concurrent
*sizeof(threadInfo), false);
for (int i = 0; i < sqlCount; i++) {
for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent;
j++) {
uint64_t seq =
i * g_queryInfo.specifiedQueryInfo.concurrent + j;
threadInfo *pThreadInfo = infos + seq;
pThreadInfo->threadID = (int)seq;
pThreadInfo->querySeq = i;
pThreadInfo->conn = initBenchConn();
pthread_create(pids + seq, NULL, specifiedSubscribe,
pThreadInfo);
}
}
for (int i = 0; i < sqlCount; i++) {
for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent;
j++) {
uint64_t seq =
i * g_queryInfo.specifiedQueryInfo.concurrent + j;
threadInfo *pThreadInfo = infos + seq;
void *result;
pthread_join(pids[seq], &result);
if (*(int32_t *)result) {
g_fail = true;
}
closeBenchConn(pThreadInfo->conn);
tmfree(result);
}
}
}
//==== create threads for super table query
if (g_queryInfo.superQueryInfo.sqlCount > 0 &&
g_queryInfo.superQueryInfo.threadCnt > 0) {
pidsOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount *
g_queryInfo.superQueryInfo.threadCnt *
sizeof(pthread_t), false);
infosOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount *
g_queryInfo.superQueryInfo.threadCnt *
sizeof(threadInfo), false);
int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
int threads = g_queryInfo.superQueryInfo.threadCnt;
int64_t a = ntables / threads;
if (a < 1) {
threads = (int)ntables;
a = 1;
}
int64_t b = 0;
if (threads != 0) {
b = ntables % threads;
}
for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
uint64_t tableFrom = 0;
for (int j = 0; j < threads; j++) {
uint64_t seq = i * threads + j;
threadInfo *pThreadInfo = infosOfStable + seq;
pThreadInfo->threadID = (int)seq;
pThreadInfo->querySeq = i;
pThreadInfo->start_table_from = tableFrom;
pThreadInfo->ntables = j < b ? a + 1 : a;
pThreadInfo->end_table_to =
j < b ? tableFrom + a : tableFrom + a - 1;
tableFrom = pThreadInfo->end_table_to + 1;
pThreadInfo->conn = initBenchConn();
pthread_create(pidsOfStable + seq, NULL, superSubscribe,
pThreadInfo);
}
}
g_queryInfo.superQueryInfo.threadCnt = threads;
for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
for (int j = 0; j < threads; j++) {
uint64_t seq = (uint64_t)i * threads + j;
threadInfo *pThreadInfo = infosOfStable + seq;
void * result;
pthread_join(pidsOfStable[seq], &result);
if (*(int32_t *)result) {
g_fail = true;
}
closeBenchConn(pThreadInfo->conn);
tmfree(result);
}
}
}
tmfree((char *)pids);
tmfree((char *)infos);
tmfree((char *)pidsOfStable);
tmfree((char *)infosOfStable);
//
if (g_fail) {
return -1;
}
return 0;
}

6
tools/taos-tools/src/benchSys.c
View File

@ -77,9 +77,7 @@ void benchPrintHelp() {
printf("%s%s%s%s\r\n", indent, "-W,", indent, BENCH_DSN);
printf("%s%s%s%s\r\n", indent, "-D,", indent, BENCH_TIMEOUT);
#endif
#ifdef TD_VER_COMPATIBLE_3_0_0_0
printf("%s%s%s%s\r\n", indent, "-v,", indent, BENCH_VGROUPS);
#endif
printf("%s%s%s%s\r\n", indent, "-V,", indent, BENCH_VERSION);
printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL);
}
@ -199,9 +197,7 @@ static struct argp_option bench_options[] = {
#endif
{"keep-trying", 'k', "NUMBER", 0, BENCH_KEEPTRYING},
{"trying-interval", 'z', "NUMBER", 0, BENCH_TRYING_INTERVAL},
#ifdef TD_VER_COMPATIBLE_3_0_0_0
{"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS},
#endif
{"version", 'V', 0, 0, BENCH_VERSION},
{"nodrop", 'Q', 0, 0, BENCH_NODROP},
{0}
@ -642,14 +638,12 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) {
g_arguments->timeout = atoi(arg);
break;
#endif
#ifdef TD_VER_COMPATIBLE_3_0_0_0
case 'v':
if (!toolsIsStringNumber(arg)) {
errorPrintReqArg2(CUS_PROMPT"Benchmark", "v");
}
g_arguments->inputted_vgroups = atoi(arg);
break;
#endif
case 'Q':
database->drop = false;
g_argFlag |= ARG_OPT_NODROP;

20
tools/taos-tools/src/taosdump.c
View File

@ -5831,26 +5831,6 @@ static int64_t dumpInAvroDataImpl(
#endif
}
} // tbName
#ifndef TD_VER_COMPATIBLE_3_0_0_0
else {
// 2.6 need call taos_stmt_set_tbname every loop
const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3;
char *escapedTbName = calloc(1, escapedTbNameLen);
snprintf(escapedTbName, escapedTbNameLen, "%s%s%s",
g_escapeChar, tbName, g_escapeChar);
if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) {
errorPrint("Failed to execute taos_stmt_set_tbname(%s)."
"reason: %s\n",
escapedTbName, taos_stmt_errstr(stmt));
free(escapedTbName);
freeTbNameIfLooseMode(tbName);
tbName = NULL;
continue;
}
free(escapedTbName);
}
#endif
debugPrint("%s() LN%d, count: %"PRId64"\n",
__func__, __LINE__, count);