xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into merge/mainto3.0

This commit is contained in:
Simon Guan 2025-02-28 09:11:14 +08:00
parent f5429637fa 5d777f9bcf
commit ed2d3b258d
85 changed files with 2802 additions and 1385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/14-reference/03-taos-sql/20-keywords.md
View File

@ -37,6 +37,7 @@ The list of keywords is as follows:
| ASOF | |
| AT_ONCE | |
| ATTACH | |
| ASSIGN | Version 3.3.6.0 and later |
### B

2
docs/en/14-reference/09-error-code.md
View File

@ -460,7 +460,7 @@ This document details the server error codes that may be encountered when using
| 0x80002667 | Invalid usage of expr: %s | Illegal expression | Check and correct the SQL statement |
| 0x80002687 | True_for duration cannot be negative | Use negative value as true_for duration | Check and correct the SQL statement |
| 0x80002688 | Cannot use 'year' or 'month' as true_for duration | Use year or month as true_for_duration | Check and correct the SQL statement |
| 0x80002680 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement |
| 0x80002689 | Invalid using cols function | Illegal using cols function | Check and correct the SQL statement |
| 0x8000268A | Cols function's first param must be a select function that output a single row | The first parameter of the cols function should be a selection function | Check and correct the SQL statement |
| 0x8000268B | Invalid using cols function with multiple output columns | Illegal using the cols function for multiple column output | Check and correct the SQL statement |
| 0x8000268C | Invalid using alias for cols function | Illegal cols function alias | Check and correct the SQL statement |

4
docs/zh/08-operation/18-ha/02-replica2.md
View File

@ -69,9 +69,13 @@ alter database <dbname> replica 2|1
| ------- | ------ |
| 没有 Vnode 发生故障： Arbitrator 故障（Mnode 宕机节点超过一个，导致 Mnode 无法选主）| **持续提供服务** |
| 仅一个 Vnode 故障：VGroup 已经达成同步后，某一个 Vnode 才发生故障的 | **持续提供服务** |
| 仅一个 Vnode 故障：2个 Vnode 同时故障，故障前 VGroup 达成同步，但是只有一个 Vnode 从故障中恢复服务，另一个 Vnode 服务故障 | **通过下面的命令，强制指定leader, 继续提供服务** |
| 仅一个 Vnode 故障：离线 Vnode 启动后，VGroup 未达成同步前，另一个 Vnode 服务故障的 | **无法提供服务** |
| 两个 Vnode 都发生故障 | **无法提供服务** |
```sql
ASSIGN LEADER FORCE;
```
## 常见问题

1
docs/zh/14-reference/03-taos-sql/20-keywords.md
View File

@ -38,6 +38,7 @@ description: TDengine 保留关键字的详细列表
| AT_ONCE | |
| ATTACH | |
| AUTO | 3.3.5.0 及后续版本 |
| ASSIGN | 3.3.6.0 及后续版本 |
### B
|关键字|说明|

1
include/common/tgrant.h
View File

@ -61,6 +61,7 @@ typedef enum {
TSDB_GRANT_ACTIVE_ACTIVE,
TSDB_GRANT_DUAL_REPLICA_HA,
TSDB_GRANT_DB_ENCRYPTION,
TSDB_GRANT_TD_GPT,
} EGrantType;
int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);

11
include/common/tmsg.h
View File

@ -359,6 +359,7 @@ typedef enum ENodeType {
QUERY_NODE_CREATE_ANODE_STMT,
QUERY_NODE_DROP_ANODE_STMT,
QUERY_NODE_UPDATE_ANODE_STMT,
QUERY_NODE_ASSIGN_LEADER_STMT,
// show statement nodes
// see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
@ -2583,6 +2584,7 @@ typedef struct {
char* arbToken;
int64_t arbTerm;
char* memberToken;
int8_t force;
} SVArbSetAssignedLeaderReq;
int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
@ -2661,6 +2663,15 @@ int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq
int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq);
void tFreeSBalanceVgroupReq(SBalanceVgroupReq* pReq);
typedef struct {
int32_t useless; // useless
int32_t sqlLen;
char* sql;
} SAssignLeaderReq;
int32_t tSerializeSAssignLeaderReq(void* buf, int32_t bufLen, SAssignLeaderReq* pReq);
int32_t tDeserializeSAssignLeaderReq(void* buf, int32_t bufLen, SAssignLeaderReq* pReq);
void tFreeSAssignLeaderReq(SAssignLeaderReq* pReq);
typedef struct {
int32_t vgId1;
int32_t vgId2;

1
include/common/tmsgdef.h
View File

@ -421,6 +421,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used
TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_ASSIGN_LEADER, "mnd-arb-assign-leader", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_MND_ARB_MSG)
TD_NEW_MSG_SEG(TDMT_MAX_MSG) // msg end mark

4
include/libs/nodes/cmdnodes.h
View File

@ -683,6 +683,10 @@ typedef struct SBalanceVgroupStmt {
ENodeType type;
} SBalanceVgroupStmt;
typedef struct SAssignLeaderStmt {
ENodeType type;
} SAssignLeaderStmt;
typedef struct SBalanceVgroupLeaderStmt {
ENodeType type;
int32_t vgId;

2
include/os/osSysinfo.h
View File

@ -47,8 +47,10 @@ int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB);
int32_t taosGetSysAvailMemory(int64_t *availSize);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
void taosSetDefaultProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
int32_t taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes);
void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes);

1
include/util/taoserror.h
View File

@ -693,6 +693,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B)
#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C)
#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D)
#define TSDB_CODE_GRANT_TD_GPT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082E)
// sync
// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

3
source/client/src/clientMain.c
View File

@ -2272,7 +2272,10 @@ int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t c
(void)atomic_add_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
int code_s = taosStmt2AsyncBind(stmtAsyncBindThreadFunc, (void *)args);
if (code_s != TSDB_CODE_SUCCESS) {
(void)taosThreadMutexLock(&(pStmt->asyncBindParam.mutex));
(void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond));
(void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1);
(void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex));
// terrno = TAOS_SYSTEM_ERROR(errno);
}

47
source/client/src/clientSml.c
View File

@ -751,7 +751,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
}
static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
SArray *checkDumplicateCols, ESchemaAction *action, bool isTag) {
SHashObj *schemaHashCheck, ESchemaAction *action, bool isTag) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
for (int j = 0; j < taosArrayGetSize(cols); ++j) {
@ -759,15 +759,11 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
SML_CHECK_NULL(kv);
SML_CHECK_CODE(smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info));
}
for (int j = 0; j < taosArrayGetSize(checkDumplicateCols); ++j) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(checkDumplicateCols, j);
SML_CHECK_NULL(kv);
if (taosHashGet(schemaHash, kv->key, kv->keyLen) != NULL) {
if (taosHashGet(schemaHashCheck, kv->key, kv->keyLen) != NULL) {
SML_CHECK_CODE(TSDB_CODE_PAR_DUPLICATED_COLUMN);
}
}
END:
RETURN
}
@ -998,14 +994,14 @@ static int32_t smlBuildFields(SArray **pColumns, SArray **pTags, STableMeta *pTa
END:
RETURN
}
static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn,
static int32_t smlModifyTag(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn,
SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){
ESchemaAction action = SCHEMA_ACTION_NULL;
SArray *pColumns = NULL;
SArray *pTags = NULL;
int32_t code = 0;
int32_t lino = 0;
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, sTableData->cols, &action, true));
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->tags, hashTmpCheck, &action, true));
if (action != SCHEMA_ACTION_NULL) {
SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE));
@ -1029,14 +1025,14 @@ END:
RETURN
}
static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmp, SRequestConnInfo *conn,
static int32_t smlModifyCols(SSmlHandle *info, SHashObj* hashTmpCheck, SHashObj* hashTmp, SRequestConnInfo *conn,
SSmlSTableMeta *sTableData, SName *pName, STableMeta **pTableMeta){
ESchemaAction action = SCHEMA_ACTION_NULL;
SArray *pColumns = NULL;
SArray *pTags = NULL;
int32_t code = 0;
int32_t lino = 0;
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, sTableData->tags, &action, false));
SML_CHECK_CODE(smlProcessSchemaAction(info, (*pTableMeta)->schema, hashTmp, sTableData->cols, hashTmpCheck, &action, false));
if (action != SCHEMA_ACTION_NULL) {
SML_CHECK_CODE(smlCheckAuth(info, conn, pName->tname, AUTH_TYPE_WRITE));
@ -1079,7 +1075,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
}
int32_t code = 0;
int32_t lino = 0;
SHashObj *hashTmp = NULL;
SHashObj *colHashTmp = NULL;
SHashObj *tagHashTmp = NULL;
STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
@ -1119,17 +1116,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
SML_CHECK_CODE(TSDB_CODE_SML_NOT_SUPPORT_PK);
}
hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
SML_CHECK_NULL(hashTmp);
SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags));
SML_CHECK_CODE(smlModifyTag(info, hashTmp, &conn, sTableData, &pName, &pTableMeta));
taosHashClear(hashTmp);
SML_CHECK_CODE(smlBuildTempHash(hashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns));
SML_CHECK_CODE(smlModifyCols(info, hashTmp, &conn, sTableData, &pName, &pTableMeta));
colHashTmp = taosHashInit(pTableMeta->tableInfo.numOfColumns, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
tagHashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
SML_CHECK_NULL(colHashTmp);
SML_CHECK_NULL(tagHashTmp);
SML_CHECK_CODE(smlBuildTempHash(tagHashTmp, pTableMeta, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags));
SML_CHECK_CODE(smlBuildTempHash(colHashTmp, pTableMeta, 0, pTableMeta->tableInfo.numOfColumns));
SML_CHECK_CODE(smlModifyTag(info, colHashTmp, tagHashTmp, &conn, sTableData, &pName, &pTableMeta));
SML_CHECK_CODE(smlModifyCols(info, tagHashTmp, colHashTmp, &conn, sTableData, &pName, &pTableMeta));
needCheckMeta = true;
taosHashCleanup(hashTmp);
hashTmp = NULL;
taosHashCleanup(colHashTmp);
taosHashCleanup(tagHashTmp);
colHashTmp = NULL;
tagHashTmp = NULL;
} else {
uError("SML:0x%" PRIx64 " %s load table meta error: %s", info->id, __FUNCTION__, tstrerror(code));
goto END;
@ -1153,7 +1154,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
END:
taosHashCancelIterate(info->superTables, tmp);
taosHashCleanup(hashTmp);
taosHashCleanup(colHashTmp);
taosHashCleanup(tagHashTmp);
taosMemoryFreeClear(pTableMeta);
(void)catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // ignore refresh meta code if there is an error
uError("SML:0x%" PRIx64 " %s end failed:%d:%s, format:%d, needModifySchema:%d", info->id, __FUNCTION__, code,
@ -1924,3 +1926,4 @@ TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
TSDB_DEFAULT_TABLE_TTL, 0);
}

13
source/client/src/clientStmt2.c
View File

@ -1695,11 +1695,11 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) {
return pStmt->errCode;
}
taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
(void)taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
while (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
(void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex);
}
taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
(void)taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
if (pStmt->sql.stbInterlaceMode) {
STMT_ERR_RET(stmtAddBatch2(pStmt));
@ -1802,11 +1802,12 @@ int stmtClose2(TAOS_STMT2* stmt) {
pStmt->bindThreadInUse = false;
}
taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
(void)taosThreadMutexLock(&pStmt->asyncBindParam.mutex);
while (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) {
(void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex);
}
taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
(void)taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex);
(void)taosThreadCondDestroy(&pStmt->queue.waitCond);
(void)taosThreadMutexDestroy(&pStmt->queue.mutex);

56
source/client/test/clientTests.cpp
View File

@ -152,13 +152,13 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start
for (int32_t i = 0; i < numOfRows; i += 20) {
char sql[1024] = {0};
(void)sprintf(sql,
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7,
i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14,
i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
"insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
"(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)",
index, i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7,
i + 7, i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14,
i + 14, i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19);
TAOS_RES* p = taos_query(pConn, sql);
if (taos_errno(p) != 0) {
(void)printf("failed to insert data, reason:%s\n", taos_errstr(p));
@ -170,14 +170,14 @@ void createNewTable(TAOS* pConn, int32_t index, int32_t numOfRows, int64_t start
for (int32_t i = 0; i < numOfRows; i += 20) {
char sql[1024*50] = {0};
(void)sprintf(sql,
"insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, "
"%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, "
"'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')",
index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4,
pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9,
pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14,
i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18,
pVarchar, startTs + 19, i + 19, pVarchar);
"insert into tu%d values(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, "
"%d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, "
"'%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')(%ld, %d, '%s')",
index, startTs, i, pVarchar, startTs + 1, i + 1, pVarchar, startTs + 2, i + 2, pVarchar, startTs + 3, i + 3, pVarchar, startTs + 4, i + 4,
pVarchar, startTs + 5, i + 5, pVarchar, startTs + 6, i + 6, pVarchar, startTs + 7, i + 7, pVarchar, startTs + 8, i + 8, pVarchar, startTs + 9, i + 9,
pVarchar, startTs + 10, i + 10, pVarchar, startTs + 11, i + 11, pVarchar, startTs + 12, i + 12, pVarchar, startTs + 13, i + 13, pVarchar, startTs + 14,
i + 14, pVarchar, startTs + 15, i + 15, pVarchar, startTs + 16, i + 16, pVarchar, startTs + 17, i + 17, pVarchar, startTs + 18, i + 18,
pVarchar, startTs + 19, i + 19, pVarchar);
TAOS_RES* p = taos_query(pConn, sql);
if (taos_errno(p) != 0) {
(void)printf("failed to insert data, reason:%s\n", taos_errstr(p));
@ -309,8 +309,8 @@ int main(int argc, char** argv) {
if (code != 0) {
return code;
}
}
numOfThreads = TMAX(numOfThreads, 1);
@ -1441,7 +1441,7 @@ TEST(clientCase, sub_tb_test) {
int32_t code = tmq_get_topic_assignment(tmq, "t1", &pAssign, &numOfAssign);
if (code != 0) {
(void)printf("error occurs:%s\n", tmq_err2str(code));
(void)tmq_consumer_close(tmq);
(void)tmq_consumer_close(tmq);
taos_close(pConn);
(void)fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows);
return;
@ -1523,29 +1523,29 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "drop database if exists db1");
TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_timezone");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "create database db1");
pRes = taos_query(pConn, "create database db_timezone");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)");
pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)");
while (taos_errno(pRes) == TSDB_CODE_MND_DB_IN_CREATING || taos_errno(pRes) == TSDB_CODE_MND_DB_IN_DROPPING) {
taosMsleep(2000);
pRes = taos_query(pConn, "create table db1.t1 (ts timestamp, v int)");
pRes = taos_query(pConn, "create table db_timezone.t1 (ts timestamp, v int)");
}
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
char sql[256] = {0};
(void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:00', 1)");
(void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:00', 1)");
pRes = taos_query(pConn, sql);
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
taos_free_result(pRes);
pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 17:00:00'");
pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 17:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1571,7 +1571,7 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 01:00:00'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 01:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1588,7 +1588,7 @@ TEST(clientCase, timezone_Test) {
taos_free_result(pRes);
char sql[256] = {0};
(void)sprintf(sql, "insert into db1.t1 values('2023-09-16 17:00:01', 1)");
(void)sprintf(sql, "insert into db_timezone.t1 values('2023-09-16 17:00:01', 1)");
pRes = taos_query(pConn, sql);
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
@ -1604,7 +1604,7 @@ TEST(clientCase, timezone_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-16 09:00:00'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-16 09:00:00'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;
@ -1620,7 +1620,7 @@ TEST(clientCase, timezone_Test) {
taos_free_result(pRes);
{
TAOS_RES* pRes = taos_query(pConn, "select * from db1.t1 where ts == '2023-09-17 01:00:01'");
TAOS_RES* pRes = taos_query(pConn, "select * from db_timezone.t1 where ts == '2023-09-17 01:00:01'");
ASSERT_EQ(taos_errno(pRes), TSDB_CODE_SUCCESS);
TAOS_ROW pRow = NULL;

27
source/client/test/stmt2Test.cpp
View File

@ -15,6 +15,9 @@
#include <gtest/gtest.h>
#include <string.h>
#include <atomic>
#include <chrono>
#include <thread>
#include "clientInt.h"
#include "geosWrapper.h"
#include "osSemaphore.h"
@ -1662,7 +1665,7 @@ void stmtAsyncBindCb2(void* param, TAOS_RES* pRes, int code) {
return;
}
TEST(stmt2Case, async_order) {
void stmt2_async_test(std::atomic<bool>& stop_task) {
int CTB_NUMS = 2;
int ROW_NUMS = 2;
int CYC_NUMS = 2;
@ -1868,5 +1871,27 @@ TEST(stmt2Case, async_order) {
taosMemoryFree(tbs[i]);
}
taosMemoryFree(tbs);
stop_task = true;
}
TEST(stmt2Case, async_order) {
std::atomic<bool> stop_task(false);
std::thread t(stmt2_async_test, std::ref(stop_task));
// 等待 60 秒钟
auto start_time = std::chrono::steady_clock::now();
while (!stop_task) {
auto elapsed_time = std::chrono::steady_clock::now() - start_time;
if (std::chrono::duration_cast<std::chrono::seconds>(elapsed_time).count() > 60) {
FAIL() << "Test[stmt2_async_test] timed out";
t.detach();
break;
}
std::this_thread::sleep_for(std::chrono::seconds(1)); // 每 1s 检查一次
}
if (t.joinable()) {
t.join();
}
}
#pragma GCC diagnostic pop

44
source/common/src/msg/tmsg.c
View File

@ -7658,6 +7658,46 @@ _exit:
void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { FREESQL(); }
int32_t tSerializeSAssignLeaderReq(void *buf, int32_t bufLen, SAssignLeaderReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
int32_t lino;
int32_t tlen;
tEncoderInit(&encoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartEncode(&encoder));
TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->useless));
ENCODESQL();
tEndEncode(&encoder);
_exit:
if (code) {
tlen = code;
} else {
tlen = encoder.pos;
}
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSAssignLeaderReq(void *buf, int32_t bufLen, SAssignLeaderReq *pReq) {
SDecoder decoder = {0};
int32_t code = 0;
int32_t lino;
tDecoderInit(&decoder, buf, bufLen);
TAOS_CHECK_EXIT(tStartDecode(&decoder));
TAOS_CHECK_EXIT(tDecodeI32(&decoder, &pReq->useless));
DECODESQL();
tEndDecode(&decoder);
_exit:
tDecoderClear(&decoder);
return code;
}
void tFreeSAssignLeaderReq(SAssignLeaderReq *pReq) { FREESQL(); }
int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) {
SEncoder encoder = {0};
int32_t code = 0;
@ -8184,6 +8224,7 @@ int32_t tSerializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSetA
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->arbToken));
TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->arbTerm));
TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->memberToken));
TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->force));
tEndEncode(&encoder);
@ -8213,6 +8254,9 @@ int32_t tDeserializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSe
TAOS_CHECK_EXIT(terrno);
}
TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->memberToken));
if (!tDecodeIsEnd(&decoder)) {
TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->force));
}
tEndDecode(&decoder);

2
source/common/src/tglobal.c
View File

@ -188,7 +188,7 @@ char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/";
// tmq
int32_t tmqMaxTopicNum = 20;
int32_t tmqRowSize = 4096;
int32_t tmqRowSize = 1000;
// query
int32_t tsQueryPolicy = 1;
bool tsQueryTbNotExistAsEmpty = false;

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -160,6 +160,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_ARB_ASSIGN_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

80
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -45,6 +45,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp);
static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp);
static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextArbGroup(SMnode *pMnode, void *pIter);
static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq);
static int32_t mndArbCheckToken(const char *token1, const char *token2) {
if (token1 == NULL || token2 == NULL) return -1;
@ -70,6 +71,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp);
mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp);
mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp);
mndSetMsgHandle(pMnode, TDMT_MND_ARB_ASSIGN_LEADER, mndProcessAssignLeaderMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndRetrieveArbGroups);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup);
@ -535,11 +537,12 @@ static bool mndCheckArbMemberHbTimeout(SArbGroup *pArbGroup, int32_t index, int6
}
static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, char *arbToken, int64_t arbTerm,
char *memberToken) {
char *memberToken, bool force) {
SVArbSetAssignedLeaderReq req = {0};
req.arbToken = arbToken;
req.arbTerm = arbTerm;
req.memberToken = memberToken;
if (force) req.force = 1;
int32_t reqLen = tSerializeSVArbSetAssignedLeaderReq(NULL, 0, &req);
int32_t contLen = reqLen + sizeof(SMsgHead);
@ -559,10 +562,10 @@ static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, ch
}
static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken,
int64_t term, char *memberToken) {
int64_t term, char *memberToken, bool force) {
int32_t code = 0;
int32_t contLen = 0;
void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken);
void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken, force);
if (!pHead) {
mError("vgId:%d, failed to build set-assigned request", vgId);
code = -1;
@ -649,6 +652,73 @@ void mndArbCheckSync(SArbGroup *pArbGroup, int64_t nowMs, ECheckSyncOp *pOp, SAr
*pOp = CHECK_SYNC_UPDATE;
}
static int32_t mndProcessAssignLeaderMsg(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1, lino = 0;
SArray *pArray = NULL;
void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL;
SAssignLeaderReq req = {0};
if (tDeserializeSAssignLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
mInfo("begin to process assign leader");
char arbToken[TSDB_ARB_TOKEN_SIZE];
TAOS_CHECK_EXIT(mndGetArbToken(pMnode, arbToken));
int64_t term = mndGetTerm(pMnode);
if (term < 0) {
mError("arb failed to get term since %s", terrstr());
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
while (1) {
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break;
SArbGroup arbGroupDup = {0};
(void)taosThreadMutexLock(&pArbGroup->mutex);
mndArbGroupDupObj(pArbGroup, &arbGroupDup);
(void)taosThreadMutexUnlock(&pArbGroup->mutex);
sdbRelease(pSdb, pArbGroup);
int32_t dnodeId = 0;
for (int32_t i = 0; i < 2; i++) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, arbGroupDup.members[i].info.dnodeId);
bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
mndReleaseDnode(pMnode, pDnode);
if (isonline) {
dnodeId = arbGroupDup.members[i].info.dnodeId;
break;
}
}
(void)mndSendArbSetAssignedLeaderReq(pMnode, dnodeId, arbGroupDup.vgId, arbToken, term, "", true);
mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", arbGroupDup.vgId, dnodeId);
}
code = 0;
// auditRecord(pReq, pMnode->clusterId, "assignLeader", "", "", req.sql, req.sqlLen);
_exit:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to assign leader since %s", tstrerror(code));
}
tFreeSAssignLeaderReq(&req);
TAOS_RETURN(code);
}
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
int32_t code = 0, lino = 0;
SMnode *pMnode = pReq->info.node;
@ -701,7 +771,7 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
mTrace("vgId:%d, arb skip to send msg by check sync", vgId);
break;
case CHECK_SYNC_SET_ASSIGNED_LEADER:
(void)mndSendArbSetAssignedLeaderReq(pMnode, assgndDnodeId, vgId, arbToken, term, pAssgndLeader->token);
(void)mndSendArbSetAssignedLeaderReq(pMnode, assgndDnodeId, vgId, arbToken, term, pAssgndLeader->token, false);
mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, assgndDnodeId);
break;
case CHECK_SYNC_CHECK_SYNC:
@ -1361,7 +1431,7 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
char strCheckSyncCode[100] = {0};
char bufUpdateTime[40] = {0};
(void)formatTimestamp(bufUpdateTime, pGroup->updateTimeMs, TSDB_TIME_PRECISION_MILLI);
tsnprintf(strCheckSyncCode, 100, "%s(%s)", tstrerror(pGroup->code), bufUpdateTime);
(void)tsnprintf(strCheckSyncCode, 100, "%s(%s)", tstrerror(pGroup->code), bufUpdateTime);
char checkSyncCode[100 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(checkSyncCode, strCheckSyncCode, 100 + VARSTR_HEADER_SIZE);

32
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1259,6 +1259,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL;
SMCreateStbReq createReq = {0};
bool isAlter = false;
SHashObj *pHash = NULL;
if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
@ -1319,6 +1320,33 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
goto _OVER;
}
pHash = taosHashInit(createReq.numOfColumns + createReq.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
false, HASH_NO_LOCK);
if (pHash == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
for (int32_t i = 0; i < createReq.numOfColumns; ++i) {
SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i);
if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) {
if (code == TSDB_CODE_DUP_KEY) {
code = TSDB_CODE_TSC_DUP_COL_NAMES;
}
goto _OVER;
}
}
for (int32_t i = 0; i < createReq.numOfTags; ++i) {
SField *pField = taosArrayGet(createReq.pTags, i);
if ((code = taosHashPut(pHash, pField->name, strlen(pField->name), NULL, 0)) != 0) {
if (code == TSDB_CODE_DUP_KEY) {
code = TSDB_CODE_TSC_DUP_COL_NAMES;
}
goto _OVER;
}
}
pDb = mndAcquireDbByStb(pMnode, createReq.name);
if (pDb == NULL) {
code = TSDB_CODE_MND_DB_NOT_SELECTED;
@ -1383,6 +1411,10 @@ _OVER:
mndReleaseDb(pMnode, pDb);
tFreeSMCreateStbReq(&createReq);
if (pHash != NULL) {
taosHashCleanup(pHash);
}
TAOS_RETURN(code);
}

89
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -25,6 +25,7 @@ class MndTestStb : public ::testing::Test {
void* BuildCreateDbReq(const char* dbname, int32_t* pContLen);
void* BuildDropDbReq(const char* dbname, int32_t* pContLen);
void* BuildCreateStbReq(const char* stbname, int32_t* pContLen);
void* BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen);
void* BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen);
void* BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen);
void* BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname,
@ -137,6 +138,71 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) {
return pHead;
}
void* MndTestStb::BuildCreateStbDuplicateReq(const char* stbname, int32_t* pContLen) {
SMCreateStbReq createReq = {0};
createReq.numOfColumns = 2;
createReq.numOfTags = 4;
createReq.igExists = 0;
createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
strcpy(createReq.name, stbname);
{
SField field = {0};
field.bytes = 8;
field.type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(field.name, "ts");
taosArrayPush(createReq.pColumns, &field);
}
{
SField field = {0};
field.bytes = 12;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "col1");
taosArrayPush(createReq.pColumns, &field);
}
{
SField field = {0};
field.bytes = 2;
field.type = TSDB_DATA_TYPE_TINYINT;
strcpy(field.name, "tag1");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 8;
field.type = TSDB_DATA_TYPE_BIGINT;
strcpy(field.name, "tag2");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 16;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "tag3");
taosArrayPush(createReq.pTags, &field);
}
{
SField field = {0};
field.bytes = 16;
field.type = TSDB_DATA_TYPE_BINARY;
strcpy(field.name, "tag3");
taosArrayPush(createReq.pTags, &field);
}
int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
void* pHead = rpcMallocCont(tlen);
tSerializeSMCreateStbReq(pHead, tlen, &createReq);
tFreeSMCreateStbReq(&createReq);
*pContLen = tlen;
return pHead;
}
void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) {
SMAlterStbReq req = {0};
strcpy(req.name, stbname);
@ -896,3 +962,26 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
rpcFreeCont(pRsp->pCont);
}
}
TEST_F(MndTestStb, 09_Create_Duplicate_Stb) {
const char* dbname = "1.d2";
const char* stbname = "1.d2.stb";
{
int32_t contLen = 0;
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
rpcFreeCont(pRsp->pCont);
}
{
int32_t contLen = 0;
void* pReq = BuildCreateStbDuplicateReq(stbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_TSC_DUP_COL_NAMES);
rpcFreeCont(pRsp->pCont);
}
}

16
source/dnode/vnode/src/tq/tqRead.c
View File

@ -1098,6 +1098,22 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* block
*pSubmitTbDataRet = pSubmitTbData;
}
if (fetchMeta == ONLY_META) {
if (pSubmitTbData->pCreateTbReq != NULL) {
if (pRsp->createTableReq == NULL){
pRsp->createTableReq = taosArrayInit(0, POINTER_BYTES);
if (pRsp->createTableReq == NULL){
return terrno;
}
}
if (taosArrayPush(pRsp->createTableReq, &pSubmitTbData->pCreateTbReq) == NULL){
return terrno;
}
pSubmitTbData->pCreateTbReq = NULL;
}
return 0;
}
int32_t sversion = pSubmitTbData->sver;
int64_t uid = pSubmitTbData->uid;
pReader->lastBlkUid = uid;

26
source/dnode/vnode/src/tq/tqScan.c
View File

@ -339,6 +339,9 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
bool tmp = (pSubmitTbData->flags & pRequest->sourceExcluded) != 0;
TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS);
if (pHandle->fetchMeta == ONLY_META){
goto END;
}
int32_t blockNum = taosArrayGetSize(pBlocks) == 0 ? 1 : taosArrayGetSize(pBlocks);
if (pRsp->withTbName) {
@ -347,7 +350,6 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
TSDB_CHECK_CODE(code, lino, END);
}
tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbData->pCreateTbReq == NULL);
TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS);
for (int32_t i = 0; i < blockNum; i++) {
if (taosArrayGetSize(pBlocks) == 0){
@ -403,14 +405,16 @@ static void preProcessSubmitMsg(STqHandle* pHandle, const SMqPollReq* pRequest,
}
int64_t uid = pSubmitTbData->uid;
if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) {
tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid);
terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT;
return;
} else {
int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES);
if (code != 0){
tqError("failed to add table uid to hash, code:%d, uid:%"PRId64, code, uid);
if (pRequest->rawData) {
if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) {
tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid);
terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT;
return;
} else {
int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES);
if (code != 0) {
tqError("failed to add table uid to hash, code:%d, uid:%" PRId64, code, uid);
}
}
}
@ -453,9 +457,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqData
}
code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver, rawList);
TSDB_CHECK_CODE(code, lino, END);
if (pRequest->rawData) {
preProcessSubmitMsg(pHandle, pRequest, &rawList);
}
preProcessSubmitMsg(pHandle, pRequest, &rawList);
// data could not contains same uid data in rawdata mode
if (pRequest->rawData != 0 && terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){
goto END;

171
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -226,6 +226,82 @@ static void tDeleteCommon(void* parm) {}
taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : \
(pRequest->rawData == 1 ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP)
static int32_t buildBatchMeta(SMqBatchMetaRsp *btMetaRsp, int16_t type, int32_t bodyLen, void* body){
int32_t code = 0;
if (!btMetaRsp->batchMetaReq) {
btMetaRsp->batchMetaReq = taosArrayInit(4, POINTER_BYTES);
TQ_NULL_GO_TO_END(btMetaRsp->batchMetaReq);
btMetaRsp->batchMetaLen = taosArrayInit(4, sizeof(int32_t));
TQ_NULL_GO_TO_END(btMetaRsp->batchMetaLen);
}
SMqMetaRsp tmpMetaRsp = {0};
tmpMetaRsp.resMsgType = type;
tmpMetaRsp.metaRspLen = bodyLen;
tmpMetaRsp.metaRsp = body;
uint32_t len = 0;
tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
if (TSDB_CODE_SUCCESS != code) {
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
goto END;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
TQ_NULL_GO_TO_END(tBuf);
void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, metaBuff, len);
code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
tEncoderClear(&encoder);
if (code < 0) {
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
goto END;
}
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp->batchMetaReq, &tBuf));
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp->batchMetaLen, &tLen));
END:
return code;
}
static int32_t buildCreateTbBatchReqBinary(SMqDataRsp *taosxRsp, void** pBuf, int32_t *len){
int32_t code = 0;
SVCreateTbBatchReq pReq = {0};
pReq.nReqs = taosArrayGetSize(taosxRsp->createTableReq);
pReq.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));
TQ_NULL_GO_TO_END(pReq.pArray);
for (int i = 0; i < taosArrayGetSize(taosxRsp->createTableReq); i++){
void *createTableReq = taosArrayGetP(taosxRsp->createTableReq, i);
TQ_NULL_GO_TO_END(taosArrayPush(pReq.pArray, createTableReq));
}
tEncodeSize(tEncodeSVCreateTbBatchReq, &pReq, *len, code);
if (code < 0) {
goto END;
}
*len += sizeof(SMsgHead);
*pBuf = taosMemoryMalloc(*len);
TQ_NULL_GO_TO_END(pBuf);
SEncoder coder = {0};
tEncoderInit(&coder, POINTER_SHIFT(*pBuf, sizeof(SMsgHead)), *len);
code = tEncodeSVCreateTbBatchReq(&coder, &pReq);
tEncoderClear(&coder);
END:
taosArrayDestroy(pReq.pArray);
return code;
}
#define SEND_BATCH_META_RSP \
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);\
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);\
goto END;
#define SEND_DATA_RSP \
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);\
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, POLL_RSP_TYPE(pRequest, taosxRsp), vgId);\
goto END;
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) {
int32_t vgId = TD_VID(pTq->pVnode);
@ -272,14 +348,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) {
if (totalMetaRows > 0) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto END;
SEND_BATCH_META_RSP
}
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
goto END;
SEND_DATA_RSP
}
SWalCont* pHead = &pHandle->pWalReader->pHead->head;
@ -289,10 +360,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
// process meta
if (pHead->msgType != TDMT_VND_SUBMIT) {
if (totalRows > 0) {
tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
goto END;
SEND_DATA_RSP
}
if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
@ -318,53 +386,20 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId);
goto END;
}
if (!btMetaRsp.batchMetaReq) {
btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
TQ_NULL_GO_TO_END(btMetaRsp.batchMetaReq);
btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
TQ_NULL_GO_TO_END(btMetaRsp.batchMetaLen);
}
code = buildBatchMeta(&btMetaRsp, pHead->msgType, pHead->bodyLen, pHead->body);
fetchVer++;
SMqMetaRsp tmpMetaRsp = {0};
tmpMetaRsp.resMsgType = pHead->msgType;
tmpMetaRsp.metaRspLen = pHead->bodyLen;
tmpMetaRsp.metaRsp = pHead->body;
uint32_t len = 0;
tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code);
if (TSDB_CODE_SUCCESS != code) {
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
if (code != 0){
goto END;
}
int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen);
TQ_NULL_GO_TO_END(tBuf);
void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
SEncoder encoder = {0};
tEncoderInit(&encoder, metaBuff, len);
code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
tEncoderClear(&encoder);
if (code < 0) {
tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue;
}
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaReq, &tBuf));
TQ_NULL_GO_TO_END (taosArrayPush(btMetaRsp.batchMetaLen, &tLen));
totalMetaRows++;
if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > pRequest->timeout)) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto END;
SEND_BATCH_META_RSP
}
continue;
}
if (totalMetaRows > 0) {
tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer);
code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId);
goto END;
if (totalMetaRows > 0 && pHandle->fetchMeta != ONLY_META) {
SEND_BATCH_META_RSP
}
// process data
@ -376,17 +411,39 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest));
if (pHandle->fetchMeta == ONLY_META && taosArrayGetSize(taosxRsp.createTableReq) > 0){
int32_t len = 0;
void *pBuf = NULL;
code = buildCreateTbBatchReqBinary(&taosxRsp, &pBuf, &len);
if (code == 0){
code = buildBatchMeta(&btMetaRsp, TDMT_VND_CREATE_TABLE, len, pBuf);
}
taosMemoryFree(pBuf);
taosArrayDestroyP(taosxRsp.createTableReq, NULL);
taosxRsp.createTableReq = NULL;
fetchVer++;
if (code != 0){
goto END;
}
totalMetaRows++;
if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) ||
(taosGetTimestampMs() - st > pRequest->timeout) ||
(!pRequest->enableBatchMeta && !pRequest->useSnapshot)) {
SEND_BATCH_META_RSP
}
continue;
}
if ((pRequest->rawData == 0 && totalRows >= pRequest->minPollRows) ||
(taosGetTimestampMs() - st > pRequest->timeout) ||
(pRequest->rawData != 0 && (taosArrayGetSize(taosxRsp.blockData) > pRequest->minPollRows ||
terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT))) {
// tqDebug("start to send rsp, block num:%d, totalRows:%d, createTableNum:%d, terrno:%d",
// (int)taosArrayGetSize(taosxRsp.blockData), totalRows, taosxRsp.createTableNum, terrno);
tqOffsetResetToLog(&taosxRsp.rspOffset, terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT ? fetchVer : fetchVer + 1);
code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp,
POLL_RSP_TYPE(pRequest, taosxRsp), vgId);
if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){terrno = 0;}
goto END;
if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){
terrno = 0;
} else{
fetchVer++;
}
SEND_DATA_RSP
} else {
fetchVer++;
}

6
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -199,6 +199,8 @@ const char* nodesNodeName(ENodeType type) {
return "ResetStreamStmt";
case QUERY_NODE_BALANCE_VGROUP_STMT:
return "BalanceVgroupStmt";
case QUERY_NODE_ASSIGN_LEADER_STMT:
return "AssignLeaderStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return "BalanceVgroupLeaderStmt";
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
@ -8195,6 +8197,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return dropStreamStmtToJson(pObj, pJson);
case QUERY_NODE_BALANCE_VGROUP_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize.
case QUERY_NODE_ASSIGN_LEADER_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize.
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
@ -8564,6 +8568,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToDropStreamStmt(pJson, pObj);
case QUERY_NODE_BALANCE_VGROUP_STMT:
return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize.
case QUERY_NODE_ASSIGN_LEADER_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
return TSDB_CODE_SUCCESS;
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -625,6 +625,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_BALANCE_VGROUP_STMT:
code = makeNode(type, sizeof(SBalanceVgroupStmt), &pNode);
break;
case QUERY_NODE_ASSIGN_LEADER_STMT:
code = makeNode(type, sizeof(SAssignLeaderStmt), &pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
code = makeNode(type, sizeof(SBalanceVgroupLeaderStmt), &pNode);
break;
@ -1506,6 +1509,7 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_RESUME_STREAM_STMT: // no pointer field
case QUERY_NODE_RESET_STREAM_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field
case QUERY_NODE_ASSIGN_LEADER_STMT:
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field
case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT: // no pointer field
case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field

1
source/libs/parser/inc/parAst.h
View File

@ -309,6 +309,7 @@ SNode* createResetStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STok
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
SNode* createAssignLeaderStmt(SAstCreateContext* pCxt);
SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId);
SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName);
SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);

11
source/libs/parser/inc/sql.y
View File

@ -50,10 +50,14 @@
%left AND.
%left UNION ALL MINUS EXCEPT INTERSECT.
%left NK_BITAND NK_BITOR NK_LSHIFT NK_RSHIFT.
%left NK_LT NK_GT NK_LE NK_GE NK_EQ NK_NE LIKE MATCH NMATCH REGEXP CONTAINS BETWEEN IS IN.
%left NK_PLUS NK_MINUS.
%left NK_STAR NK_SLASH NK_REM.
%left NK_CONCAT.
%right NOT.
%left NK_ARROW.
%right INNER LEFT RIGHT FULL OUTER SEMI ANTI ASOF WINDOW JOIN ON WINDOW_OFFSET JLIMIT.
/************************************************ create/alter account *****************************************/
@ -862,6 +866,9 @@ cmd ::= KILL COMPACT NK_INTEGER(A).
/************************************************ merge/redistribute/ vgroup ******************************************/
cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
cmd ::= ASSIGN LEADER FORCE. { pCxt->pRootNode = createAssignLeaderStmt(pCxt); }
cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A). { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); }
cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A). { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); }
cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
@ -1285,9 +1292,9 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).
function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::= cols_func(B) NK_LP cols_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
function_expression(A) ::=
CAST(B) NK_LP expr_or_subquery(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
CAST(B) NK_LP common_expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
function_expression(A) ::=
CAST(B) NK_LP expr_or_subquery(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
CAST(B) NK_LP common_expression(C) AS type_name_default_len(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
function_expression(A) ::=
POSITION(B) NK_LP expr_or_subquery(C) IN expr_or_subquery(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createPositionFunctionNode(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); }
function_expression(A) ::=

10
source/libs/parser/src/parAstCreater.c
View File

@ -3927,6 +3927,16 @@ _err:
return NULL;
}
SNode* createAssignLeaderStmt(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt);
SAssignLeaderStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_ASSIGN_LEADER_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
return (SNode*)pStmt;
_err:
return NULL;
}
SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
CHECK_PARSER_STATUS(pCxt);
SBalanceVgroupLeaderStmt* pStmt = NULL;

7
source/libs/parser/src/parInsertUtil.c
View File

@ -1079,11 +1079,14 @@ end:
}
int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data) {
transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion);
int code = transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion);
if (code != 0){
return code;
}
SVgroupDataCxt* pVgCxt = NULL;
void** pp = taosHashGet(pVgroupHash, &pTableMeta->vgId, sizeof(pTableMeta->vgId));
if (NULL == pp) {
int code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt);
code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt);
if (code != 0){
return code;
}

1
source/libs/parser/src/parTokenizer.c
View File

@ -360,6 +360,7 @@ static SKeyword keywordTable[] = {
{"ON_FAILURE", TK_ON_FAILURE},
{"NOTIFY_HISTORY", TK_NOTIFY_HISTORY},
{"REGEXP", TK_REGEXP},
{"ASSIGN", TK_ASSIGN},
{"TRUE_FOR", TK_TRUE_FOR}
};
// clang-format on

14
source/libs/parser/src/parTranslater.c
View File

@ -8934,6 +8934,10 @@ static int32_t fillCmdSql(STranslateContext* pCxt, int16_t msgType, void* pReq)
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SBalanceVgroupReq, pReq);
break;
}
case TDMT_MND_ARB_ASSIGN_LEADER: {
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SAssignLeaderReq, pReq);
break;
}
case TDMT_MND_REDISTRIBUTE_VGROUP: {
FILL_CMD_SQL(sql, sqlLen, pCmdReq, SRedistributeVgroupReq, pReq);
break;
@ -13182,6 +13186,13 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
return code;
}
static int32_t translateAssignLeader(STranslateContext* pCxt, SAssignLeaderStmt* pStmt) {
SAssignLeaderReq req = {0};
int32_t code = buildCmdMsg(pCxt, TDMT_MND_ARB_ASSIGN_LEADER, (FSerializeFunc)tSerializeSAssignLeaderReq, &req);
tFreeSAssignLeaderReq(&req);
return code;
}
static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
SBalanceVgroupLeaderReq req = {0};
req.vgId = pStmt->vgId;
@ -13939,6 +13950,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_BALANCE_VGROUP_STMT:
code = translateBalanceVgroup(pCxt, (SBalanceVgroupStmt*)pNode);
break;
case QUERY_NODE_ASSIGN_LEADER_STMT:
code = translateAssignLeader(pCxt, (SAssignLeaderStmt*)pNode);
break;
case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
break;

10
source/libs/sync/src/syncMain.c
View File

@ -334,10 +334,12 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg) {
ths->arbTerm = TMAX(req.arbTerm, ths->arbTerm);
if (strncmp(req.memberToken, ths->arbToken, TSDB_ARB_TOKEN_SIZE) != 0) {
sInfo("vgId:%d, skip to set assigned leader, token mismatch, local:%s, msg:%s", ths->vgId, ths->arbToken,
req.memberToken);
goto _OVER;
if (!req.force) {
if (strncmp(req.memberToken, ths->arbToken, TSDB_ARB_TOKEN_SIZE) != 0) {
sInfo("vgId:%d, skip to set assigned leader, token mismatch, local:%s, msg:%s", ths->vgId, ths->arbToken,
req.memberToken);
goto _OVER;
}
}
if (ths->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) {

2
source/os/src/osRand.c
View File

@ -87,7 +87,7 @@ void taosRandStr(char* str, int32_t size) {
void taosRandStr2(char* str, int32_t size) {
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789@";
int32_t len = strlen(set);
int32_t len = 37;
for (int32_t i = 0; i < size; ++i) {
str[i] = set[taosRand() % len];

2
source/os/src/osSignal.c
View File

@ -114,6 +114,7 @@ int32_t taosDflSignal(int32_t signum) {
return 0;
}
#if 0
int32_t taosKillChildOnParentStopped() {
#ifndef _TD_DARWIN_64
int32_t code = prctl(PR_SET_PDEATHSIG, SIGKILL);
@ -126,5 +127,6 @@ int32_t taosKillChildOnParentStopped() {
#endif
return 0;
}
#endif
#endif

4
source/os/src/osString.c
View File

@ -294,6 +294,7 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) {
//#endif
}
#if 0
int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) {
if (target_ucs4 == NULL || source_ucs4 == NULL || len_ucs4 <= 0) {
return TSDB_CODE_INVALID_PARA;
@ -307,11 +308,12 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4)
return TSDB_CODE_SUCCESS;
}
#endif
iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) {
if(idx == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return (iconv_t)-1;
return (iconv_t)NULL;
}
if (charsetCxt == NULL){
charsetCxt = tsCharsetCxt;

5
source/os/src/osSysinfo.c
View File

@ -147,7 +147,6 @@ static void taosGetProcIOnfos() {
#endif
static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
OS_PARAM_CHECK(cpuInfo);
int32_t code = 0;
#ifdef WINDOWS
FILETIME pre_idleTime = {0};
@ -200,7 +199,6 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
}
static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
OS_PARAM_CHECK(cpuInfo);
int32_t code = 0;
#ifdef WINDOWS
@ -541,7 +539,6 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
// Returns the container's CPU quota if successful, otherwise returns the physical CPU cores
static int32_t taosCntrGetCpuCores(float *numOfCores) {
OS_PARAM_CHECK(numOfCores);
#ifdef WINDOWS
return TSDB_CODE_UNSUPPORT_OS;
#elif defined(_TD_DARWIN_64)
@ -1092,6 +1089,7 @@ void taosSetDefaultCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes
if (transmit_bytes) *transmit_bytes = 0;
}
#if 0
void taosKillSystem() {
#ifdef WINDOWS
printf("function taosKillSystem, exit!");
@ -1105,6 +1103,7 @@ void taosKillSystem() {
(void)kill(tsProcId, 2);
#endif
}
#endif
#define UUIDLEN (36)
int32_t taosGetSystemUUIDLimit36(char *uid, int32_t uidlen) {

91
source/os/test/CMakeLists.txt
View File

@ -15,36 +15,47 @@ ENDIF()
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc)
if(TD_LINUX)
add_executable(osAtomicTests "osAtomicTests.cpp")
target_link_libraries(osAtomicTests os util gtest_main)
add_test(
NAME osAtomicTests
COMMAND osAtomicTests
)
endif()
add_executable(osAtomicTests "osAtomicTests.cpp")
target_link_libraries(osAtomicTests os util gtest_main)
add_test(
NAME osAtomicTests
COMMAND osAtomicTests
)
if(TD_LINUX)
add_executable(osDirTests "osDirTests.cpp")
target_link_libraries(osDirTests os util gtest_main)
add_test(
NAME osDirTests
COMMAND osDirTests
)
add_executable(osFileTests "osFileTests.cpp")
target_link_libraries(osFileTests os util gtest_main)
add_test(
NAME osFileTests
COMMAND osFileTests
)
endif()
add_executable(osDirTests "osDirTests.cpp")
target_link_libraries(osDirTests os util gtest_main)
add_test(
NAME osDirTests
COMMAND osDirTests
)
if(TD_LINUX)
add_executable(osEnvTests "osEnvTests.cpp")
target_link_libraries(osEnvTests os util gtest_main)
add_test(
NAME osEnvTests
COMMAND osEnvTests
)
add_executable(osFileTests "osFileTests.cpp")
target_link_libraries(osFileTests os util gtest_main)
add_test(
NAME osFileTests
COMMAND osFileTests
)
add_executable(osEnvTests "osEnvTests.cpp")
target_link_libraries(osEnvTests os util gtest_main)
add_test(
NAME osEnvTests
COMMAND osEnvTests
)
add_executable(osTests "osTests.cpp")
target_link_libraries(osTests os util gtest_main)
add_test(
NAME osTests
COMMAND osTests
)
add_executable(osThreadTests "osThreadTests.cpp")
target_link_libraries(osThreadTests os util gtest_main)
add_test(
NAME osThreadTests
COMMAND osThreadTests
)
endif()
add_executable(osMathTests "osMathTests.cpp")
@ -61,13 +72,6 @@ add_test(
COMMAND osSemaphoreTests
)
add_executable(osSignalTests "osSignalTests.cpp")
target_link_libraries(osSignalTests os util gtest_main)
add_test(
NAME osSignalTests
COMMAND osSignalTests
)
add_executable(osSleepTests "osSleepTests.cpp")
target_link_libraries(osSleepTests os util gtest_main)
add_test(
@ -82,13 +86,6 @@ add_test(
COMMAND osStringTests
)
add_executable(osTests "osTests.cpp")
target_link_libraries(osTests os util gtest_main)
add_test(
NAME osTests
COMMAND osTests
)
add_executable(osSystemTests "osSystemTests.cpp")
target_link_libraries(osSystemTests os util gtest_main)
add_test(
@ -96,19 +93,9 @@ add_test(
COMMAND osSystemTests
)
if(TD_LINUX)
add_executable(osThreadTests "osThreadTests.cpp")
target_link_libraries(osThreadTests os util gtest_main)
add_test(
NAME osThreadTests
COMMAND osThreadTests
)
endif()
add_executable(osTimeTests "osTimeTests.cpp")
target_link_libraries(osTimeTests os util gtest_main)
add_test(
NAME osTimeTests
COMMAND osTimeTests
)

40
source/os/test/osSignalTests.cpp
View File

@ -1,40 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wformat"
#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
#pragma GCC diagnostic ignored "-Wpointer-arith"
#include "os.h"
#include "tlog.h"
TEST(osSignalTests, taosSetSignal) {
// Set up SIGINT handler using taosSetSignal
//taosSetSignal(SIGINT, sigint_handler);
// Print PID for testing purposes
// printf("PID: %d\n", getpid());
// Wait for signal to be received
}

943
source/os/test/osStringTests.cpp
View File

File diff suppressed because it is too large Load Diff

369
source/os/test/osTests.cpp
View File

@ -14,8 +14,8 @@
*/
#include <gtest/gtest.h>
#include <iostream>
#include <inttypes.h>
#include <iostream>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -36,6 +36,230 @@
#include <arpa/inet.h>
TEST(osTest, locale) {
char *ret = taosCharsetReplace(NULL);
EXPECT_EQ(ret, nullptr);
ret = taosCharsetReplace("utf8");
EXPECT_NE(ret, nullptr);
ret = taosCharsetReplace("utf-8");
EXPECT_NE(ret, nullptr);
taosGetSystemLocale(NULL, "");
taosGetSystemLocale("", NULL);
}
TEST(osTest, memory) {
int32_t ret = taosMemoryDbgInitRestore();
EXPECT_EQ(ret, 0);
int64_t ret64 = taosMemSize(NULL);
EXPECT_EQ(ret64, 0);
}
TEST(osTest, rand2) {
char str[128] = {0};
taosRandStr2(str, 100);
}
TEST(osTest, socket2) {
int32_t ret32 = taosCloseSocket(NULL);
EXPECT_NE(ret32, 0);
ret32 = taosSetSockOpt(NULL, 0, 0, NULL, 0);
EXPECT_NE(ret32, 0);
#if defined(LINUX)
struct in_addr ipInt;
ipInt.s_addr = htonl(0x7F000001);
char buf[128] = {0};
taosInetNtop(ipInt, buf, 32);
#endif
ret32 = taosGetIpv4FromFqdn("localhost", NULL);
EXPECT_NE(ret32, 0);
uint32_t ip = 0;
ret32 = taosGetIpv4FromFqdn(NULL, &ip);
EXPECT_NE(ret32, 0);
taosInetNtoa(NULL, ip);
ret32 = taosInetAddr(NULL);
EXPECT_EQ(ret32, 0);
ret32 = taosWinSocketInit();
EXPECT_EQ(ret32, 0);
}
TEST(osTest, time2) {
taosGetLocalTimezoneOffset();
char buf[12] = {0};
char fmt[12] = {0};
void *retptr = taosStrpTime(NULL, fmt, NULL);
EXPECT_EQ(retptr, nullptr);
retptr = taosStrpTime(buf, NULL, NULL);
EXPECT_EQ(retptr, nullptr);
size_t ret = taosStrfTime(NULL, 0, fmt, NULL);
EXPECT_EQ(ret, 0);
ret = taosStrfTime(buf, 0, NULL, NULL);
EXPECT_EQ(ret, 0);
time_t tp = {0};
struct tm *retptr2 = taosGmTimeR(&tp, NULL);
EXPECT_EQ(retptr2, nullptr);
retptr2 = taosGmTimeR(NULL, NULL);
EXPECT_EQ(retptr2, nullptr);
time_t rett = taosTimeGm(NULL);
EXPECT_EQ(rett, -1);
timezone_t tz = {0};
retptr2 = taosLocalTime(&tp, NULL, NULL, 0, tz);
EXPECT_EQ(retptr2, nullptr);
retptr2 = taosLocalTime(NULL, NULL, NULL, 0, tz);
EXPECT_EQ(retptr2, nullptr);
}
TEST(osTest, system) {
#if defined(LINUX)
taosSetConsoleEcho(false);
taosSetConsoleEcho(true);
taosGetOldTerminalMode();
taosCloseCmd(NULL);
TdCmdPtr ptr = taosOpenCmd(NULL);
EXPECT_EQ(ptr, nullptr);
taosCloseCmd(&ptr);
ptr = taosOpenCmd("echo 'hello world'");
ASSERT_NE(ptr, nullptr);
char buf[256] = {0};
int64_t ret64 = taosGetsCmd(NULL, 0, NULL);
EXPECT_LE(ret64, 0);
ret64 = taosGetsCmd(ptr, 0, NULL);
EXPECT_LE(ret64, 0);
ret64 = taosGetsCmd(ptr, 255, buf);
EXPECT_GT(ret64, 0);
taosCloseCmd(&ptr);
ptr = taosOpenCmd("echoxxx 'hello world'");
ASSERT_NE(ptr, nullptr);
ret64 = taosGetsCmd(ptr, 255, buf);
EXPECT_LE(ret64, 0);
taosCloseCmd(&ptr);
ret64 = taosGetLineCmd(NULL, NULL);
EXPECT_LE(ret64, 0);
ret64 = taosGetLineCmd(ptr, NULL);
EXPECT_LE(ret64, 0);
ptr = taosOpenCmd("echo 'hello world'");
ASSERT_NE(ptr, nullptr);
char *ptrBuf = NULL;
ret64 = taosGetLineCmd(ptr, &ptrBuf);
EXPECT_GE(ret64, 0);
taosCloseCmd(&ptr);
ptr = taosOpenCmd("echoxxx 'hello world'");
ASSERT_NE(ptr, nullptr);
ret64 = taosGetLineCmd(ptr, &ptrBuf);
EXPECT_LE(ret64, 0);
taosCloseCmd(&ptr);
int32_t ret32 = taosEOFCmd(NULL);
EXPECT_EQ(ret32, 0);
#endif
}
TEST(osTest, sysinfo) {
#if defined(LINUX)
int32_t ret32 = 0;
ret32 = taosGetEmail(NULL, 0);
EXPECT_NE(ret32, 0);
ret32 = taosGetOsReleaseName(NULL, NULL, NULL, 0);
EXPECT_NE(ret32, 0);
char buf[128] = {0};
float numOfCores = 0;
ret32 = taosGetCpuInfo(buf, 0, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetCpuInfo(NULL, 0, &numOfCores);
EXPECT_NE(ret32, 0);
ret32 = taosGetCpuCores(NULL, false);
EXPECT_NE(ret32, 0);
ret32 = taosGetTotalMemory(NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcMemory(NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetSysMemory(NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetDiskSize(buf, NULL);
EXPECT_NE(ret32, 0);
SDiskSize disksize = {0};
ret32 = taosGetDiskSize(NULL, &disksize);
EXPECT_NE(ret32, 0);
int64_t tmp = 0;
ret32 = taosGetProcIO(NULL, NULL, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIO(&tmp, NULL, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIO(&tmp, &tmp, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIO(&tmp, &tmp, &tmp, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIODelta(NULL, NULL, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIODelta(&tmp, NULL, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIODelta(&tmp, &tmp, NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetProcIODelta(&tmp, &tmp, &tmp, NULL);
EXPECT_NE(ret32, 0);
taosGetProcIODelta(NULL, NULL, NULL, NULL);
taosGetProcIODelta(&tmp, &tmp, &tmp, &tmp);
ret32 = taosGetCardInfo(NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetCardInfo(&tmp, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetCardInfoDelta(NULL, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetCardInfoDelta(&tmp, NULL);
EXPECT_NE(ret32, 0);
taosGetCardInfoDelta(NULL, NULL);
taosGetCardInfoDelta(&tmp, &tmp);
ret32 = taosGetSystemUUIDLimit36(NULL, 0);
EXPECT_NE(ret32, 0);
ret32 = taosGetSystemUUIDLen(NULL, 0);
EXPECT_NE(ret32, 0);
ret32 = taosGetSystemUUIDLen(buf, -1);
EXPECT_NE(ret32, 0);
taosSetCoreDump(false);
ret32 = taosGetlocalhostname(NULL, 0);
EXPECT_NE(ret32, 0);
#endif
}
TEST(osTest, osFQDNSuccess) {
char fqdn[TD_FQDN_LEN];
char ipString[INET_ADDRSTRLEN];
@ -47,8 +271,8 @@ TEST(osTest, osFQDNSuccess) {
struct in_addr addr;
addr.s_addr = htonl(ipv4);
(void)snprintf(ipString, INET_ADDRSTRLEN, "%u.%u.%u.%u", (unsigned int)(addr.s_addr >> 24) & 0xFF,
(unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF,
(unsigned int)(addr.s_addr) & 0xFF);
(unsigned int)(addr.s_addr >> 16) & 0xFF, (unsigned int)(addr.s_addr >> 8) & 0xFF,
(unsigned int)(addr.s_addr) & 0xFF);
(void)printf("fqdn:%s ip:%s\n", fqdn, ipString);
}
@ -56,7 +280,7 @@ TEST(osTest, osFQDNFailed) {
char fqdn[1024] = "fqdn_test_not_found";
char ipString[24];
uint32_t ipv4 = 0;
int32_t code = taosGetIpv4FromFqdn(fqdn, &ipv4);
int32_t code = taosGetIpv4FromFqdn(fqdn, &ipv4);
ASSERT_NE(code, 0);
terrno = TSDB_CODE_RPC_FQDN_ERROR;
@ -79,7 +303,7 @@ TEST(osTest, osSystem) {
}
void fileOperateOnFree(void *param) {
char * fname = (char *)param;
char *fname = (char *)param;
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE);
(void)printf("On free thread open file\n");
ASSERT_NE(pFile, nullptr);
@ -101,7 +325,7 @@ void *fileOperateOnFreeThread(void *param) {
return NULL;
}
void fileOperateOnBusy(void *param) {
char * fname = (char *)param;
char *fname = (char *)param;
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE);
(void)printf("On busy thread open file\n");
if (pFile == NULL) return;
@ -165,9 +389,9 @@ TEST(osTest, osFile) {
(void)taosThreadJoin(thread2, NULL);
taosThreadClear(&thread2);
//int ret = taosRemoveFile(fname);
//ASSERT_EQ(ret, 0);
//printf("remove file success");
// int ret = taosRemoveFile(fname);
// ASSERT_EQ(ret, 0);
// printf("remove file success");
}
#ifndef OSFILE_PERFORMANCE_TEST
@ -178,18 +402,109 @@ TEST(osTest, osFile) {
#define TESTTIMES 1000
char *getRandomWord() {
static char words[][MAX_WORD_LENGTH] = {
"Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit",
"sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna",
"aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco",
"Why", "do", "programmers", "prefer", "using", "dark", "mode?", "Because", "light", "attracts",
"bugs", "and", "they", "want", "to", "code", "in", "peace,", "like", "a", "ninja", "in", "the", "shadows."
"aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco",
"laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat", "Duis", "aute", "irure",
"dolor", "in", "reprehenderit", "in", "voluptate", "velit", "esse", "cillum", "dolore", "eu",
"fugiat", "nulla", "pariatur", "Excepteur", "sint", "occaecat", "cupidatat", "non", "proident",
"sunt", "in", "culpa", "qui", "officia", "deserunt", "mollit", "anim", "id", "est", "laborum"
};
static char words[][MAX_WORD_LENGTH] = {"Lorem",
"ipsum",
"dolor",
"sit",
"amet",
"consectetur",
"adipiscing",
"elit",
"sed",
"do",
"eiusmod",
"tempor",
"incididunt",
"ut",
"labore",
"et",
"dolore",
"magna",
"aliqua",
"Ut",
"enim",
"ad",
"minim",
"veniam",
"quis",
"nostrud",
"exercitation",
"ullamco",
"Why",
"do",
"programmers",
"prefer",
"using",
"dark",
"mode?",
"Because",
"light",
"attracts",
"bugs",
"and",
"they",
"want",
"to",
"code",
"in",
"peace,",
"like",
"a",
"ninja",
"in",
"the",
"shadows."
"aliqua",
"Ut",
"enim",
"ad",
"minim",
"veniam",
"quis",
"nostrud",
"exercitation",
"ullamco",
"laboris",
"nisi",
"ut",
"aliquip",
"ex",
"ea",
"commodo",
"consequat",
"Duis",
"aute",
"irure",
"dolor",
"in",
"reprehenderit",
"in",
"voluptate",
"velit",
"esse",
"cillum",
"dolore",
"eu",
"fugiat",
"nulla",
"pariatur",
"Excepteur",
"sint",
"occaecat",
"cupidatat",
"non",
"proident",
"sunt",
"in",
"culpa",
"qui",
"officia",
"deserunt",
"mollit",
"anim",
"id",
"est",
"laborum"};
return words[taosRand() % MAX_WORDS];
}
@ -197,7 +512,7 @@ char *getRandomWord() {
int64_t fillBufferWithRandomWords(char *buffer, int64_t maxBufferSize) {
int64_t len = 0;
while (len < maxBufferSize) {
char * word = getRandomWord();
char *word = getRandomWord();
size_t wordLen = strlen(word);
if (len + wordLen + 1 < maxBufferSize) {
@ -246,11 +561,11 @@ TEST(osTest, osFilePerformance) {
int64_t OpenForWriteCloseFileCost;
int64_t OpenForReadCloseFileCost;
char * buffer;
char * writeBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE);
char * readBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE);
char *buffer;
char *writeBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE);
char *readBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE);
int64_t size = fillBufferWithRandomWords(writeBuffer, MAX_TEST_FILE_SIZE);
char * fname = "./osFilePerformanceTest.txt";
char *fname = "./osFilePerformanceTest.txt";
TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
ASSERT_NE(pOutFD, nullptr);
@ -342,6 +657,6 @@ TEST(osTest, osFilePerformance) {
(void)printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost);
}
#endif // OSFILE_PERFORMANCE_TEST
#endif // OSFILE_PERFORMANCE_TEST
#pragma GCC diagnostic pop

40
source/os/test/osThreadTests.cpp
View File

@ -110,7 +110,7 @@ TEST(osThreadTests, thread) {
taosMsleep(300);
(void)taosThreadCancel(tid1);
reti = taosThreadCreate(&tid2, NULL, funcPtr501, NULL);
EXPECT_EQ(reti, 0);
taosMsleep(1000);
@ -498,4 +498,42 @@ TEST(osThreadTests, spinlock) {
TEST(osThreadTests, others) {
taosThreadTestCancel();
taosThreadClear(NULL);
TdThread tid1 = {0};
TdThread tid2 = {0};
TdThread *thread = &tid1;
taosResetPthread(NULL);
taosResetPthread(thread);
bool retb = taosComparePthread(tid1, tid2);
EXPECT_TRUE(retb);
int32_t ret32 = taosGetAppName(NULL, NULL);
EXPECT_NE(ret32, 0);
char name[32] = {0};
int32_t pid;
ret32 = taosGetPIdByName(name, NULL);
EXPECT_NE(ret32, 0);
ret32 = taosGetPIdByName(NULL, &pid);
EXPECT_NE(ret32, 0);
ret32 = tsem_timewait(NULL, 124);
EXPECT_NE(ret32, 0);
ret32 = tsem_wait(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem2_init(NULL, 0, 0);
EXPECT_NE(ret32, 0);
ret32 = tsem_post(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem_destroy(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem2_post(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem2_destroy(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem2_wait(NULL);
EXPECT_NE(ret32, 0);
ret32 = tsem2_timewait(NULL, 128);
EXPECT_NE(ret32, 0);
}

1
source/util/src/terror.c
View File

@ -564,6 +564,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for m
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TD_GPT_EXPIRED, "License expired for TDgpt function")
// sync
TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout")

1
source/util/test/errorCodeTable.ini
View File

@ -400,6 +400,7 @@ TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED = 0x8000082A
TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED = 0x8000082B
TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED = 0x8000082C
TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED = 0x8000082D
TSDB_CODE_GRANT_TD_GPT_EXPIRED = 0x8000082E
TSDB_CODE_SYN_TIMEOUT = 0x80000903
TSDB_CODE_SYN_MISMATCHED_SIGNATURE = 0x80000907
TSDB_CODE_SYN_NOT_LEADER = 0x8000090C

123
tests/army/cluster/arbitrator_restart.py Normal file
View File

@ -0,0 +1,123 @@
import taos
import sys
import os
import subprocess
import glob
import shutil
import time
from frame.log import *
from frame.cases import *
from frame.sql import *
from frame.srvCtl import *
from frame.caseBase import *
from frame import *
from frame.autogen import *
from frame import epath
# from frame.server.dnodes import *
# from frame.server.cluster import *
class TDTestCase(TBase):
def init(self, conn, logSql, replicaVar=1):
updatecfgDict = {'dDebugFlag':131}
super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1")
self.valgrind = 0
self.db = "test"
self.stb = "meters"
self.childtable_count = 10
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;')
time.sleep(1)
tdSql.execute("use db;")
tdSql.execute("CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);")
tdSql.execute("CREATE TABLE d0 USING meters TAGS (\"California.SanFrancisco\", 2);");
count = 0
while count < 100:
tdSql.query("show arbgroups;")
if tdSql.getData(0, 4) == True:
break
tdLog.info("wait 1 seconds for is sync")
time.sleep(1)
count += 1
if count == 100:
tdLog.exit("arbgroup sync failed")
return
sc.dnodeStop(2)
sc.dnodeStop(3)
sc.dnodeStart(2)
count = 0
while count < 100:
tdSql.query("show db.vgroups;")
if(tdSql.getData(0, 4) == "candidate") or (tdSql.getData(0, 6) == "candidate"):
break
tdLog.info("wait 1 seconds for candidate")
time.sleep(1)
count += 1
if count == 100:
tdLog.exit("wait candidate failed")
return
tdSql.execute("ASSIGN LEADER FORCE;")
count = 0
while count < 100:
tdSql.query("show db.vgroups;")
if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "):
break
tdLog.info("wait 1 seconds for set assigned")
time.sleep(1)
count += 1
if count == 100:
tdLog.exit("assign leader failed")
return
tdSql.execute("INSERT INTO d0 VALUES (NOW, 10.3, 219, 0.31);")
sc.dnodeStart(3)
count = 0
while count < 100:
tdSql.query("show arbgroups;")
if tdSql.getData(0, 4) == 1:
break
tdLog.info("wait 1 seconds for is sync")
time.sleep(1)
count += 1
if count == 100:
tdLog.exit("arbgroup sync failed")
return
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

1
tests/army/cmdline/data/pwd.txt Normal file
View File

@ -0,0 +1 @@
taosdata

37
tests/army/cmdline/data/source.sql Normal file
View File

@ -0,0 +1,37 @@
drop database if exists db;
create database db ;
create table db.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table db.d0 using db.meters tags(1, 'china');
insert into db.d0 values(1700000000001,1,1,1);
insert into db.d0 values(1700000000002,2,2,2);
insert into db.d0 values(1700000000003,3,3,4);
insert into db.d0 values(1700000000004,4,4,4);
insert into db.d0 values(1700000000005,5,5,5);
show db.tables;
set max_binary_display_width 100;
explain select * from db.meters;
show dnodes;
desc db.d0;
select * from db.meters limit 2;
drop database if exists dbus;
create database dbus precision 'us';
create table dbus.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table dbus.d0 using dbus.meters tags(1, 'china');
insert into dbus.d0 values(1700000000000001,1,1,1);
insert into dbus.d0 values(1700000000000002,2,2,2);
insert into dbus.d0 values(1700000000000003,3,3,3);
insert into dbus.d0 values(1700000000000004,4,4,4);
insert into dbus.d0 values(1700000000000005,5,5,5);
select * from dbus.meters;
select * from dbus.meters\G;
drop database if exists duns;
create database duns precision 'ns';
create table duns.meters(ts timestamp, current float, voltage int, phase float) tags(groupid int, location varchar(24));
create table duns.d0 using duns.meters tags(1, 'china');
insert into duns.d0 values(1700000000000000001,1,1,1);
insert into duns.d0 values(1700000000000000002,2,2,2);
insert into duns.d0 values(1700000000000000003,3,3,4);
insert into duns.d0 values(1700000000000000004,4,4,4);
insert into duns.d0 values(1700000000000000005,5,5,5);
select * from duns.meters;
select * from duns.meters\G;

14
tests/army/cmdline/fullopt.py
View File

@ -30,7 +30,7 @@ class TDTestCase(TBase):
updatecfgDict = {
'queryMaxConcurrentTables': '2K',
'streamMax': '1M',
'totalMemoryKB': '1G',
'totalMemoryKB': '32000000',
'streamMax': '1P',
'streamBufferSize':'1T',
'slowLogScope':"query"
@ -47,10 +47,6 @@ class TDTestCase(TBase):
# taosBenchmark run
etool.benchMark(command = f"-d {self.db} -t {self.childtable_count} -n {self.insert_rows} -v 2 -y")
def checkQueryOK(self, rets):
if rets[-2][:9] != "Query OK,":
tdLog.exit(f"check taos -s return unexpect: {rets}")
def doTaos(self):
tdLog.info(f"check taos command options...")
@ -71,10 +67,10 @@ class TDTestCase(TBase):
]
# exec
for option in options:
rets = etool.runBinFile("taos", f"-s \"alter local '{option}'\";")
self.checkQueryOK(rets)
rlist = self.taos(f"-s \"alter local '{option}'\"")
self.checkListString(rlist, "Query OK,")
# error
etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\";")
etool.runBinFile("taos", f"-s \"alter local 'nocmd check'\"")
# help
rets = etool.runBinFile("taos", "--help")
@ -158,6 +154,8 @@ class TDTestCase(TBase):
sc.dnodeStop(idx)
etool.exeBinFile("taos", f'-n server', wait=False)
time.sleep(3)
rlist = self.taos("-n client")
self.checkListString(rlist, "total succ: 100/100")
eos.exe("pkill -9 taos")
# call enter password

55
tests/army/cmdline/json/taosCli.json
View File

@ -23,32 +23,45 @@
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 1,
"insert_rows":10,
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "int", "count": 1000, "max": 2000, "min": 0 }
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{
"type":"binary",
"name":"location",
"max":64,
"min":1,
"values":[
"San Francisco",
"Los Angles",
"San Diego",
"San Jose",
"Palo Alto",
"Campbell",
"Mountain View",
"Sunnyvale",
"Santa Clara",
"Cupertino"
]
}
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]

57
tests/army/cmdline/json/taosCliDesc.json Normal file
View File

@ -0,0 +1,57 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 1,
"insert_rows":10,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "int", "count": 1000, "max": 2000, "min": 0 }
],
"tags":[
{
"type":"binary",
"name":"location",
"max":64,
"min":1,
"values":[
"San Francisco",
"Los Angles",
"San Diego",
"San Jose",
"Palo Alto",
"Campbell",
"Mountain View",
"Sunnyvale",
"Santa Clara",
"Cupertino"
]
}
]
}
]
}
]
}

178
tests/army/cmdline/taosCli.py
View File

@ -42,7 +42,7 @@ class TDTestCase(TBase):
tdLog.info(f"check describe show full.")
# insert
json = "cmdline/json/taosCli.json"
json = "cmdline/json/taosCliDesc.json"
db, stb, childCount, insertRows = self.insertBenchJson(json)
# describe
sql = f"describe {db}.{stb};"
@ -53,6 +53,167 @@ class TDTestCase(TBase):
tdSql.query(sql)
tdSql.checkRows(2 + 1000)
def checkResultWithMode(self, db, stb, arg):
result = "Query OK, 10 row(s)"
mode = arg[0]
rowh = arg[1]
rowv = arg[2]
idx = arg[3]
idxv = arg[4]
# use db
if mode != "-R":
rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ')
self.checkListString(rlist, "Database changed")
# hori
cmd = f'{mode} -s "select * from {db}.{stb} limit 10'
rlist = self.taos(cmd + '"')
# line count
self.checkSame(len(rlist), rowh)
# last line
self.checkSame(rlist[idx][:len(result)], result)
# vec
rlist = self.taos(cmd + '\G"')
# line count
self.checkSame(len(rlist), rowv)
self.checkSame(rlist[idxv], "*************************** 10.row ***************************")
# last line
self.checkSame(rlist[idx][:len(result)], result)
# -B have some problem need todo
self.taos(f'{mode} -B -s "select * from {db}.{stb} where ts < 1"')
# get empty result
rlist = self.taos(f'{mode} -r -s "select * from {db}.{stb} where ts < 1"')
self.checkListString(rlist, "Query OK, 0 row(s) in set")
def checkBasic(self):
tdLog.info(f"check describe show full.")
# insert
json = "cmdline/json/taosCli.json"
db, stb, childCount, insertRows = self.insertBenchJson(json)
# native restful websock test
args = [
["", 18, 346, -2, 310],
["-R", 22, 350, -3, 313],
["-T 40 -E http://localhost:6041", 21, 349, -3, 312]
]
for arg in args:
self.checkResultWithMode(db, stb, arg)
def checkDumpInOutMode(self, source, arg, db, insertRows):
mode = arg[0]
self.taos(f'{mode} -s "source {source}" ')
self.taos(f'{mode} -s "select * from {db}.d0>>d0.csv" ')
# use db
rlist = self.taos(f'{mode} -s "show databases;use {db};show databases;" ')
# update sql
rlist = self.taos(f'{mode} -s "alter local \'resetlog\';" ')
self.checkListString(rlist, "Query O")
# only native support csv import
if mode == "":
self.taos(f'{mode} -s "delete from {db}.d0" ')
self.taos(f'{mode} -s "insert into {db}.d0 file d0.csv" ')
sql = f"select count(*) from {db}.d0"
self.taos(f'{mode} -B -s "{sql}" ')
tdSql.checkAgg(sql, insertRows)
sql = f"select first(voltage) from {db}.d0"
tdSql.checkFirstValue(sql, 1)
sql = f"select last(voltage) from {db}.d0"
tdSql.checkFirstValue(sql, 5)
def checkDumpInOut(self):
args = [
["", 18],
["-R ", 22],
["-E http://localhost:6041", 21]
]
source = "cmdline/data/source.sql"
db = "db"
insertRows = 5
for arg in args:
# insert
self.checkDumpInOutMode(source, arg, db, insertRows)
def checkVersion(self):
rlist1 = self.taos("-V")
rlist2 = self.taos("--version")
self.checkSame(rlist1, rlist2)
self.checkSame(len(rlist1), 5)
if len(rlist1[2]) < 42:
tdLog.exit("git commit id length is invalid: " + rlist1[2])
def checkHelp(self):
# help
rlist1 = self.taos("--help")
rlist2 = self.taos("-?")
self.checkSame(rlist1, rlist2)
# check return
strings = [
"--auth=AUTH",
"--database=DATABASE",
"--version",
" --help"
]
for string in strings:
self.checkListString(rlist1, string)
def checkCommand(self):
# check coredump
# o logpath
char = 'a'
lname =f'-o "/root/log/{char * 1000}/" -s "quit;"'
queryOK = "Query OK"
# invalid input check
args = [
[lname, "failed to create log at"],
['-uroot -w 40 -ptaosdata -c /root/taos/ -s"show databases"', queryOK],
['-o "./current/log/files/" -s"show databases;"', queryOK],
['-a ""', "Invalid auth"],
['-s "quit;"', "Welcome to the TDengine Command Line Interface"],
['-a "abc"', "[0x80000357]"],
['-h "" -s "show dnodes;"', "Invalid host"],
['-u "" -s "show dnodes;"', "Invalid user"],
['-P "" -s "show dnodes;"', "Invalid port"],
['-u "AA" -s "show dnodes;"', "failed to connect to server"],
['-p"abc" -s "show dnodes;"', "[0x80000357]"],
['-d "abc" -s "show dnodes;"', "[0x80000388]"],
['-N 0 -s "show dnodes;"', "Invalid pktNum"],
['-N 10 -s "show dnodes;"', queryOK],
['-w 0 -s "show dnodes;"', "Invalid displayWidth"],
['-w 10 -s "show dnodes;"', queryOK],
['-W 10 -s "show dnodes;"', None],
['-l 0 -s "show dnodes;"', "Invalid pktLen"],
['-l 10 -s "show dnodes;"', queryOK],
['-C', "buildinfo"],
['-B -s "show dnodes;"', queryOK],
['-s "help;"', "Timestamp expression Format"],
['-s ""', "Invalid commands"],
['-t', "2: service ok"],
['-uroot -p < cmdline/data/pwd.txt -s "show dnodes;"', queryOK],
]
for arg in args:
rlist = self.taos(arg[0])
if arg[1] != None:
self.checkListString(rlist, arg[1])
# run
def run(self):
tdLog.debug(f"start to excute {__file__}")
@ -60,6 +221,21 @@ class TDTestCase(TBase):
# check show whole
self.checkDescribe()
# check basic
self.checkBasic()
# version
self.checkVersion()
# help
self.checkHelp()
# check command
self.checkCommand()
# check data in/out
self.checkDumpInOut()
tdLog.success(f"{__file__} successfully executed")

48
tests/army/frame/caseBase.py
View File

@ -19,6 +19,7 @@ import random
import copy
import json
import frame.etool
import frame.eutil
from frame.log import *
from frame.sql import *
@ -231,6 +232,14 @@ class TBase:
tdLog.info("sql1 same result with sql2.")
# check same value
def checkSame(self, real, expect, show = True):
if real == expect:
if show:
tdLog.info(f"check same succ. real={real} expect={expect}.")
else:
tdLog.exit(f"check same failed. real={real} expect={expect}.")
#
# get db information
#
@ -270,6 +279,16 @@ class TBase:
print(dics)
return dics
#
# run bin file
#
# taos
def taos(self, command, show = True, checkRun = False):
return frame.etool.runBinFile("taos", command, show, checkRun)
def taosdump(self, command, show = True, checkRun = True, retFail = True):
return frame.etool.runBinFile("taosdump", command, show, checkRun, retFail)
#
# util
@ -312,6 +331,17 @@ class TBase:
tdLog.exit(f"list is empty {tips}")
# check list have str
def checkListString(self, vlist, s):
for i in range(len(vlist)):
if vlist[i].find(s) != -1:
# found
tdLog.info(f'found "{s}" on index {i} , line={vlist[i]}')
return
# not found
tdLog.exit(f'faild, not found "{s}" on list:{vlist}')
#
# str util
#
@ -328,7 +358,7 @@ class TBase:
# taosBenchmark
#
# run taosBenchmark and check insert Result
# insert
def insertBenchJson(self, jsonFile, options="", checkStep=False):
# exe insert
cmd = f"{options} -f {jsonFile}"
@ -395,4 +425,18 @@ class TBase:
if vgroups != None:
tdSql.checkData(0, 0, vgroups)
return db, stb,child_count, insert_rows
return db, stb, child_count, insert_rows
# tmq
def tmqBenchJson(self, jsonFile, options="", checkStep=False):
# exe insert
command = f"{options} -f {jsonFile}"
rlist = frame.etool.runBinFile("taosBenchmark", command, checkRun = True)
#
# check insert result
#
print(rlist)
return rlist

66
tests/army/frame/eos.py
View File

@ -52,8 +52,11 @@ def isArm64Cpu():
#
# wait util execute file finished
def exe(file):
return os.system(file)
def exe(command, show = False):
code = os.system(command)
if show:
print(f"eos.exe retcode={code} command:{command}")
return code
# execute file and return immediately
def exeNoWait(file):
@ -64,24 +67,63 @@ def exeNoWait(file):
return exe(cmd)
# run return output and error
def run(command, timeout = 10):
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait(timeout)
def run(command, show = True):
# out to file
id = time.clock_gettime_ns(time.CLOCK_REALTIME) % 100000
out = f"out_{id}.txt"
err = f"err_{id}.txt"
code = exe(command + f" 1>{out} 2>{err}", show)
output = process.stdout.read().decode(encoding="gbk")
error = process.stderr.read().decode(encoding="gbk")
# read from file
output = readFileContext(out)
error = readFileContext(err)
return output, error
# del
if os.path.exists(out):
os.remove(out)
if os.path.exists(err):
os.remove(err)
return output, error, code
# return list after run
def runRetList(command, timeout=10):
output,error = run(command, timeout)
return output.splitlines()
def runRetList(command, show = True, checkRun = False, retFail = False):
output, error, code = run(command, show)
if checkRun and code != 0:
print(f"eos.runRetList checkRun return code failed. code={code} error={error}")
assert code == 0
rList = output.splitlines()
if retFail and error != "":
rList += error.splitlines()
return rList
#
# file
#
def delFile(file):
return exe(f"rm -rf {file}")
return exe(f"rm -rf {file}")
def readFileContext(filename):
file = open(filename)
context = file.read()
file.close()
return context
def writeFileContext(filename, context):
file = open(filename, "w")
file.write(context)
file.close()
def appendFileContext(filename, context):
global resultContext
resultContext += context
try:
file = open(filename, "a")
wsize = file.write(context)
file.close()
except:
print(f"appand file error context={context} .")

6
tests/army/frame/etool.py
View File

@ -51,7 +51,7 @@ def benchMark(command = "", json = "") :
# run
if command != "":
frame.eos.exe(bmFile + " " + command)
status = frame.eos.run(bmFile + " " + command)
if json != "":
cmd = f"{bmFile} -f {json}"
print(cmd)
@ -66,7 +66,7 @@ def curFile(fullPath, filename):
# run build/bin file
def runBinFile(fname, command, show=True):
def runBinFile(fname, command, show = True, checkRun = False, retFail = False ):
binFile = frame.epath.binFile(fname)
if frame.eos.isWin():
binFile += ".exe"
@ -74,7 +74,7 @@ def runBinFile(fname, command, show=True):
cmd = f"{binFile} {command}"
if show:
tdLog.info(cmd)
return frame.eos.runRetList(cmd)
return frame.eos.runRetList(cmd, show, checkRun, retFail)
# exe build/bin file
def exeBinFile(fname, command, wait=True, show=True):

71
tests/army/tools/benchmark/basic/json/tmqBasicInsert.json Normal file
View File

@ -0,0 +1,71 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count": 2,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes",
"vgroups": 4
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 10,
"insert_rows": 100,
"timestamp_step": 1,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]
}
]
}

29
tests/army/tools/benchmark/basic/json/tmqBasicPara.json Normal file
View File

@ -0,0 +1,29 @@
{
"filetype": "subscribe",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"databases": "test",
"confirm_parameter_prompt": "no",
"result_file": "./tmq_result_para.txt",
"tmq_info": {
"concurrent": 2,
"poll_delay": 3000,
"create_mode": "parallel",
"group_mode": "independent",
"client.id": "clientId",
"auto.offset.reset": "earliest",
"enable.auto.commit": "true",
"auto.commit.interval.ms": 1000,
"enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "false",
"rows_file": "./rows_file_para",
"expect_rows": 100,
"topic_list": [
{"name": "topic_benchmark_d0", "sql": "select * from test.d0;"}
]
}
}

13
tests/army/tools/benchmark/basic/json/tmq.json → tests/army/tools/benchmark/basic/json/tmqBasicSequ.json
View File

@ -5,12 +5,13 @@
"port": 6030,
"user": "root",
"password": "taosdata",
"databases": "db",
"databases": "test",
"confirm_parameter_prompt": "no",
"result_file": "./tmq_result1.txt",
"result_file": "./tmq_result_sequ.txt",
"tmq_info": {
"concurrent": 2,
"concurrent": 4,
"poll_delay": 3000,
"group_mode": "independent",
"group.id": "grpId_0",
"client.id": "clientId",
"auto.offset.reset": "earliest",
@ -19,10 +20,10 @@
"enable.heartbeat.background": "true",
"experimental.snapshot.enable": "true",
"msg.with.table.name": "false",
"rows_file": "./consumed_rows1",
"expect_rows": 50,
"rows_file": "./rows_file_sequ",
"expect_rows": 1000,
"topic_list": [
{"name": "tmq_topic_0", "sql": "select c0 from db.stb;"}
{"name": "topic_benchmark_meters", "sql": "select * from test.meters;"}
]
}
}

37
tests/army/tools/benchmark/basic/default_tmq_json.py → tests/army/tools/benchmark/basic/tmqBasic.py
View File

@ -25,37 +25,22 @@ from frame import *
class TDTestCase(TBase):
def caseDescription(self):
"""
[TD-11510] taosBenchmark test cases
taosBenchmark tmp->Basic test cases
"""
def run(self):
tdSql.execute("drop topic if exists topic_0")
binPath = etool.benchMarkFile()
cmd = "%s -f ./tools/benchmark/basic/json/default.json" % binPath
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("reset query cache")
# insert data
json = "tools/benchmark/basic/json/tmqBasicInsert.json"
db, stb, child_count, insert_rows = self.insertBenchJson(json, checkStep = True)
tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000")
# tmq Sequ
json = "tools/benchmark/basic/json/tmqBasicSequ.json"
self.tmqBenchJson(json)
cmd = "%s -f ./tools/benchmark/basic/json/tmq.json " % binPath
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
sleep(15)
# try:
# for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"):
# fields = line.split()
# pid = fields[0]
# os.kill(int(pid), signal.SIGINT)
# time.sleep(3)
# print("taosBenchmark be killed on purpose")
# except:
# tdLog.exit("failed to kill taosBenchmark")
# tmq Parallel
json = "tools/benchmark/basic/json/tmqBasicPara.json"
self.tmqBenchJson(json)
def stop(self):
tdSql.close()

66
tests/army/tools/taosdump/native/json/insertFullType.json Normal file
View File

@ -0,0 +1,66 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"test",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8}
]
}
]
}
]
}

70
tests/army/tools/taosdump/native/json/insertOther.json Normal file
View File

@ -0,0 +1,70 @@
{
"filetype":"insert",
"cfgdir":"/etc/taos",
"host":"127.0.0.1",
"port":6030,
"user":"root",
"password":"taosdata",
"thread_count":1,
"create_table_thread_count":1,
"confirm_parameter_prompt":"no",
"prepare_rand":100,
"num_of_records_per_req":100,
"databases": [
{
"dbinfo":{
"name":"testother",
"drop":"yes"
},
"super_tables":[
{
"name":"meters",
"child_table_exists":"no",
"childtable_prefix":"d",
"data_source":"rand",
"insert_mode":"taosc",
"childtable_count": 2,
"insert_rows":100,
"timestamp_step":1000,
"start_timestamp":"2022-10-01 00:00:00.000",
"columns":[
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "si", "max": 100, "min": -50 },
{ "type": "int", "name": "ic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "bi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "bin", "len": 4},
{ "type": "nchar", "name": "nch", "len": 8},
{ "type": "varbinary", "name": "vab", "len": 8},
{ "type": "varchar", "name": "vac", "len": 8},
{ "type": "geometry", "name": "geo", "len": 32}
],
"tags":[
{ "type": "bool", "name": "tbc"},
{ "type": "float", "name": "tfc", "max": 1, "min": 0 },
{ "type": "double", "name": "tdc", "max": 10, "min": 0 },
{ "type": "tinyint", "name": "tti", "max": 100, "min": -100 },
{ "type": "smallint", "name": "tsi", "max": 100, "min": -50 },
{ "type": "int", "name": "tic", "max": 1000, "min": -1000 },
{ "type": "bigint", "name": "tbi", "max": 100, "min": -1000 },
{ "type": "utinyint", "name": "tuti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "tusi", "max": 100, "min": 0 },
{ "type": "uint", "name": "tui", "max": 1000, "min": 0 },
{ "type": "ubigint", "name": "tubi", "max": 10000, "min": 0 },
{ "type": "binary", "name": "tbin", "len": 4},
{ "type": "nchar", "name": "tnch", "len": 8},
{ "type": "varbinary", "name": "tvab", "len": 8},
{ "type": "varchar", "name": "tvac", "len": 8},
{ "type": "geometry", "name": "tgeo", "len": 32}
]
}
]
}
]
}

144
tests/army/tools/taosdump/native/taosdumpCommandline.py
View File

@ -14,6 +14,7 @@
import os
import json
import frame
import frame.eos
import frame.etool
from frame.log import *
from frame.cases import *
@ -28,23 +29,19 @@ class TDTestCase(TBase):
test taosdump support commandline arguments
"""
def exec(self, command):
tdLog.info(command)
return os.system(command)
def clearPath(self, path):
os.system("rm -rf %s/*" % path)
def findPrograme(self):
# taosdump
taosdump = etool.taosDumpFile()
taosdump = frame.etool.taosDumpFile()
if taosdump == "":
tdLog.exit("taosdump not found!")
else:
tdLog.info("taosdump found in %s" % taosdump)
# taosBenchmark
benchmark = etool.benchMarkFile()
benchmark = frame.etool.benchMarkFile()
if benchmark == "":
tdLog.exit("benchmark not found!")
else:
@ -60,7 +57,7 @@ class TDTestCase(TBase):
return taosdump, benchmark,tmpdir
def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval=False):
def checkCorrectWithJson(self, jsonFile, newdb = None, checkInterval = True):
#
# check insert result
#
@ -73,9 +70,9 @@ class TDTestCase(TBase):
else:
db = newdb
stb = data["databases"][0]["super_tables"][0]["name"]
child_count = data["databases"][0]["super_tables"][0]["childtable_count"]
insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"]
stb = data["databases"][0]["super_tables"][0]["name"]
child_count = data["databases"][0]["super_tables"][0]["childtable_count"]
insert_rows = data["databases"][0]["super_tables"][0]["insert_rows"]
timestamp_step = data["databases"][0]["super_tables"][0]["timestamp_step"]
tdLog.info(f"get json: db={db} stb={stb} child_count={child_count} insert_rows={insert_rows} \n")
@ -91,15 +88,9 @@ class TDTestCase(TBase):
tdSql.query(sql)
tdSql.checkRows(0)
def testBenchmarkJson(self, benchmark, jsonFile, options="", checkInterval=False):
# exe insert
cmd = f"{benchmark} {options} -f {jsonFile}"
self.exec(cmd)
self.checkCorrectWithJson(jsonFile)
def insertData(self, benchmark, json, db):
def insertData(self, json):
# insert super table
self.testBenchmarkJson(benchmark, json)
db, stb, child_count, insert_rows = self.insertBenchJson(json)
# normal table
sqls = [
@ -112,14 +103,8 @@ class TDTestCase(TBase):
]
for sql in sqls:
tdSql.execute(sql)
def dumpOut(self, taosdump, db , outdir):
# dump out
self.exec(f"{taosdump} -D {db} -o {outdir}")
def dumpIn(self, taosdump, db, newdb, indir):
# dump in
self.exec(f'{taosdump} -W "{db}={newdb}" -i {indir}')
return db, stb, child_count, insert_rows
def checkSame(self, db, newdb, stb, aggfun):
# sum pk db
@ -136,31 +121,75 @@ class TDTestCase(TBase):
else:
tdLog.exit(f"{aggfun} source db:{sum1} import db:{sum2} not equal.")
def verifyResult(self, db, newdb, json):
# compare with insert json
self.checkCorrectWithJson(json, newdb)
# compare sum(pk)
stb = "meters"
self.checkSame(db, newdb, stb, "sum(pk)")
self.checkSame(db, newdb, stb, "sum(usi)")
self.checkSame(db, newdb, stb, "sum(fc)")
self.checkSame(db, newdb, stb, "sum(ti)")
self.checkSame(db, newdb, stb, "sum(si)")
self.checkSame(db, newdb, stb, "sum(ic)")
self.checkSame(db, newdb, stb, "avg(bi)")
self.checkSame(db, newdb, stb, "sum(uti)")
self.checkSame(db, newdb, stb, "sum(usi)")
self.checkSame(db, newdb, stb, "sum(ui)")
self.checkSame(db, newdb, stb, "avg(ubi)")
# check normal table
self.checkSame(db, newdb, "ntb", "sum(c1)")
# basic commandline
def basicCommandLine(self, taosdump, tmpdir):
# -h -P -u -p -o
self.exec(taosdump + f" -h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}")
# with Native Rest and WebSocket
def dumpInOutMode(self, mode, db, json, tmpdir):
# dump out
self.clearPath(tmpdir)
self.taosdump(f"{mode} -D {db} -o {tmpdir}")
# dump in
newdb = "new" + db
self.taosdump(f"{mode} -W '{db}={newdb}' -i {tmpdir}")
# check same
self.verifyResult(db, newdb, json)
# basic commandline
def basicCommandLine(self, tmpdir):
#command and check result
checkItems = [
[f"-h 127.0.0.1 -P 6030 -uroot -ptaosdata -A -N -o {tmpdir}", ["OK: Database test dumped"]],
[f"-r result -a -e test d0 -o {tmpdir}", ["OK: table: d0 dumped", "OK: 100 row(s) dumped out!"]],
[f"-n -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]],
[f"-L -D test -o {tmpdir}", ["OK: Database test dumped", "OK: 205 row(s) dumped out!"]],
[f"-s -D test -o {tmpdir}", ["dumping out schema: 1 from meters.d0", "OK: Database test dumped", "OK: 0 row(s) dumped out!"]],
[f"-N -d deflate -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f"-N -d lzma -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f"-N -d snappy -S '2022-10-01 00:00:50.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 100 row(s) dumped out!"]],
[f" -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}",["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]],
[f"-T 2 -B 1000 -S '2022-10-01 00:00:50.000' -E '2022-10-01 00:00:60.000' test meters -o {tmpdir}", ["OK: table: meters dumped", "OK: 22 row(s) dumped out!"]],
[f"-g -E '2022-10-01 00:00:60.000' test -o {tmpdir}", ["OK: Database test dumped", "OK: 122 row(s) dumped out!"]],
[f"--help", ["Report bugs to"]],
[f"-?", ["Report bugs to"]],
[f"-V", ["version:"]],
[f"--usage", ["taosdump [OPTION...] -o outpath"]]
]
# executes
for item in checkItems:
self.clearPath(tmpdir)
command = item[0]
results = item[1]
rlist = self.taosdump(command)
for result in results:
self.checkListString(rlist, result)
# clear tmp
# check except
def checkExcept(self, command):
try:
code = self.exec(command)
code = frame.eos.exe(command, show = True)
if code == 0:
tdLog.exit(f"Failed, not report error cmd:{command}")
else:
@ -170,7 +199,7 @@ class TDTestCase(TBase):
# except commandline
def exceptCommandLine(self, taosdump, tmpdir):
def exceptCommandLine(self, taosdump, db, stb, tmpdir):
# -o
self.checkExcept(taosdump + " -o= ")
self.checkExcept(taosdump + " -o")
@ -178,34 +207,47 @@ class TDTestCase(TBase):
self.checkExcept(taosdump + " -A -o ")
self.checkExcept(taosdump + " -A -o ./noexistpath/")
self.checkExcept(taosdump + f" -d invalidAVRO -o {tmpdir}")
self.checkExcept(taosdump + f" -d unknown -o {tmpdir}")
self.checkExcept(taosdump + f" -P invalidport")
self.checkExcept(taosdump + f" -D")
self.checkExcept(taosdump + f" -P 65536")
self.checkExcept(taosdump + f" -t 2 -k 2 -z 1 -C https://not-exist.com:80/cloud -D test -o {tmpdir}")
self.checkExcept(taosdump + f" -P 65536")
# run
def run(self):
# database
db = "pridb"
newdb = "npridb"
# find
taosdump, benchmark, tmpdir = self.findPrograme()
json = "./tools/taosdump/ws/json/primaryKey.json"
json = "./tools/taosdump/native/json/insertFullType.json"
# insert data with taosBenchmark
self.insertData(benchmark, json, db)
db, stb, childCount, insertRows = self.insertData(json)
# dumpInOut
modes = ["", "-R" , "--cloud=http://localhost:6041"]
for mode in modes:
self.dumpInOutMode(mode, db , json, tmpdir)
tdLog.info("1. native rest ws dumpIn Out .......................... [Passed]")
# basic commandline
self.basicCommandLine(taosdump, tmpdir)
self.basicCommandLine(tmpdir)
tdLog.info("2. basic command line .................................. [Passed]")
# except commandline
self.exceptCommandLine(taosdump, tmpdir)
self.exceptCommandLine(taosdump, db, stb, tmpdir)
tdLog.info("3. except command line ................................. [Passed]")
# dump out
#self.dumpOut(taosdump, db, tmpdir)
# dump in
#self.dumpIn(taosdump, db, newdb, tmpdir)
# verify db
#self.verifyResult(db, newdb, json)
#
# varbinary and geometry for native
#
json = "./tools/taosdump/native/json/insertOther.json"
# insert
db, stb, childCount, insertRows = self.insertData(json)
# dump in/out
self.dumpInOutMode("", db , json, tmpdir)
tdLog.info("4. native varbinary geometry ........................... [Passed]")
def stop(self):

2
tests/army/whole/checkErrorCode.py
View File

@ -50,7 +50,7 @@ ignoreCodes = [
'0x80000734', '0x80000735', '0x80000736', '0x80000737', '0x80000738', '0x8000080E', '0x8000080F', '0x80000810', '0x80000811', '0x80000812',
'0x80000813', '0x80000814', '0x80000815', '0x80000816', '0x80000817', '0x80000818', '0x80000819', '0x80000820', '0x80000821', '0x80000822',
'0x80000823', '0x80000824', '0x80000825', '0x80000826', '0x80000827', '0x80000828', '0x80000829', '0x8000082A', '0x8000082B', '0x8000082C',
'0x8000082D', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00',
'0x8000082D', '0x8000082E', '0x80000907', '0x80000919', '0x8000091A', '0x8000091B', '0x8000091C', '0x8000091D', '0x8000091E', '0x8000091F', '0x80000A00',
'0x80000A01', '0x80000A03', '0x80000A06', '0x80000A07', '0x80000A08', '0x80000A09', '0x80000A0A', '0x80000A0B', '0x80000A0E', '0x80002206',
'0x80002207', '0x80002406', '0x80002407', '0x80002503', '0x80002506', '0x80002507', '0x8000261B', '0x80002653', '0x80002668', '0x80002669',
'0x8000266A', '0x8000266B', '0x8000266C', '0x8000266D', '0x8000266E', '0x8000266F', '0x80002670', '0x80002671', '0x80002672', '0x80002673',

10
tests/parallel_test/cases.task
View File

@ -22,6 +22,7 @@
,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py -N 3 -M 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator_restart.py -N 3
,,n,army,python3 ./test.py -f storage/s3/s3Basic.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2
,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py
@ -70,7 +71,7 @@
,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py
,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py
,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py
,,y,army,./pytest.sh python3 ./test.py -f cmdline/taosCli.py -B
,,n,army,python3 ./test.py -f whole/checkErrorCode.py
#
@ -88,7 +89,6 @@
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/custom_col_tag.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_json.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/default_tmq_json.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/demo.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/exportCsv.py
@ -140,6 +140,8 @@
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestInsertWithJsonStmt-otherPara.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/taosdemoTestQueryWithJson.py -R
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/telnet_tcp.py -R
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmqBasic.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/basic/tmq_case.py
,,y,army,./pytest.sh python3 ./test.py -f tools/benchmark/cloud/cloud-test.py
@ -206,7 +208,7 @@
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeBool.py -B
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpRetry.py -B
,,n,army,python3 ./test.py -f tools/taosdump/ws/taosdumpTestTypeTinyInt.py -B
,,y,army,./pytest.sh python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py
,,n,army,python3 ./test.py -f tools/taosdump/native/taosdumpCommandline.py -B
#
# system test
@ -470,8 +472,6 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5776.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py

2
tests/parallel_test/cases_tdengine.task
View File

@ -298,7 +298,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_c_test.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts4563.py

2
tests/pytest/util/tserror.py
View File

@ -16,4 +16,4 @@ TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF)
TSDB_CODE_PAR_SYNTAX_ERROR = (TAOS_DEF_ERROR_CODE | 0x2600)
TSDB_CODE_PAR_INVALID_COLS_FUNCTION = (TAOS_DEF_ERROR_CODE | 0x2687)
TSDB_CODE_PAR_INVALID_COLS_FUNCTION = (TAOS_DEF_ERROR_CODE | 0x2689)

2
tests/system-test/0-others/grant.py
View File

@ -237,7 +237,7 @@ class TDTestCase:
tdLog.info(f"expireTime: {expireTime}, serviceTime: {serviceTime}")
tdSql.checkEqual(True, abs(expireTime - serviceTime - 864000) < 15)
tdSql.query(f'show grants full;')
nGrantItems = 31
nGrantItems = 32
tdSql.checkEqual(len(tdSql.queryResult), nGrantItems)
tdSql.checkEqual(tdSql.queryResult[0][2], serviceTimeStr)
for i in range(1, nGrantItems):

1
tests/system-test/0-others/information_schema.py
View File

@ -283,6 +283,7 @@ class TDTestCase:
'active_active':'Active-Active',
'dual_replica':'Dual-Replica HA',
'db_encryption':'Database Encryption',
'tdgpt':'TDgpt',
'opc_da':'OPC_DA',
'opc_ua':'OPC_UA',
'pi':'Pi',

13
tests/system-test/2-query/test_ts4467.py
View File

@ -55,6 +55,19 @@ class TDTestCase:
tdLog.debug("res2: %s" % str(res2))
assert(len(res1) == len(res2) and res1[0][0] == res2[4][0])
# test ts-5613
sql = "select cast(2<>3 as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,1)
sql = "select cast(2 not in(3) as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,1)
sql = "select cast(2 is NULL as int) from t"
tdSql.query(sql)
tdSql.checkData(0,0,0)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)

6
tests/system-test/3-enterprise/restore/kill_restore_dnode.py
View File

@ -75,6 +75,12 @@ class TDTestCase:
def restoreDnodeThread(self, p, newTdSql):
sleep(1)
count = 0
while count < 100:
newTdSql.query('show dnodes')
if newTdSql.queryResult[1][4] == "ready":
break
count+=1
sql = f"restore dnode 2"
tdLog.info(sql)

8
tests/system-test/7-tmq/tmq_ts-5473.py → tests/system-test/7-tmq/tmq_c_test.py
View File

@ -29,6 +29,14 @@ class TDTestCase:
tdLog.info(cmdStr)
os.system(cmdStr)
cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath)
tdLog.info(cmdStr)
os.system(cmdStr)
cmdStr = '%s/build/bin/tmq_td33798'%(buildPath)
tdLog.info(cmdStr)
os.system(cmdStr)
return
def stop(self):

39
tests/system-test/7-tmq/tmq_ts-5776.py
View File

@ -1,39 +0,0 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from taos.tmq import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def run(self):
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath)
tdLog.info(cmdStr)
os.system(cmdStr)
return
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

2
tools/shell/src/shellArguments.c
View File

@ -443,7 +443,7 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) {
shellInitArgs(argc, argv);
shell.info.clientVersion =
"Welcome to the %s Command Line Interface, Client Version:%s\r\n"
"Copyright (c) 2023 by %s, all rights reserved.\r\n\r\n";
"Copyright (c) 2025 by %s, all rights reserved.\r\n\r\n";
#ifdef CUS_NAME
strcpy(shell.info.cusName, CUS_NAME);
#else

1
tools/shell/src/shellAuto.c
View File

@ -112,6 +112,7 @@ SWords shellCommands[] = {
{"create view <anyword> as select", 0, 0, NULL},
{"compact database <db_name>", 0, 0, NULL},
#endif
{"desc <all_table>;", 0, 0, NULL},
{"describe <all_table>;", 0, 0, NULL},
{"delete from <all_table> where ", 0, 0, NULL},
{"drop database <db_name>;", 0, 0, NULL},

3
tools/shell/src/shellEngine.c
View File

@ -254,8 +254,7 @@ void shellRunSingleCommandImp(char *command) {
}
if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) {
fprintf(stdout, "Database changed.\r\n\r\n");
fflush(stdout);
printf("Database changed.\r\n\r\n");
// call back auto tab module
callbackAutoTab(command, pSql, true);

30
tools/shell/src/shellUtil.c
View File

@ -122,6 +122,7 @@ void shellCheckServerStatus() {
} while (1);
}
#ifdef WEBSOCKET
char dsn[1024] = "ws://localhost:6041";
void shellCheckConnectMode() {
if (shell.args.dsn) {
shell.args.cloud = true;
@ -144,21 +145,22 @@ void shellCheckConnectMode() {
shell.args.restful = false;
return;
}
if (shell.args.restful) {
if (!shell.args.host) {
shell.args.host = "localhost";
}
if (!shell.args.port) {
shell.args.port = 6041;
}
shell.args.dsn = taosMemoryCalloc(1, 1024);
snprintf(shell.args.dsn, 1024, "ws://%s:%d",
shell.args.host, shell.args.port);
}
shell.args.cloud = false;
return;
}
if (shell.args.restful) {
if (!shell.args.host) {
shell.args.host = "localhost";
}
if (!shell.args.port) {
shell.args.port = 6041;
}
shell.args.dsn = dsn;
snprintf(shell.args.dsn, 1024, "ws://%s:%d",
shell.args.host, shell.args.port);
}
shell.args.cloud = false;
return;
}
#endif

4
tools/taos-tools/inc/bench.h
View File

@ -617,10 +617,8 @@ typedef struct SDataBase_S {
int durMinute; // duration minutes
BArray *cfgs;
BArray *superTbls;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
int32_t vgroups;
BArray *vgArray;
#endif // TD_VER_COMPATIBLE_3_0_0_0
bool flush;
} SDataBase;
@ -838,9 +836,7 @@ typedef struct SThreadInfo_S {
BArray* delayList;
uint64_t *query_delay_list;
double avg_delay;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
SVGroup *vg;
#endif
int posOfTblCreatingBatch;
int posOfTblCreatingInterval;

30
tools/taos-tools/src/CMakeLists.txt
View File

@ -78,7 +78,6 @@ ADD_DEFINITIONS(-DBUILD_INFO="${BUILD_INFO}")
MESSAGE(STATUS "build:${BUILD_INFO}")
MESSAGE("")
#
# build proj
#
@ -88,15 +87,8 @@ INCLUDE_DIRECTORIES(/usr/local/taos/include)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/avro/lang/c/src)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_LIST_DIR}/../deps/toolscJson/src)
if(NOT DEFINED TD_VER_COMPATIBLE)
SET(TD_VER_COMPATIBLE "3.0.0.0")
ENDIF()
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread)
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex)
ADD_DEFINITIONS(-DTD_VER_COMPATIBLE_3_0_0_0)
ENDIF ()
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/pthread)
INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/contrib/msvcregex)
MESSAGE("Current system is: ${CMAKE_SYSTEM_NAME}")
@ -128,22 +120,14 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
ADD_DEPENDENCIES(deps-snappy apache-avro)
ADD_DEPENDENCIES(taosdump deps-jansson)
ADD_DEPENDENCIES(taosdump deps-snappy)
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE ()
INCLUDE_DIRECTORIES(/usr/local/include)
ADD_DEFINITIONS(-DDARWIN)
LINK_DIRECTORIES(/usr/local/lib)
SET(OS_ID "Darwin")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF ()
# websocket
@ -337,11 +321,7 @@ ELSE ()
SET(CMAKE_C_STANDARD 11)
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ELSE ()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF ()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchLog.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosdump taosdump.c dumpUtil.c wsdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c)
ADD_DEPENDENCIES(apache-avro tools-zlib)

3
tools/taos-tools/src/benchCommandOpt.c
View File

@ -245,9 +245,8 @@ void initArgument() {
g_arguments->trying_interval = 0;
g_arguments->iface = TAOSC_IFACE;
g_arguments->rest_server_ver_major = -1;
#ifdef TD_VER_COMPATIBLE_3_0_0_0
g_arguments->inputted_vgroups = -1;
#endif
g_arguments->mistMode = false;
initDatabase();

11
tools/taos-tools/src/benchData.c
View File

@ -812,17 +812,6 @@ static int generateRandDataSQL(SSuperTable *stbInfo, char *sampleDataBuf,
continue;
}
}
if (field->type == TSDB_DATA_TYPE_TIMESTAMP && !tag) {
n = snprintf(sampleDataBuf + pos, bufLen - pos, "now,");
if (n < 0 || n >= bufLen - pos) {
errorPrint("%s() LN%d snprintf overflow\n",
__func__, __LINE__);
return -1;
} else {
pos += n;
continue;
}
}
switch (field->type) {
case TSDB_DATA_TYPE_BOOL: {
bool boolTmp = tmpBool(field);

2
tools/taos-tools/src/benchJsonOpt.c
View File

@ -2254,7 +2254,6 @@ static int getMetaFromQueryJsonFile(tools_cJSON *json) {
return 0;
}
#ifdef TD_VER_COMPATIBLE_3_0_0_0
static int getMetaFromTmqJsonFile(tools_cJSON *json) {
int32_t code = -1;
@ -2422,7 +2421,6 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) {
TMQ_PARSE_OVER:
return code;
}
#endif
int readJsonConfig(char * file) {
int32_t code = -1;

458
tools/taos-tools/src/benchSubscribe.c
View File

@ -1,458 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the MIT license as published by the Free Software
* Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*/
#include <bench.h>
#include "benchLog.h"
extern int g_majorVersionOfClient;
static void stable_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param,
int code) {
if (res == NULL || taos_errno(res) != 0) {
errorPrint("failed to subscribe result, code:%d, reason:%s\n",
code, taos_errstr(res));
return;
}
if (param) fetchResult(res, ((threadInfo *)param)->filePath);
// tao_unsubscribe() will free result.
}
static void specified_sub_callback(TAOS_SUB *tsub, TAOS_RES *res, void *param,
int code) {
if (res == NULL || taos_errno(res) != 0) {
errorPrint("failed to subscribe result, code:%d, reason:%s\n",
code, taos_errstr(res));
return;
}
if (param) fetchResult(res, ((threadInfo *)param)->filePath);
// tao_unsubscribe() will free result.
}
static TAOS_SUB *subscribeImpl(QUERY_CLASS class, threadInfo *pThreadInfo,
char *sql, char *topic, bool restart,
uint64_t interval) {
TAOS_SUB *tsub = NULL;
if (taos_select_db(pThreadInfo->conn->taos, g_queryInfo.dbName)) {
errorPrint("failed to select database(%s)\n", g_queryInfo.dbName);
return NULL;
}
if ((SPECIFIED_CLASS == class) &&
(ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
tsub = taos_subscribe(
pThreadInfo->conn->taos, restart,
topic, sql, specified_sub_callback,
(void *)pThreadInfo,
(int)g_queryInfo.specifiedQueryInfo.subscribeInterval);
} else if ((STABLE_CLASS == class) &&
(ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
tsub = taos_subscribe(pThreadInfo->conn->taos, restart, topic, sql,
stable_sub_callback, (void *)pThreadInfo,
(int)g_queryInfo.superQueryInfo.subscribeInterval);
} else {
tsub = taos_subscribe(pThreadInfo->conn->taos,
restart, topic, sql, NULL,
NULL, (int)interval);
}
if (tsub == NULL) {
errorPrint("failed to create subscription. topic:%s, sql:%s\n",
topic, sql);
return NULL;
}
return tsub;
}
static void *specifiedSubscribe(void *sarg) {
int32_t *code = benchCalloc(1, sizeof(int32_t), false);
*code = -1;
threadInfo *pThreadInfo = (threadInfo *)sarg;
#ifdef LINUX
prctl(PR_SET_NAME, "specSub");
#endif
sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
"taosbenchmark-subscribe-%" PRIu64 "-%d", pThreadInfo->querySeq,
pThreadInfo->threadID);
SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls,
pThreadInfo->querySeq);
if (sql->result[0] !=
'\0') {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN,
"%s-%d", sql->result, pThreadInfo->threadID);
}
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
SPECIFIED_CLASS, pThreadInfo, sql->command,
g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeRestart,
g_queryInfo.specifiedQueryInfo.subscribeInterval);
if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
goto free_of_specified_subscribe;
}
// start loop to consume result
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
while ((g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq] == -1) ||
(g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq])) {
infoPrint(
"consumed[%d]: %d, endAfterConsum[%" PRId64 "]: %d\n",
pThreadInfo->threadID,
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID],
pThreadInfo->querySeq,
g_queryInfo.specifiedQueryInfo
.endAfterConsume[pThreadInfo->querySeq]);
g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] =
taos_consume(
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
if (sql->result[0] != 0) {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN,
"%s-%d", sql->result, pThreadInfo->threadID);
}
fetchResult(
g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID],
pThreadInfo->filePath);
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID]++;
if ((g_queryInfo.specifiedQueryInfo
.resubAfterConsume[pThreadInfo->querySeq] != -1) &&
(g_queryInfo.specifiedQueryInfo
.consumed[pThreadInfo->threadID] >=
g_queryInfo.specifiedQueryInfo
.resubAfterConsume[pThreadInfo->querySeq])) {
infoPrint(
"keepProgress:%d, resub specified query: %" PRIu64
"\n",
g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
pThreadInfo->querySeq);
g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] =
0;
taos_unsubscribe(
g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
g_queryInfo.specifiedQueryInfo
.tsub[pThreadInfo->threadID] = subscribeImpl(
SPECIFIED_CLASS, pThreadInfo, sql->command,
g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
g_queryInfo.specifiedQueryInfo.subscribeRestart,
g_queryInfo.specifiedQueryInfo.subscribeInterval);
if (NULL == g_queryInfo.specifiedQueryInfo
.tsub[pThreadInfo->threadID]) {
goto free_of_specified_subscribe;
}
}
}
}
*code = 0;
taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
free_of_specified_subscribe:
return code;
}
static void *superSubscribe(void *sarg) {
int32_t *code = benchCalloc(1, sizeof(int32_t), false);
*code = -1;
threadInfo *pThreadInfo = (threadInfo *)sarg;
TAOS_SUB * tsub[MAX_QUERY_SQL_COUNT] = {0};
uint64_t tsubSeq;
char * subSqlStr = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false);
#ifdef LINUX
prctl(PR_SET_NAME, "superSub");
#endif
if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
errorPrint(
"The table number(%" PRId64
") of the thread is more than max query sql count: %d\n",
pThreadInfo->ntables, MAX_QUERY_SQL_COUNT);
goto free_of_super_subscribe;
}
char topic[32] = {0};
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
sprintf(topic, "taosbenchmark-subscribe-%" PRIu64 "-%" PRIu64 "", i,
pThreadInfo->querySeq);
memset(subSqlStr, 0, TSDB_MAX_ALLOWED_SQL_LEN);
replaceChildTblName(
g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq], subSqlStr,
(int)i);
if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
snprintf(pThreadInfo->filePath, MAX_PATH_LEN, "%s-%d",
g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
pThreadInfo->threadID);
}
tsub[tsubSeq] =
subscribeImpl(STABLE_CLASS, pThreadInfo, subSqlStr, topic,
g_queryInfo.superQueryInfo.subscribeRestart,
g_queryInfo.superQueryInfo.subscribeInterval);
if (NULL == tsub[tsubSeq]) {
goto free_of_super_subscribe;
}
}
// start loop to consume result
int consumed[MAX_QUERY_SQL_COUNT];
for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
consumed[i] = 0;
}
TAOS_RES *res = NULL;
uint64_t st = 0, et = 0;
while (
(g_queryInfo.superQueryInfo.endAfterConsume == -1) ||
(g_queryInfo.superQueryInfo.endAfterConsume >
consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) {
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
continue;
}
st = toolsGetTimestampMs();
perfPrint(
"st: %" PRIu64 " et: %" PRIu64 " st-et: %" PRIu64 "\n",
st, et, (st - et));
res = taos_consume(tsub[tsubSeq]);
et = toolsGetTimestampMs();
perfPrint(
"st: %" PRIu64 " et: %" PRIu64 " delta: %" PRIu64 "\n",
st, et, (et - st));
if (res) {
if (g_queryInfo.superQueryInfo
.result[pThreadInfo->querySeq][0] != 0) {
snprintf(pThreadInfo->filePath,
MAX_PATH_LEN,
"%s-%d",
g_queryInfo.superQueryInfo
.result[pThreadInfo->querySeq],
pThreadInfo->threadID);
}
fetchResult(res, pThreadInfo->filePath);
consumed[tsubSeq]++;
if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1) &&
(consumed[tsubSeq] >=
g_queryInfo.superQueryInfo.resubAfterConsume)) {
taos_unsubscribe(
tsub[tsubSeq],
g_queryInfo.superQueryInfo.subscribeKeepProgress);
consumed[tsubSeq] = 0;
tsub[tsubSeq] = subscribeImpl(
STABLE_CLASS, pThreadInfo, subSqlStr, topic,
g_queryInfo.superQueryInfo.subscribeRestart,
g_queryInfo.superQueryInfo.subscribeInterval);
if (NULL == tsub[tsubSeq]) {
goto free_of_super_subscribe;
}
}
}
}
}
taos_free_result(res);
for (uint64_t i = pThreadInfo->start_table_from;
i <= pThreadInfo->end_table_to; i++) {
tsubSeq = i - pThreadInfo->start_table_from;
taos_unsubscribe(tsub[tsubSeq], 0);
}
*code = 0;
free_of_super_subscribe:
tmfree(subSqlStr);
return code;
}
int subscribeTestProcess() {
prompt(0);
if (REST_IFACE == g_queryInfo.iface) {
encodeAuthBase64();
}
if (0 != g_queryInfo.superQueryInfo.sqlCount) {
SBenchConn* conn = initBenchConn();
if (conn == NULL) {
return -1;
}
char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0";
if (3 == g_majorVersionOfClient) {
snprintf(cmd, SHORT_1K_SQL_BUFF_LEN,
"SELECT COUNT(*) FROM( SELECT DISTINCT(TBNAME) FROM %s.%s)",
g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName);
} else {
snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(TBNAME) FROM %s.%s",
g_queryInfo.dbName, g_queryInfo.superQueryInfo.stbName);
}
TAOS_RES *res = taos_query(conn->taos, cmd);
int32_t code = taos_errno(res);
if (code) {
printErrCmdCodeStr(cmd, code, res);
return -1;
}
TAOS_ROW row = NULL;
int num_fields = taos_num_fields(res);
TAOS_FIELD *fields = taos_fetch_fields(res);
while ((row = taos_fetch_row(res)) != NULL) {
if (0 == strlen((char *)(row[0]))) {
errorPrint("stable %s have no child table\n",
g_queryInfo.superQueryInfo.stbName);
return -1;
}
char temp[256] = {0};
taos_print_row(temp, row, fields, num_fields);
g_queryInfo.superQueryInfo.childTblCount = (int64_t)atol(temp);
}
infoPrint("%s's childTblCount: %" PRId64 "\n",
g_queryInfo.superQueryInfo.stbName,
g_queryInfo.superQueryInfo.childTblCount);
taos_free_result(res);
g_queryInfo.superQueryInfo.childTblName =
benchCalloc(g_queryInfo.superQueryInfo.childTblCount,
sizeof(char *), false);
if (getAllChildNameOfSuperTable(
conn->taos, g_queryInfo.dbName,
g_queryInfo.superQueryInfo.stbName,
g_queryInfo.superQueryInfo.childTblName,
g_queryInfo.superQueryInfo.childTblCount)) {
closeBenchConn(conn);
return -1;
}
closeBenchConn(conn);
}
pthread_t * pids = NULL;
threadInfo *infos = NULL;
pthread_t * pidsOfStable = NULL;
threadInfo *infosOfStable = NULL;
//==== create threads for query for specified table
int sqlCount = g_queryInfo.specifiedQueryInfo.sqls->size;
if (sqlCount > 0) {
pids = benchCalloc(1, sqlCount
*g_queryInfo.specifiedQueryInfo.concurrent
*sizeof(pthread_t), false);
infos = benchCalloc(1, sqlCount
*g_queryInfo.specifiedQueryInfo.concurrent
*sizeof(threadInfo), false);
for (int i = 0; i < sqlCount; i++) {
for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent;
j++) {
uint64_t seq =
i * g_queryInfo.specifiedQueryInfo.concurrent + j;
threadInfo *pThreadInfo = infos + seq;
pThreadInfo->threadID = (int)seq;
pThreadInfo->querySeq = i;
pThreadInfo->conn = initBenchConn();
pthread_create(pids + seq, NULL, specifiedSubscribe,
pThreadInfo);
}
}
for (int i = 0; i < sqlCount; i++) {
for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent;
j++) {
uint64_t seq =
i * g_queryInfo.specifiedQueryInfo.concurrent + j;
threadInfo *pThreadInfo = infos + seq;
void *result;
pthread_join(pids[seq], &result);
if (*(int32_t *)result) {
g_fail = true;
}
closeBenchConn(pThreadInfo->conn);
tmfree(result);
}
}
}
//==== create threads for super table query
if (g_queryInfo.superQueryInfo.sqlCount > 0 &&
g_queryInfo.superQueryInfo.threadCnt > 0) {
pidsOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount *
g_queryInfo.superQueryInfo.threadCnt *
sizeof(pthread_t), false);
infosOfStable = benchCalloc(1, g_queryInfo.superQueryInfo.sqlCount *
g_queryInfo.superQueryInfo.threadCnt *
sizeof(threadInfo), false);
int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
int threads = g_queryInfo.superQueryInfo.threadCnt;
int64_t a = ntables / threads;
if (a < 1) {
threads = (int)ntables;
a = 1;
}
int64_t b = 0;
if (threads != 0) {
b = ntables % threads;
}
for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
uint64_t tableFrom = 0;
for (int j = 0; j < threads; j++) {
uint64_t seq = i * threads + j;
threadInfo *pThreadInfo = infosOfStable + seq;
pThreadInfo->threadID = (int)seq;
pThreadInfo->querySeq = i;
pThreadInfo->start_table_from = tableFrom;
pThreadInfo->ntables = j < b ? a + 1 : a;
pThreadInfo->end_table_to =
j < b ? tableFrom + a : tableFrom + a - 1;
tableFrom = pThreadInfo->end_table_to + 1;
pThreadInfo->conn = initBenchConn();
pthread_create(pidsOfStable + seq, NULL, superSubscribe,
pThreadInfo);
}
}
g_queryInfo.superQueryInfo.threadCnt = threads;
for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
for (int j = 0; j < threads; j++) {
uint64_t seq = (uint64_t)i * threads + j;
threadInfo *pThreadInfo = infosOfStable + seq;
void * result;
pthread_join(pidsOfStable[seq], &result);
if (*(int32_t *)result) {
g_fail = true;
}
closeBenchConn(pThreadInfo->conn);
tmfree(result);
}
}
}
tmfree((char *)pids);
tmfree((char *)infos);
tmfree((char *)pidsOfStable);
tmfree((char *)infosOfStable);
//
if (g_fail) {
return -1;
}
return 0;
}

6
tools/taos-tools/src/benchSys.c
View File

@ -77,9 +77,7 @@ void benchPrintHelp() {
printf("%s%s%s%s\r\n", indent, "-W,", indent, BENCH_DSN);
printf("%s%s%s%s\r\n", indent, "-D,", indent, BENCH_TIMEOUT);
#endif
#ifdef TD_VER_COMPATIBLE_3_0_0_0
printf("%s%s%s%s\r\n", indent, "-v,", indent, BENCH_VGROUPS);
#endif
printf("%s%s%s%s\r\n", indent, "-V,", indent, BENCH_VERSION);
printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL);
}
@ -199,9 +197,7 @@ static struct argp_option bench_options[] = {
#endif
{"keep-trying", 'k', "NUMBER", 0, BENCH_KEEPTRYING},
{"trying-interval", 'z', "NUMBER", 0, BENCH_TRYING_INTERVAL},
#ifdef TD_VER_COMPATIBLE_3_0_0_0
{"vgroups", 'v', "NUMBER", 0, BENCH_VGROUPS},
#endif
{"version", 'V', 0, 0, BENCH_VERSION},
{"nodrop", 'Q', 0, 0, BENCH_NODROP},
{0}
@ -642,14 +638,12 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) {
g_arguments->timeout = atoi(arg);
break;
#endif
#ifdef TD_VER_COMPATIBLE_3_0_0_0
case 'v':
if (!toolsIsStringNumber(arg)) {
errorPrintReqArg2(CUS_PROMPT"Benchmark", "v");
}
g_arguments->inputted_vgroups = atoi(arg);
break;
#endif
case 'Q':
database->drop = false;
g_argFlag |= ARG_OPT_NODROP;

20
tools/taos-tools/src/taosdump.c
View File

@ -5831,26 +5831,6 @@ static int64_t dumpInAvroDataImpl(
#endif
}
} // tbName
#ifndef TD_VER_COMPATIBLE_3_0_0_0
else {
// 2.6 need call taos_stmt_set_tbname every loop
const int escapedTbNameLen = TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN + 3;
char *escapedTbName = calloc(1, escapedTbNameLen);
snprintf(escapedTbName, escapedTbNameLen, "%s%s%s",
g_escapeChar, tbName, g_escapeChar);
if (0 != taos_stmt_set_tbname(stmt, escapedTbName)) {
errorPrint("Failed to execute taos_stmt_set_tbname(%s)."
"reason: %s\n",
escapedTbName, taos_stmt_errstr(stmt));
free(escapedTbName);
freeTbNameIfLooseMode(tbName);
tbName = NULL;
continue;
}
free(escapedTbName);
}
#endif
debugPrint("%s() LN%d, count: %"PRId64"\n",
__func__, __LINE__, count);

8
utils/test/c/CMakeLists.txt
View File

@ -8,6 +8,7 @@ add_executable(tmq_td32526 tmq_td32526.c)
add_executable(tmq_td32187 tmq_td32187.c)
add_executable(tmq_ts5776 tmq_ts5776.c)
add_executable(tmq_td32471 tmq_td32471.c)
add_executable(tmq_td33798 tmq_td33798.c)
add_executable(tmq_write_raw_test tmq_write_raw_test.c)
add_executable(write_raw_block_test write_raw_block_test.c)
add_executable(sml_test sml_test.c)
@ -81,6 +82,13 @@ target_link_libraries(
PUBLIC common
PUBLIC os
)
target_link_libraries(
tmq_td33798
PUBLIC ${TAOS_LIB}
PUBLIC util
PUBLIC common
PUBLIC os
)
target_link_libraries(
tmq_td32526
PUBLIC ${TAOS_LIB}

312
utils/test/c/tmq_td33798.c Normal file
View File

@ -0,0 +1,312 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "cJSON.h"
#include "taos.h"
#include "tmsg.h"
#include "types.h"
bool batchMeta = false;
int32_t consumeIndex = 0;
static TAOS* use_db() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
return NULL;
}
TAOS_RES* pRes = taos_query(pConn, "use db_taosx");
if (taos_errno(pRes) != 0) {
printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes));
return NULL;
}
taos_free_result(pRes);
return pConn;
}
void checkBatchMeta(TAOS_RES* msg){
char* result = tmq_get_json_meta(msg);
printf("meta result: %s\n", result);
switch (consumeIndex) {
case 0:
ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct11\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct10\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]},{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}]}") == 0);
break;
default:
ASSERT(0);
break;
}
tmq_free_json_meta(result);
}
void checkNonBatchMeta(TAOS_RES* msg){
char* result = tmq_get_json_meta(msg);
printf("meta result: %s\n", result);
switch (consumeIndex) {
case 0:
ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9,\"isPrimarykey\":false,\"encode\":\"delta-i\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c1\",\"type\":4,\"isPrimarykey\":false,\"encode\":\"simple8b\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c2\",\"type\":6,\"isPrimarykey\":false,\"encode\":\"delta-d\",\"compress\":\"lz4\",\"level\":\"medium\"},{\"name\":\"c3\",\"type\":8,\"length\":16,\"isPrimarykey\":false,\"encode\":\"disabled\",\"compress\":\"zstd\",\"level\":\"medium\"}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}") == 0);
break;
case 1:
ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}") == 0);
break;
case 2:
ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}]}") == 0);
break;
case 3:
ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct11\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}]}") == 0);
break;
case 4:
ASSERT(strcmp(result, "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct10\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}") == 0);
break;
case 5:
ASSERT(strcmp(result, "{\"tmq_meta_version\":\"1.0\",\"metas\":[{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}]}") == 0);
break;
default:
ASSERT(0);
break;
}
tmq_free_json_meta(result);
}
static void msg_process(TAOS_RES* msg) {
printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg));
printf("db: %s\n", tmq_get_db_name(msg));
printf("vg: %d\n", tmq_get_vgroup_id(msg));
TAOS* pConn = use_db();
ASSERT (tmq_get_res_type(msg) == TMQ_RES_TABLE_META);
if (batchMeta){
checkBatchMeta(msg);
} else {
checkNonBatchMeta(msg);
}
taos_close(pConn);
}
int buildDatabase(TAOS* pConn, TAOS_RES* pRes) {
pRes = taos_query(pConn,
"create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 "
"nchar(8), t4 bool)");
if (taos_errno(pRes) != 0) {
printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into ct1 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')");
if (taos_errno(pRes) != 0) {
printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')");
if (taos_errno(pRes) != 0) {
printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into ct11 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')");
if (taos_errno(pRes) != 0) {
printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists ct10 using st1 tags(1000, \"ttt\", true)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
taosSsleep(1);
pRes = taos_query(pConn, "insert into ct1 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a') ct2 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a') ct3 using st1 tags(1000, \"ttt\", true) values(1626006833400, 1, 2, 'a')");
if (taos_errno(pRes) != 0) {
printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')");
if (taos_errno(pRes) != 0) {
printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
return 0;
}
int32_t init_env() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
return -1;
}
TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx");
if (taos_errno(pRes) != 0) {
printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create database if not exists db_taosx vgroups 1 wal_retention_period 3600");
if (taos_errno(pRes) != 0) {
printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "drop topic if exists topic_db");
if (taos_errno(pRes) != 0) {
printf("error in drop topic, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "drop database if exists abc1");
if (taos_errno(pRes) != 0) {
printf("error in drop db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1 wal_retention_period 3600");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
buildDatabase(pConn, pRes);
taos_close(pConn);
return 0;
}
int32_t create_topic() {
printf("create topic\n");
TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
return -1;
}
pRes = taos_query(pConn, "create topic topic_db only meta as database abc1");
if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
taos_close(pConn);
return 0;
}
void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) {
printf("commit %d tmq %p param %p\n", code, tmq, param);
}
tmq_t* build_consumer() {
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "group.id", batchMeta ? "batch" : "nonbatch");
tmq_conf_set(conf, "client.id", "my app 1");
tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "msg.with.table.name", "true");
tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "msg.consume.excluded", "1");
if (batchMeta) {
tmq_conf_set(conf, "msg.enable.batchmeta", "1");
}
tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
assert(tmq);
tmq_conf_destroy(conf);
return tmq;
}
tmq_list_t* build_topic_list() {
tmq_list_t* topic_list = tmq_list_new();
tmq_list_append(topic_list, "topic_db");
return topic_list;
}
void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
int32_t code;
if ((code = tmq_subscribe(tmq, topics))) {
fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code));
printf("subscribe err\n");
return;
}
while (1) {
TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000);
if (tmqmessage) {
msg_process(tmqmessage);
consumeIndex++;
taos_free_result(tmqmessage);
} else {
break;
}
}
code = tmq_consumer_close(tmq);
if (code)
fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code));
else
fprintf(stderr, "%% Consumer closed\n");
}
int main(int argc, char* argv[]) {
if (init_env() < 0) {
return -1;
}
create_topic();
tmq_list_t* topic_list = build_topic_list();
tmq_t* tmq = build_consumer();
basic_consume_loop(tmq, topic_list);
batchMeta = true;
consumeIndex = 0;
tmq = build_consumer();
basic_consume_loop(tmq, topic_list);
tmq_list_destroy(topic_list);
}