xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/hz-testcase

This commit is contained in:
Hongze Cheng 2024-12-27 10:26:56 +08:00
parent 34a4608c7d 757fcc70be
commit 0163948679
90 changed files with 1556 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/08-operation/04-maintenance.md
View File

@ -17,7 +17,7 @@ TDengine is designed for various writing scenarios, and many of these scenarios
```sql
COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY'];
SHOW COMPACTS [compact_id];
SHOW COMPACT [compact_id];
KILL COMPACT compact_id;
```

3
docs/en/14-reference/01-components/01-taosd.md
View File

@ -190,7 +190,8 @@ The effective value of charset is UTF-8.
|Parameter Name |Supported Version |Dynamic Modification|Description|
|-----------------------|-------------------------|--------------------|------------|
|supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5|
|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4|
|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-1024, default value 4|
|numOfCompactThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-16, default value 2|
|numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)|
|numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)|
|numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)|

3
docs/en/14-reference/03-taos-sql/24-show.md
View File

@ -304,9 +304,10 @@ Displays information about all topics in the current database.
```sql
SHOW TRANSACTIONS;
SHOW TRANSACTION [tranaction_id];
```
Displays information about transactions currently being executed in the system (these transactions are only for metadata level, not for regular tables).
Displays information about one of or all transaction(s) currently being executed in the system (these transactions are only for metadata level, not for regular tables).
## SHOW USERS

2
docs/zh/08-operation/04-maintenance.md
View File

@ -19,7 +19,7 @@ TDengine 面向多种写入场景，而很多写入场景下，TDengine 的存
```SQL
COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']；
COMPACT [db_name.]VGROUPS IN (vgroup_id1, vgroup_id2, ...) [start with 'XXXX'] [end with 'YYYY']；
SHOW COMPACTS [compact_id]；
SHOW COMPACT [compact_id]；
KILL COMPACT compact_id；
```

3
docs/zh/14-reference/01-components/01-taosd.md
View File

@ -185,7 +185,8 @@ charset 的有效值是 UTF-8。
|参数名称|支持版本|动态修改|参数含义|
|--------------------------|----------|-------------------------|-|
|supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5|
|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4|
|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-1024，默认值为 4|
|numOfCompactThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-16，默认值为 2|
|numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）|
|numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|

2
docs/zh/14-reference/03-taos-sql/03-table.md
View File

@ -6,7 +6,7 @@ description: 对表的各种管理操作
## 创建表
`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表。
`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表（也可以通过指定 TAGS 字段创建超级表）。
```sql
CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options]

3
docs/zh/14-reference/03-taos-sql/24-show.md
View File

@ -306,9 +306,10 @@ SHOW TOPICS;
```sql
SHOW TRANSACTIONS;
SHOW TRANSACTION [tranaction_id];
```
显示当前系统中正在执行的事务的信息(该事务仅针对除普通表以外的元数据级别)
显示当前系统中正在执行的所有或者某一个事务的信息(该事务仅针对除普通表以外的元数据级别)
## SHOW USERS

1
include/common/systable.h
View File

@ -63,6 +63,7 @@ extern "C" {
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_INS_DISK_USAGE "ins_disk_usage"
#define TSDB_INS_TABLE_FILESETS "ins_filesets"
#define TSDB_INS_TABLE_TRANSACTION_DETAILS "ins_transaction_details"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"

2
include/common/tdataformat.h
View File

@ -182,7 +182,7 @@ void tColDataClear(SColData *pColData);
void tColDataDeepClear(SColData *pColData);
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward);
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key);

1
include/common/tglobal.h
View File

@ -112,7 +112,6 @@ extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsQueueMemoryAllowed;
extern int32_t tsRetentionSpeedLimitMB;
extern const char *tsAlterCompactTaskKeywords;
extern int32_t tsNumOfCompactThreads;
extern int32_t tsNumOfRetentionThreads;

2
include/common/tmsg.h
View File

@ -163,6 +163,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_ANODE_FULL,
TSDB_MGMT_TABLE_USAGE,
TSDB_MGMT_TABLE_FILESETS,
TSDB_MGMT_TABLE_TRANSACTION_DETAIL,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -405,6 +406,7 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
QUERY_NODE_SHOW_FILESETS_STMT,
QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,

2
include/libs/executor/storageapi.h
View File

@ -389,7 +389,6 @@ typedef struct SStateStore {
int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
void (*streamStateSetFillInfo)(SStreamState* pState);
void (*streamStateClearExpiredState)(SStreamState* pState);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
@ -455,7 +454,6 @@ typedef struct SStateStore {
int32_t (*streamStateBegin)(SStreamState* pState);
void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
} SStateStore;

5
include/libs/nodes/cmdnodes.h
View File

@ -439,6 +439,11 @@ typedef struct SShowCompactDetailsStmt {
SNode* pCompactId;
} SShowCompactDetailsStmt;
typedef struct SShowTransactionDetailsStmt {
ENodeType type;
SNode* pTransactionId;
} SShowTransactionDetailsStmt;
typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
typedef struct SIndexOptions {

2
include/libs/stream/streamState.h
View File

@ -34,7 +34,6 @@ void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
void streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
int32_t streamStateDelTaskDb(SStreamState* pState);
int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
@ -108,7 +107,6 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
// twa
void streamStateSetFillInfo(SStreamState* pState);
void streamStateClearExpiredState(SStreamState* pState);
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);

1
include/libs/stream/tstreamFileState.h
View File

@ -67,7 +67,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list);
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState);
void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts);

1
include/util/taoserror.h
View File

@ -411,6 +411,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8)
#define TSDB_CODE_MND_TRANS_CONFLICT_COMPACT TAOS_DEF_ERROR_CODE(0, 0x03D9)
#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF)
#define TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED TAOS_DEF_ERROR_CODE(0, 0x03D2)
// mnode-mq
#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0)

4
include/util/tdef.h
View File

@ -330,6 +330,10 @@ typedef enum ELogicConditionType {
#define TSDB_TRANS_STAGE_LEN 12
#define TSDB_TRANS_TYPE_LEN 16
#define TSDB_TRANS_ERROR_LEN 512
#define TSDB_TRANS_OBJTYPE_LEN 40
#define TSDB_TRANS_RESULT_LEN 100
#define TSDB_TRANS_TARGET_LEN 300
#define TSDB_TRANS_DETAIL_LEN 100
#define TSDB_STEP_NAME_LEN 32
#define TSDB_STEP_DESC_LEN 128

1
source/client/src/clientStmt2.c
View File

@ -848,6 +848,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) {
STMT_DLOG("start to set dbName: %s", dbName);
pStmt->db = taosStrdup(dbName);
(void)strdequote(pStmt->db);
STMT_ERR_RET(stmtCreateRequest(pStmt));
// The SQL statement specifies a database name, overriding the previously specified database

4
source/client/test/clientTests.cpp
View File

@ -126,6 +126,10 @@ void fetchCallback(void* param, void* res, int32_t numOfRow) {
void queryCallback(void* param, void* res, int32_t code) {
if (code != TSDB_CODE_SUCCESS) {
(void)printf("failed to execute, reason:%s\n", taos_errstr(res));
taos_free_result(res);
tsem_t *sem = (tsem_t *)param;
tsem_post(sem);
return;
}
(void)printf("start to fetch data\n");
taos_fetch_raw_block_a(res, fetchCallback, param);

12
source/common/src/systable.c
View File

@ -314,6 +314,8 @@ static const SSysDbTableSchema transSchema[] = {
{.name = "oper", .bytes = TSDB_TRANS_OPER_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "stable", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "killable", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
//{.name = "kill_mnode", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "failed_times", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "last_action_info", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
@ -403,6 +405,15 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "remain_time(s)", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
};
static const SSysDbTableSchema userTransactionDetailSchema[] = {
{.name = "transaction_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "action", .bytes = 30 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "obj_type", .bytes = TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "result", .bytes = TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target", .bytes = TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "detail", .bytes = TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema anodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "url", .bytes = TSDB_ANALYTIC_ANODE_URL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@ -521,6 +532,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true},
{TSDB_INS_DISK_USAGE, diskUsageSchema, tListLen(diskUsageSchema), false},
{TSDB_INS_TABLE_FILESETS, filesetsFullSchema, tListLen(filesetsFullSchema), false},
{TSDB_INS_TABLE_TRANSACTION_DETAILS, userTransactionDetailSchema, tListLen(userTransactionDetailSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

22
source/common/src/tdataformat.c
View File

@ -2671,8 +2671,13 @@ static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal
tColDataGetValue6, // HAS_VALUE | HAS_NULL
tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE
};
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
if (iVal < 0 || iVal >= pColData->nVal ||
(pColData->flag <= 0 || pColData->flag >= sizeof(tColDataGetValueImpl)/POINTER_BYTES)){
return TSDB_CODE_INVALID_PARA;
}
tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
return TSDB_CODE_SUCCESS;
}
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
@ -3436,7 +3441,10 @@ static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, S
for (int32_t i = 0; i < nColData; i++) {
SColVal cv = {0};
tColDataGetValue(&aFromColData[i], iFromRow, &cv);
code = tColDataGetValue(&aFromColData[i], iFromRow, &cv);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = tColDataAppendValue(&aToColData[i], &cv);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -3575,7 +3583,10 @@ static int32_t tColDataMerge(SArray **colArr) {
SColData *dstCol = taosArrayGet(dst, j);
SColVal cv;
tColDataGetValue(srcCol, i, &cv);
code = tColDataGetValue(srcCol, i, &cv);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
code = tColDataAppendValue(dstCol, &cv);
if (code) {
goto _exit;
@ -3588,7 +3599,10 @@ static int32_t tColDataMerge(SArray **colArr) {
SColData *dstCol = taosArrayGet(dst, j);
SColVal cv;
tColDataGetValue(srcCol, i, &cv);
code = tColDataGetValue(srcCol, i, &cv);
if (code != TSDB_CODE_SUCCESS) {
goto _exit;
}
code = tColDataUpdateValue(dstCol, &cv, true);
if (code) {
goto _exit;

28
source/common/src/tglobal.c
View File

@ -102,7 +102,6 @@ int32_t tsMaxStreamBackendCache = 128; // M
int32_t tsPQSortMemThreshold = 16; // M
int32_t tsRetentionSpeedLimitMB = 0; // unlimited
const char *tsAlterCompactTaskKeywords = "max_compact_tasks";
int32_t tsNumOfCompactThreads = 2;
int32_t tsNumOfRetentionThreads = 1;
@ -746,7 +745,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX,
CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(
cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
@ -795,9 +795,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
tsNumOfCommitThreads = tsNumOfCores / 2;
tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
tsNumOfCompactThreads = tsNumOfCommitThreads;
tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4);
tsNumOfSupportVnodes = tsNumOfCores * 2 + 5;
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
@ -842,7 +839,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCompactThreads", tsNumOfCompactThreads, 1, 16, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0);
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0);
@ -1040,10 +1037,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem->stype = stype;
}
pItem = cfgGetItem(pCfg, "maxCompactConcurrency");
pItem = cfgGetItem(pCfg, "numOfCompactThreads");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
tsNumOfCompactThreads = numOfCores / 2;
tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4);
pItem->i32 = tsNumOfCompactThreads;
pItem->stype = stype;
}
@ -1548,7 +1543,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads");
tsNumOfCommitThreads = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency");
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCompactThreads");
tsNumOfCompactThreads = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB");
@ -2354,6 +2349,8 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
TAOS_RETURN(code);
}
extern void tsdbAlterNumCompactThreads();
static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = -1;
@ -2404,6 +2401,17 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
goto _exit;
}
if (strcasecmp(name, "numOfCompactThreads") == 0) {
#ifdef TD_ENTERPRISE
tsNumOfCompactThreads = pItem->i32;
code = TSDB_CODE_SUCCESS;
// tsdbAlterNumCompactThreads();
#else
code = TSDB_CODE_INVALID_CFG;
#endif
goto _exit;
}
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function
static OptionNameAndVar debugOptions[] = {
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag},

14
source/common/test/dataformatTest.cpp
View File

@ -449,6 +449,20 @@ static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) {
checkSColVal(data[i], &colVal, pCol->type);
}
}
#ifndef WINDOWS
TEST(testCase, tColDataGetValue) {
SColData pColData = {0};
SColVal pColVal = {0};
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
pColData = {.flag = 8};
pColVal = {0};
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
pColData = {.nVal = 1, .flag = 8};
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
}
#endif
TEST(testCase, AllNormTest) {
int16_t nCols = 14;

26
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -475,27 +475,6 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return 0;
}
extern void tsdbAlterNumCompactThreads();
static int32_t dmAlterMaxCompactTask(const char *value) {
int32_t max_compact_tasks;
char *endptr = NULL;
max_compact_tasks = taosStr2Int32(value, &endptr, 10);
if (endptr == value || endptr[0] != '\0') {
return TSDB_CODE_INVALID_MSG;
}
if (max_compact_tasks != tsNumOfCompactThreads) {
dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks);
tsNumOfCompactThreads = max_compact_tasks;
#ifdef TD_ENTERPRISE
(void)tsdbAlterNumCompactThreads();
#endif
}
return TSDB_CODE_SUCCESS;
}
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0};
@ -509,10 +488,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return taosUpdateTfsItemDisable(pCfg, cfgReq.value, pMgmt->pTfs);
}
if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) {
return dmAlterMaxCompactTask(cfgReq.value);
}
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true);
@ -706,6 +681,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
return TSDB_CODE_INVALID_MSG;
}
dInfo("retrieve table:%s, user:%s, compactId:%" PRId64, retrieveReq.tb, retrieveReq.user, retrieveReq.compactId);
#if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
code = TSDB_CODE_MND_NO_RIGHTS;

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -129,7 +129,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

8
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -133,6 +133,12 @@ typedef enum {
TRN_EXEC_SERIAL = 1,
} ETrnExec;
typedef enum {
TRN_KILL_MODE_SKIP = 0,
TRN_KILL_MODE_INTERUPT = 1,
//TRN_KILL_MODE_ROLLBACK = 2,
} ETrnKillMode;
typedef enum {
DND_REASON_ONLINE = 0,
DND_REASON_STATUS_MSG_TIMEOUT,
@ -201,6 +207,8 @@ typedef struct {
SRWLatch lockRpcArray;
int64_t mTraceId;
TdThreadMutex mutex;
bool ableToBeKilled;
ETrnKillMode killMode;
} STrans;
typedef struct {

4
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -54,6 +54,8 @@ typedef struct {
SSdbRaw *pRaw;
int64_t mTraceId;
int64_t startTime;
int64_t endTime;
} STransAction;
typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
@ -80,6 +82,8 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, voi
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId);
void mndTransSetSerial(STrans *pTrans);
void mndTransSetBeKilled(STrans *pTrans, bool ableToBeKilled);
void mndTransSetKillMode(STrans *pTrans, ETrnKillMode killMode);
void mndTransSetParallel(STrans *pTrans);
void mndTransSetChangeless(STrans *pTrans);
void mndTransSetOper(STrans *pTrans, EOperType oper);

2
source/dnode/mnode/impl/src/mndCompactDetail.c
View File

@ -45,6 +45,8 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
char *sep = NULL;
SDbObj *pDb = NULL;
mInfo("retrieve compact detail");
if (strlen(pShow->db) > 0) {
sep = strchr(pShow->db, '.');
if (sep &&

3
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1270,6 +1270,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
TAOS_RETURN(code);
}
mInfo("trans:%d, used to alter db:%s", pTrans->id, pOld->name);
mInfo("trans:%d, used to alter db, ableToBeKilled:%d, killMode:%d", pTrans->id, pTrans->ableToBeKilled, pTrans->killMode);
mndTransSetDbName(pTrans, pOld->name, NULL);
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
@ -1278,6 +1279,8 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
TAOS_CHECK_GOTO(mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew), NULL, _OVER);
mInfo("trans:%d, used to alter db, ableToBeKilled:%d, killMode:%d", pTrans->id, pTrans->ableToBeKilled, pTrans->killMode);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;

5
source/dnode/mnode/impl/src/mndShow.c
View File

@ -132,6 +132,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_COMPACT;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT_DETAIL;
} else if (strncasecmp(name, TSDB_INS_TABLE_TRANSACTION_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_TRANSACTION_DETAIL;
} else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_LOGS, len) == 0) {
@ -236,7 +238,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
SRetrieveTableReq retrieveReq = {0};
TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq));
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
mDebug("process to retrieve systable req db:%s, tb:%s, compactId:%" PRId64, retrieveReq.db, retrieveReq.tb,
retrieveReq.compactId);
if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));

462
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -14,19 +14,21 @@
*/
#define _DEFAULT_SOURCE
#include "mndTrans.h"
#include "mndDb.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndSubscribe.h"
#include "mndSync.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "osTime.h"
#define TRANS_VER1_NUMBER 1
#define TRANS_VER2_NUMBER 2
#define TRANS_ARRAY_SIZE 8
#define TRANS_RESERVE_SIZE 44
#define TRANS_RESERVE_SIZE 42
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
@ -70,7 +72,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter);
static int32_t mndRetrieveTransDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static int32_t tsMaxTransId = 0;
int32_t mndInitTrans(SMnode *pMnode) {
@ -89,6 +91,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndCancelGetNextTrans);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANSACTION_DETAIL, mndRetrieveTransDetail);
return sdbSetTable(pMnode->pSdb, table);
}
@ -156,7 +159,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans) {
int32_t code = 0;
int32_t lino = 0;
terrno = TSDB_CODE_INVALID_MSG;
int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER;
int8_t sver = TRANS_VER2_NUMBER;
int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen;
rawDataLen += mndTransGetActionsSize(pTrans->prepareActions);
@ -220,6 +223,11 @@ SSdbRaw *mndTransEncode(STrans *pTrans) {
pIter = taosHashIterate(pTrans->arbGroupIds, pIter);
}
if (sver > TRANS_VER1_NUMBER) {
SDB_SET_INT8(pRaw, dataPos, pTrans->ableToBeKilled, _OVER)
SDB_SET_INT32(pRaw, dataPos, pTrans->killMode, _OVER)
}
SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -310,7 +318,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TRANS_VER1_NUMBER && sver != TRANS_VER2_NUMBER) {
if (sver > TRANS_VER2_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER;
}
@ -389,6 +397,13 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
if ((terrno = taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0)) != 0) goto _OVER;
}
int8_t ableKill = 0;
int8_t killMode = 0;
SDB_GET_INT8(pRaw, dataPos, &ableKill, _OVER)
SDB_GET_INT8(pRaw, dataPos, &killMode, _OVER)
pTrans->ableToBeKilled = ableKill;
pTrans->killMode = killMode;
SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
terrno = 0;
@ -430,12 +445,25 @@ static const char *mndTransStr(ETrnStage stage) {
}
}
static const char *mndTransTypeStr(ETrnAct actionType) {
switch (actionType) {
case TRANS_ACTION_MSG:
return "msg";
case TRANS_ACTION_RAW:
return "sdb";
default:
return "invalid";
}
}
static void mndSetTransLastAction(STrans *pTrans, STransAction *pAction) {
if (pAction != NULL) {
if (pAction->errCode != TSDB_CODE_ACTION_IN_PROGRESS) {
pTrans->lastAction = pAction->id;
pTrans->lastMsgType = pAction->msgType;
pTrans->lastEpset = pAction->epSet;
pTrans->lastErrorNo = pAction->errCode;
}
} else {
pTrans->lastAction = 0;
pTrans->lastMsgType = 0;
@ -636,6 +664,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->policy = policy;
pTrans->conflict = conflict;
pTrans->exec = TRN_EXEC_PARALLEL;
pTrans->ableToBeKilled = false;
pTrans->createdTime = taosGetTimestampMs();
pTrans->prepareActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
@ -804,6 +833,13 @@ void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) {
void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
void mndTransSetBeKilled(STrans *pTrans, bool ableToBeKilled) { pTrans->ableToBeKilled = ableToBeKilled; }
void mndTransSetKillMode(STrans *pTrans, ETrnKillMode killMode) {
pTrans->ableToBeKilled = true;
pTrans->killMode = killMode;
}
void mndTransSetParallel(STrans *pTrans) { pTrans->exec = TRN_EXEC_PARALLEL; }
void mndTransSetChangeless(STrans *pTrans) { pTrans->changeless = true; }
@ -1043,6 +1079,39 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
return TSDB_CODE_INVALID_PARA;
}
mInfo("trans:%d, action list:", pTrans->id);
int32_t index = 0;
for (int32_t i = 0; i < taosArrayGetSize(pTrans->prepareActions); ++i, ++index) {
STransAction *pAction = taosArrayGet(pTrans->prepareActions, i);
mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index,
mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status));
}
for (int32_t i = 0; i < taosArrayGetSize(pTrans->redoActions); ++i, ++index) {
STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
mInfo("trans:%d, action:%d, %s:%d msgType:%s", pTrans->id, index,
mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType));;
}
for (int32_t i = 0; i < taosArrayGetSize(pTrans->commitActions); ++i, ++index) {
STransAction *pAction = taosArrayGet(pTrans->commitActions, i);
mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index,
mndTransStr(pAction->stage), i, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status));
}
for (int32_t i = 0; i < taosArrayGetSize(pTrans->undoActions); ++i, ++index) {
STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
if(pAction->actionType == TRANS_ACTION_MSG){
mInfo("trans:%d, action:%d, %s:%d msgType:%s", pTrans->id, index,
mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType));;
}
else{
mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index,
mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status));
}
}
TAOS_CHECK_RETURN(mndTransCheckConflict(pMnode, pTrans));
TAOS_CHECK_RETURN(mndTransCheckParallelActions(pMnode, pTrans));
@ -1260,7 +1329,10 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
if (pAction != NULL) {
pAction->msgReceived = 1;
pAction->errCode = pRsp->code;
pTrans->lastErrorNo = pRsp->code;
pAction->endTime = taosGetTimestampMs();
// pTrans->lastErrorNo = pRsp->code;
mndSetTransLastAction(pTrans, pAction);
mInfo("trans:%d, %s:%d response is received, received code:0x%x(%s), accept:0x%x(%s) retry:0x%x(%s)", transId,
mndTransStr(pAction->stage), action, pRsp->code, tstrerror(pRsp->code), pAction->acceptableCode,
@ -1374,6 +1446,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
pAction->msgSent = 1;
// pAction->msgReceived = 0;
pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS;
pAction->startTime = taosGetTimestampMs();
pAction->endTime = 0;
mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail);
mndSetTransLastAction(pTrans, pAction);
@ -1527,8 +1601,9 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr
for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pActions, action);
mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d", pTrans->id, pTrans->actionPos,
mndTransStr(pAction->stage), pAction->actionType);
mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d, msgSent:%d, msgReceived:%d",
pTrans->id, pTrans->actionPos, mndTransStr(pAction->stage), pAction->actionType, pAction->msgSent,
pAction->msgReceived);
code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
if (code == 0) {
@ -1924,6 +1999,11 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
TAOS_RETURN(TSDB_CODE_MND_TRANS_INVALID_STAGE);
}
if(pTrans->ableToBeKilled == false){
return TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED;
}
if(pTrans->killMode == TRN_KILL_MODE_SKIP){
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
STransAction *pAction = taosArrayGet(pArray, i);
mInfo("trans:%d, %s:%d set processed for kill msg received, errCode from %s to success", pTrans->id,
@ -1932,6 +2012,13 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
pAction->msgReceived = 1;
pAction->errCode = 0;
}
}
else if(pTrans->killMode == TRN_KILL_MODE_INTERUPT){
pTrans->stage = TRN_STAGE_PRE_FINISH;
}
else{
return TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED;
}
mndTransExecute(pMnode, pTrans);
return 0;
@ -2002,6 +2089,114 @@ void mndTransPullup(SMnode *pMnode) {
taosArrayDestroy(pArray);
}
static char *formatTimestamp(char *buf, int64_t val, int precision) {
time_t tt;
if (precision == TSDB_TIME_PRECISION_MICRO) {
tt = (time_t)(val / 1000000);
}
if (precision == TSDB_TIME_PRECISION_NANO) {
tt = (time_t)(val / 1000000000);
} else {
tt = (time_t)(val / 1000);
}
struct tm tm;
if (taosLocalTime(&tt, &tm, NULL, 0, NULL) == NULL) {
mError("failed to get local time");
return NULL;
}
size_t pos = taosStrfTime(buf, 32, "%Y-%m-%d %H:%M:%S", &tm);
if (precision == TSDB_TIME_PRECISION_MICRO) {
sprintf(buf + pos, ".%06d", (int)(val % 1000000));
} else if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", (int)(val % 1000000000));
} else {
sprintf(buf + pos, ".%03d", (int)(val % 1000));
}
return buf;
}
static void mndTransLogAction(STrans *pTrans) {
char detail[512] = {0};
int32_t len = 0;
int32_t index = 0;
if (pTrans->stage == TRN_STAGE_PREPARE) {
for (int32_t i = 0; i < taosArrayGetSize(pTrans->prepareActions); ++i, ++index) {
len = 0;
STransAction *pAction = taosArrayGet(pTrans->prepareActions, i);
len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index,
mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type),
sdbStatusName(pAction->pRaw->status));
mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail);
}
}
if (pTrans->stage == TRN_STAGE_REDO_ACTION) {
for (int32_t i = 0; i < taosArrayGetSize(pTrans->redoActions); ++i, ++index) {
len = 0;
STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
if (pAction->actionType == TRANS_ACTION_MSG) {
char bufStart[40] = {0};
(void)formatTimestamp(bufStart, pAction->startTime, TSDB_TIME_PRECISION_MILLI);
char endStart[40] = {0};
(void)formatTimestamp(endStart, pAction->endTime, TSDB_TIME_PRECISION_MILLI);
len += snprintf(detail + len, sizeof(detail) - len,
"action:%d, %s:%d msgType:%s,"
"sent:%d, received:%d, startTime:%s, endTime:%s, ",
index, mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType), pAction->msgSent,
pAction->msgReceived, bufStart, endStart);
SEpSet epset = pAction->epSet;
if (epset.numOfEps > 0) {
len += snprintf(detail + len, sizeof(detail) - len, "numOfEps:%d inUse:%d ", epset.numOfEps, epset.inUse);
for (int32_t i = 0; i < epset.numOfEps; ++i) {
len +=
snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port);
}
}
len += snprintf(detail + len, sizeof(detail) - len, ", errCode:0x%x(%s)\n", pAction->errCode & 0xFFFF,
tstrerror(pAction->errCode));
} else {
len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s, written:%d\n",
index, mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type),
sdbStatusName(pAction->pRaw->status), pAction->rawWritten);
}
mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail);
}
}
if (pTrans->stage == TRN_STAGE_COMMIT_ACTION) {
for (int32_t i = 0; i < taosArrayGetSize(pTrans->commitActions); ++i, ++index) {
len = 0;
STransAction *pAction = taosArrayGet(pTrans->commitActions, i);
len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index,
mndTransStr(pAction->stage), i, sdbTableName(pAction->pRaw->type),
sdbStatusName(pAction->pRaw->status));
mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail);
}
for (int32_t i = 0; i < taosArrayGetSize(pTrans->undoActions); ++i, ++index) {
len = 0;
STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
if (pAction->actionType == TRANS_ACTION_MSG) {
len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d msgType:%s\n", index,
mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType));
;
} else {
len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index,
mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type),
sdbStatusName(pAction->pRaw->status));
}
mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail);
}
}
}
static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -2044,6 +2239,20 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbname, false), pTrans, &lino, _OVER);
const char *killableStr = pTrans->ableToBeKilled ? "yes" : "no";
char killableVstr[10 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(killableVstr, killableStr, 10 + VARSTR_HEADER_SIZE);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)killableVstr, false), pTrans, &lino, _OVER);
/*
const char *killModeStr = pTrans->killMode == TRN_KILL_MODE_SKIP ? "skip" : "interrupt";
char killModeVstr[10 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(killModeVstr, killModeStr, 24);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)killModeVstr, false), pTrans, &lino, _OVER);
*/
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false), pTrans, &lino,
_OVER);
@ -2061,13 +2270,15 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
len += tsnprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ",
TMSG_INFO(pTrans->lastMsgType), epset.numOfEps, epset.inUse);
for (int32_t i = 0; i < pTrans->lastEpset.numOfEps; ++i) {
len += tsnprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port);
len += snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port);
}
}
STR_WITH_MAXSIZE_TO_VARSTR(lastInfo, detail, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)lastInfo, false), pTrans, &lino, _OVER);
mndTransLogAction(pTrans);
numOfRows++;
sdbRelease(pSdb, pTrans);
}
@ -2078,6 +2289,239 @@ _OVER:
return numOfRows;
}
static int32_t mndShowTransCommonColumns(SShowObj *pShow, SSDataBlock *pBlock, STransAction *pAction,
int32_t transactionId, int32_t curActionId, int32_t numOfRows, int32_t *cols) {
int32_t code = 0;
int32_t lino = 0;
int32_t len = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, (*cols)++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&transactionId, false), &lino, _OVER);
char action[30 + 1] = {0};
if (curActionId == pAction->id) {
len += snprintf(action + len, sizeof(action) - len, "%s:%d(%s)<-last", mndTransStr(pAction->stage), pAction->id,
mndTransTypeStr(pAction->actionType));
} else {
len += snprintf(action + len, sizeof(action) - len, "%s:%d(%s)", mndTransStr(pAction->stage), pAction->id,
mndTransTypeStr(pAction->actionType));
}
char actionVStr[30 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(actionVStr, action, pShow->pMeta->pSchemas[*cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, (*cols)++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)actionVStr, false), &lino, _OVER);
_OVER:
if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code));
return code;
}
static void mndShowTransAction(SShowObj *pShow, SSDataBlock *pBlock, STransAction *pAction, int32_t transactionId,
int32_t curActionId, int32_t rows, int32_t numOfRows) {
int32_t code = 0;
int32_t lino = 0;
int32_t len = 0;
int32_t cols = 0;
cols = 0;
if (mndShowTransCommonColumns(pShow, pBlock, pAction, transactionId, curActionId, numOfRows, &cols) != 0) return;
if (pAction->actionType == TRANS_ACTION_MSG) {
int32_t len = 0;
char objType[TSDB_TRANS_OBJTYPE_LEN + 1] = {0};
len += snprintf(objType + len, sizeof(objType) - len, "%s(s:%d,r:%d)", TMSG_INFO(pAction->msgType),
pAction->msgSent, pAction->msgReceived);
char objTypeVStr[TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(objTypeVStr, objType, pShow->pMeta->pSchemas[cols].bytes);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)objTypeVStr, false), &lino, _OVER);
char result[TSDB_TRANS_RESULT_LEN + 1] = {0};
len = 0;
len += snprintf(result + len, sizeof(result) - len, "errCode:0x%x(%s)", pAction->errCode & 0xFFFF,
tstrerror(pAction->errCode));
char resultVStr[TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(resultVStr, result, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)resultVStr, false), &lino, _OVER);
char target[TSDB_TRANS_TARGET_LEN] = {0};
len = 0;
SEpSet epset = pAction->epSet;
if (epset.numOfEps > 0) {
for (int32_t i = 0; i < epset.numOfEps; ++i) {
len += snprintf(target + len, sizeof(target) - len, "ep:%d-%s:%u,", i, epset.eps[i].fqdn, epset.eps[i].port);
}
len += snprintf(target + len, sizeof(target) - len, "(%d:%d) ", epset.numOfEps, epset.inUse);
}
char targetVStr[TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(targetVStr, target, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)targetVStr, false), &lino, _OVER);
char detail[TSDB_TRANS_DETAIL_LEN] = {0};
len = 0;
char bufStart[40] = {0};
if (pAction->startTime > 0) (void)formatTimestamp(bufStart, pAction->startTime, TSDB_TIME_PRECISION_MILLI);
char bufEnd[40] = {0};
if (pAction->endTime > 0) (void)formatTimestamp(bufEnd, pAction->endTime, TSDB_TIME_PRECISION_MILLI);
len += snprintf(detail + len, sizeof(detail) - len, "startTime:%s, endTime:%s, ", bufStart, bufEnd);
char detailVStr[TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(detailVStr, detail, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)detailVStr, false), &lino, _OVER);
} else {
int32_t len = 0;
char objType[TSDB_TRANS_OBJTYPE_LEN + 1] = {0};
if (pAction->pRaw->type == SDB_VGROUP) {
SSdbRow *pRow = mndVgroupActionDecode(pAction->pRaw);
SVgObj *pVgroup = sdbGetRowObj(pRow);
len += snprintf(objType + len, sizeof(objType) - len, "%s(%d)", sdbTableName(pAction->pRaw->type), pVgroup->vgId);
taosMemoryFreeClear(pRow);
} else {
strcpy(objType, sdbTableName(pAction->pRaw->type));
}
char objTypeVStr[TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(objTypeVStr, objType, pShow->pMeta->pSchemas[cols].bytes);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)objTypeVStr, false), &lino, _OVER);
char result[TSDB_TRANS_RESULT_LEN + 1] = {0};
len = 0;
len += snprintf(result + len, sizeof(result) - len, "rawWritten:%d", pAction->rawWritten);
char resultVStr[TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(resultVStr, result, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)resultVStr, false), &lino, _OVER);
char target[TSDB_TRANS_TARGET_LEN] = "";
char targetVStr[TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(targetVStr, target, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)targetVStr, false), &lino, _OVER);
char detail[TSDB_TRANS_DETAIL_LEN] = {0};
len = 0;
len += snprintf(detail + len, sizeof(detail) - len, "sdbStatus:%s", sdbStatusName(pAction->pRaw->status));
char detailVStr[TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(detailVStr, detail, pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)detailVStr, false), &lino, _OVER);
}
_OVER:
if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code));
}
static SArray *mndTransGetAction(STrans *pTrans, ETrnStage stage) {
if (stage == TRN_STAGE_PREPARE) {
return pTrans->prepareActions;
}
if (stage == TRN_STAGE_REDO_ACTION) {
return pTrans->redoActions;
}
if (stage == TRN_STAGE_COMMIT_ACTION) {
return pTrans->commitActions;
}
if (stage == TRN_STAGE_UNDO_ACTION) {
return pTrans->undoActions;
}
return NULL;
}
typedef struct STransDetailIter {
void *pIter;
STrans *pTrans;
ETrnStage stage;
int32_t num;
} STransDetailIter;
static void mndTransShowActions(SSdb *pSdb, STransDetailIter *pShowIter, SShowObj *pShow, SSDataBlock *pBlock,
int32_t rows, int32_t *numOfRows, SArray *pActions, int32_t end, int32_t start) {
int32_t actionNum = taosArrayGetSize(pActions);
mInfo("stage:%s, Actions num:%d", mndTransStr(pShowIter->stage), actionNum);
for (int32_t i = start; i < actionNum; ++i) {
STransAction *pAction = taosArrayGet(pShowIter->pTrans->redoActions, i);
mndShowTransAction(pShow, pBlock, pAction, pShowIter->pTrans->id, pShowIter->pTrans->lastAction, rows, *numOfRows);
(*numOfRows)++;
if (*numOfRows >= rows) break;
}
if (*numOfRows == end) {
sdbRelease(pSdb, pShowIter->pTrans);
pShowIter->pTrans = NULL;
pShowIter->num = 0;
} else {
pShowIter->pTrans = pShowIter->pTrans;
pShowIter->stage = pShowIter->pTrans->stage;
pShowIter->num += (*numOfRows);
}
}
static int32_t mndRetrieveTransDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t code = 0;
int32_t lino = 0;
mInfo("start to mndRetrieveTransDetail, rows:%d, pShow->numOfRows:%d, pShow->pIter:%p", rows, pShow->numOfRows,
pShow->pIter);
if (pShow->pIter == NULL) {
pShow->pIter = taosMemoryMalloc(sizeof(STransDetailIter));
if (pShow->pIter == NULL) {
mError("failed to malloc for pShow->pIter");
return 0;
}
memset(pShow->pIter, 0, sizeof(STransDetailIter));
}
STransDetailIter *pShowIter = (STransDetailIter *)pShow->pIter;
while (numOfRows < rows) {
if (pShowIter->pTrans == NULL) {
pShowIter->pIter = sdbFetch(pSdb, SDB_TRANS, pShowIter->pIter, (void **)&(pShowIter->pTrans));
mDebug("retrieve trans detail from fetch, pShow->pIter:%p, pTrans:%p", pShowIter->pIter, pShowIter->pTrans);
if (pShowIter->pIter == NULL) break;
mInfo("retrieve trans detail from fetch, id:%d, trans stage:%d, IterNum:%d", pShowIter->pTrans->id,
pShowIter->pTrans->stage, pShowIter->num);
SArray *pActions = mndTransGetAction(pShowIter->pTrans, pShowIter->pTrans->stage);
mndTransShowActions(pSdb, pShowIter, pShow, pBlock, rows, &numOfRows, pActions, taosArrayGetSize(pActions), 0);
break;
} else {
mInfo("retrieve trans detail from iter, id:%d, iterStage:%d, IterNum:%d", pShowIter->pTrans->id, pShowIter->stage,
pShowIter->num);
SArray *pActions = mndTransGetAction(pShowIter->pTrans, pShowIter->stage);
mndTransShowActions(pSdb, pShowIter, pShow, pBlock, rows, &numOfRows, pActions,
taosArrayGetSize(pActions) - pShowIter->num, pShowIter->num);
break;
}
}
_OVER:
pShow->numOfRows += numOfRows;
if (code != 0) {
mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code));
} else {
mInfo("retrieve trans detail, numOfRows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows)
}
if (numOfRows == 0) {
taosMemoryFree(pShow->pIter);
pShow->pIter = NULL;
}
return numOfRows;
}
static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_TRANS);

12
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -2766,12 +2766,14 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
mndTransSetSerial(pTrans);
if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 3) {
if (pNewDb->cfg.replications == 3) {
mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);
// add second
if (pNewVgroup->replica == 1){
TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
}
// learner stage
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
@ -2790,7 +2792,9 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup));
// add third
TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
if (pNewVgroup->replica == 2){
TAOS_CHECK_RETURN (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray));
}
pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER;
pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER;
@ -2802,7 +2806,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[2]));
TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup));
} else if (pNewVgroup->replica == 3 && pNewDb->cfg.replications == 1) {
} else if (pNewDb->cfg.replications == 1) {
mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName,
pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId);
@ -2821,7 +2825,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb
TAOS_CHECK_RETURN(
mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId));
TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup));
} else if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 2) {
} else if (pNewDb->cfg.replications == 2) {
mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId,
pVgroup->vnodeGid[0].dnodeId);

2
source/dnode/snode/src/snodeInitApi.c
View File

@ -68,7 +68,6 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur;
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateSetFillInfo = streamStateSetFillInfo;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
@ -117,7 +116,6 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit;
pStore->streamStateDestroy = streamStateDestroy;
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
pStore->streamStateReloadInfo = streamStateReloadInfo;
pStore->streamStateCopyBackend = streamStateCopyBackend;
}

1
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -325,7 +325,6 @@ int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8
ctx->suidInfo = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (ctx->suidInfo == NULL) {
return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
;
}
taosHashSetFreeFp(ctx->suidInfo, destroySTableInfoForChildTable);

7
source/dnode/vnode/src/tq/tqRead.c
View File

@ -796,7 +796,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {
for (int32_t i = 0; i < pCol->nVal; i++) {
tColDataGetValue(pCol, i, &colVal);
code = tColDataGetValue(pCol, i, &colVal);
TSDB_CHECK_CODE(code, line, END);
code = doSetVal(pColData, i, &colVal);
TSDB_CHECK_CODE(code, line, END);
}
@ -937,7 +938,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
pCol = taosArrayGet(pCols, j);
TQ_NULL_GO_TO_END(pCol);
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal));
PROCESS_VAL
}
@ -961,7 +962,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
TQ_NULL_GO_TO_END(pColData);
SColVal colVal = {0};
tColDataGetValue(pCol, i, &colVal);
TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal));
SET_DATA
}

2
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1715,7 +1715,7 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow);
if (colType == 2) {
SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type);
tColDataGetValue(pColData, tRow.iRow, &colVal);
TAOS_CHECK_GOTO(tColDataGetValue(pColData, tRow.iRow, &colVal), &lino, _exit);
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
if (!taosArrayPush(ctxArray, &updateCtx)) {

9
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -123,7 +123,8 @@ static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int
pColData = &pBlock->aColData[slotId];
tColDataGetValue(pColData, irow, &cv);
code = tColDataGetValue(pColData, irow, &cv);
TSDB_CHECK_CODE(code, lino, _end);
pKey->numOfPKs = 1;
pKey->pks[0].type = cv.value.type;
@ -1603,7 +1604,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
TSDB_CHECK_CODE(code, lino, _end);
} else { // varchar/nchar type
for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {
tColDataGetValue(pData, j, &cv);
code = tColDataGetValue(pData, j, &cv);
TSDB_CHECK_CODE(code, lino, _end);
code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
TSDB_CHECK_CODE(code, lino, _end);
}
@ -5282,7 +5284,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pData->cid == pSupInfo->colId[i]) {
tColDataGetValue(pData, rowIndex, &cv);
code = tColDataGetValue(pData, rowIndex, &cv);
TSDB_CHECK_CODE(code, lino, _end);
code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
TSDB_CHECK_CODE(code, lino, _end);
j += 1;

15
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -622,7 +622,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
if (pColData) {
tColDataGetValue(pColData, pRow->iRow, pColVal);
if (tColDataGetValue(pColData, pRow->iRow, pColVal) != 0){
tsdbError("failed to tColDataGetValue");
}
} else {
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
}
@ -645,7 +647,9 @@ void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
SColData *pColData = &pBlock->aColData[i];
if (pColData->cflag & COL_IS_KEY) {
SColVal cv;
tColDataGetValue(pColData, irow, &cv);
if (tColDataGetValue(pColData, irow, &cv) != 0){
break;
}
key->pks[key->numOfPKs] = cv.value;
key->numOfPKs++;
} else {
@ -719,7 +723,9 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
}
if (pIter->iColData <= pIter->pRow->pBlockData->nColData) {
tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv);
if (tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv) != 0){
return NULL;
}
++pIter->iColData;
return &pIter->cv;
} else {
@ -1251,7 +1257,8 @@ static int32_t tBlockDataUpsertBlockRow(SBlockData *pBlockData, SBlockData *pBlo
cv = COL_VAL_NONE(pColDataTo->cid, pColDataTo->type);
if (flag == 0 && (code = tColDataAppendValue(pColDataTo, &cv))) goto _exit;
} else {
tColDataGetValue(pColDataFrom, iRow, &cv);
code = tColDataGetValue(pColDataFrom, iRow, &cv);
if (code) goto _exit;
if (flag) {
code = tColDataUpdateValue(pColDataTo, &cv, flag > 0);

2
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -191,7 +191,6 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur;
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
pStore->streamStateSetFillInfo = streamStateSetFillInfo;
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
@ -243,7 +242,6 @@ void initStateStoreAPI(SStateStore* pStore) {
pStore->streamStateBegin = streamStateBegin;
pStore->streamStateCommit = streamStateCommit;
pStore->streamStateDestroy = streamStateDestroy;
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
pStore->streamStateReloadInfo = streamStateReloadInfo;
pStore->streamStateCopyBackend = streamStateCopyBackend;
}

5
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -243,7 +243,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_VND_HASH_MISMATCH;
goto _exit;
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK);
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN);
@ -279,7 +279,8 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
}
} else {
vError("vnodeGetTableCfg get invalid table type:%d", mer1.me.type);
return TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
goto _exit;
}
cfgRsp.numOfTags = schemaTag.nCols;

13
source/libs/catalog/src/catalog.c
View File

@ -1986,10 +1986,19 @@ void catalogDestroy(void) {
}
if (gCtgMgmt.cacheTimer) {
if (taosTmrStop(gCtgMgmt.cacheTimer)) {
qTrace("stop catalog cache timer may failed");
if (!taosTmrStop(gCtgMgmt.cacheTimer)) {
/*
qDebug("catalog cacheTimer %" PRIuPTR " not stopped", (uintptr_t)gCtgMgmt.cacheTimer);
while (!taosTmrIsStopped(&gCtgMgmt.cacheTimer)) {
taosMsleep(1);
}
*/
}
qDebug("catalog cacheTimer %" PRIuPTR " is stopped", (uintptr_t)gCtgMgmt.cacheTimer);
gCtgMgmt.cacheTimer = NULL;
taosTmrCleanUp(gCtgMgmt.timer);
gCtgMgmt.timer = NULL;
}

8
source/libs/catalog/src/ctgDbg.c
View File

@ -480,6 +480,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
dbCache = (SCtgDBCache *)pIter;
CTG_LOCK(CTG_READ, &dbCache->dbLock);
dbFName = taosHashGetKey(pIter, &len);
int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0;
@ -509,6 +511,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
hashMethod, hashPrefix, hashSuffix, vgNum);
if (dbCache->vgCache.vgInfo) {
CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock);
int32_t i = 0;
void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL);
while (pVgIter) {
@ -524,6 +528,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter);
}
CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock);
}
if (dbCache->cfgCache.cfgInfo) {
@ -544,6 +550,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
pCfg->schemaless, pCfg->sstTrigger);
}
CTG_UNLOCK(CTG_READ, &dbCache->dbLock);
++i;
pIter = taosHashIterate(dbHash, pIter);
}

5
source/libs/catalog/test/catalogTests.cpp
View File

@ -149,12 +149,13 @@ void ctgTestInitLogFile() {
return;
}
const char *defaultLogFileNamePrefix = "taoslog";
const char *defaultLogFileNamePrefix = "catalogTest";
const int32_t maxLogFileNum = 10;
tsAsyncLog = 0;
qDebugFlag = 159;
tmrDebugFlag = 159;
tsNumOfLogLines = 1000000000;
TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH);
(void)ctgdEnableDebug("api", true);
@ -1839,7 +1840,7 @@ TEST(tableMeta, updateStbMeta) {
while (true) {
uint64_t n = 0;
ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n));
if (n != 3) {
if (n < 3) {
taosMsleep(50);
} else {
break;

47
source/libs/executor/src/anomalywindowoperator.c
View File

@ -61,9 +61,13 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock*
int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_PARAM_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = NULL;
const char* id = GET_TASKID(pTaskInfo);
SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode;
@ -74,13 +78,13 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p
}
if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) {
qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt);
qError("%s failed to get anomaly_window algorithm name from %s", id, pAnomalyNode->anomalyOpt);
code = TSDB_CODE_ANA_ALGO_NOT_FOUND;
goto _error;
}
if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) {
qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName);
qError("%s failed to get anomaly_window algorithm url from %s", id, pInfo->algoName);
code = TSDB_CODE_ANA_ALGO_NOT_LOAD;
goto _error;
}
@ -94,20 +98,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p
SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
}
size_t keyBufSize = 0;
int32_t num = 0;
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, id, pTaskInfo->streamInfo.pState,
&pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc);
@ -124,27 +126,19 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p
pInfo->anomalyCol = extractColumnFromColumnNode(pColNode);
pInfo->anomalyKey.type = pInfo->anomalyCol.type;
pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes;
pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes);
if (pInfo->anomalyKey.pData == NULL) {
goto _error;
}
QUERY_CHECK_NULL(pInfo->anomalyKey.pData, code, lino, _error, terrno)
int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes;
pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize);
if (pInfo->anomalySup.pResultRow == NULL) {
code = terrno;
goto _error;
}
QUERY_CHECK_NULL(pInfo->anomalySup.pResultRow, code, lino, _error, terrno)
pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*));
if (pInfo->anomalySup.blocks == NULL) {
code = terrno;
goto _error;
}
QUERY_CHECK_NULL(pInfo->anomalySup.blocks, code, lino, _error, terrno)
pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow));
if (pInfo->anomalySup.windows == NULL) {
code = terrno;
goto _error;
}
QUERY_CHECK_NULL(pInfo->anomalySup.windows, code, lino, _error, terrno)
code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
QUERY_CHECK_CODE(code, lino, _error);
@ -162,18 +156,21 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p
*pOptrInfo = pOperator;
qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl,
qDebug("%s anomaly_window operator is created, algo:%s url:%s opt:%s", id, pInfo->algoName, pInfo->algoUrl,
pInfo->anomalyOpt);
return TSDB_CODE_SUCCESS;
_error:
qError("%s failed to create anomaly_window operator, line:%d algo:%s code:%s", id, lino, pAnomalyNode->anomalyOpt,
tstrerror(code));
if (pInfo != NULL) {
anomalyDestroyOperatorInfo(pInfo);
}
destroyOperatorAndDownstreams(pOperator, &downstream, 1);
pTaskInfo->code = code;
qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code);
return code;
}

2
source/libs/executor/src/mergejoin.c
View File

@ -46,7 +46,7 @@ static FORCE_INLINE bool mJoinBlkReachThreshold(SMJoinOperatorInfo* pInfo, int64
return blkRows >= pInfo->ctx.mergeCtx.blkThreshold;
}
return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit;
return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit || blkRows >= pInfo->ctx.mergeCtx.blkThreshold;
}

5
source/libs/executor/src/sysscanoperator.c
View File

@ -2284,7 +2284,7 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) {
// db_name
pColInfoData = taosArrayGet(p->pDataBlock, index++);
QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno);
code = colDataSetVal(pColInfoData, numOfRows, db, false);
code = colDataSetVal(pColInfoData, numOfRows, dbname, false);
QUERY_CHECK_CODE(code, lino, _end);
// vgroup_id
@ -2543,7 +2543,8 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes)
if (pInfo->showRewrite) {
getDBNameFromCondition(pInfo->pCondition, dbName);
if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0 &&
strncasecmp(name, TSDB_INS_TABLE_TRANSACTION_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
TAOS_UNUSED(tsnprintf(pInfo->req.db, sizeof(pInfo->req.db), "%d.%s", pInfo->accountId, dbName));
}
} else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {

2
source/libs/index/inc/indexFstRegex.h
View File

@ -28,7 +28,7 @@
extern "C" {
#endif
typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType;
typedef enum { INS_MATCH, INS_JUMP, INS_SPLIT, INS_RANGE } InstType;
typedef struct MatchValue {
#ifdef WINDOWS

18
source/libs/index/src/indexFstDfa.c
View File

@ -159,14 +159,14 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r
if (false == sparSetGet(set, i, &ip)) continue;
Inst *inst = taosArrayGet(builder->dfa->insts, ip);
if (inst->ty == JUMP || inst->ty == SPLIT) {
if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) {
continue;
} else if (inst->ty == RANGE) {
} else if (inst->ty == INS_RANGE) {
if (taosArrayPush(tinsts, &ip) == NULL) {
code = terrno;
goto _exception;
}
} else if (inst->ty == MATCH) {
} else if (inst->ty == INS_MATCH) {
isMatch = true;
if (taosArrayPush(tinsts, &ip) == NULL) {
code = terrno;
@ -234,11 +234,11 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) {
}
bool succ = sparSetAdd(set, ip, NULL);
Inst *inst = taosArrayGet(dfa->insts, ip);
if (inst->ty == MATCH || inst->ty == RANGE) {
if (inst->ty == INS_MATCH || inst->ty == INS_RANGE) {
// do nothing
} else if (inst->ty == JUMP) {
} else if (inst->ty == INS_JUMP) {
dfaAdd(dfa, set, inst->jv.step);
} else if (inst->ty == SPLIT) {
} else if (inst->ty == INS_SPLIT) {
dfaAdd(dfa, set, inst->sv.len1);
dfaAdd(dfa, set, inst->sv.len2);
}
@ -253,11 +253,11 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) {
if (false == sparSetGet(from, i, &ip)) continue;
Inst *inst = taosArrayGet(dfa->insts, ip);
if (inst->ty == JUMP || inst->ty == SPLIT) {
if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) {
continue;
} else if (inst->ty == MATCH) {
} else if (inst->ty == INS_MATCH) {
isMatch = true;
} else if (inst->ty == RANGE) {
} else if (inst->ty == INS_RANGE) {
if (inst->rv.start <= byte && byte <= inst->rv.end) {
dfaAdd(dfa, to, ip + 1);
}

6
source/libs/index/src/indexTfile.c
View File

@ -106,6 +106,11 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) {
SArray* files = NULL;
int32_t code = tfileGetFileList(path, &files);
if (code != 0) {
indexError("failed to get file list since %s", tstrerror(code));
goto End;
}
for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char* file = taosArrayGetP(files, i);
@ -1182,7 +1187,6 @@ _exception:
TAOS_UNUSED(taosCloseDir(&pDir));
if (files != NULL) {
taosArrayDestroyEx(files, tfileDestroyFileName);
taosArrayDestroy(files);
}
return code;
}

56
source/libs/index/test/utilUT.cc
View File

@ -17,6 +17,7 @@
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
#include "indexFstDfa.h"
class UtilEnv : public ::testing::Test {
protected:
@ -41,6 +42,29 @@ class UtilEnv : public ::testing::Test {
SArray *rslt;
};
class UtilComm : public ::testing::Test {
protected:
virtual void SetUp() {
// src = (SArray *)taosArrayInit(2, sizeof(void *));
// for (int i = 0; i < 3; i++) {
// SArray *m = taosArrayInit(10, sizeof(uint64_t));
// taosArrayPush(src, &m);
// }
// rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
}
virtual void TearDown() {
// for (int i = 0; i < taosArrayGetSize(src); i++) {
// SArray *m = (SArray *)taosArrayGetP(src, i);
// taosArrayDestroy(m);
// }
// taosArrayDestroy(src);
}
// SArray *src;
// SArray *rslt;
};
static void clearSourceArray(SArray *p) {
for (int i = 0; i < taosArrayGetSize(p); i++) {
SArray *m = (SArray *)taosArrayGetP(p, i);
@ -369,3 +393,35 @@ TEST_F(UtilEnv, testDictComm) {
EXPECT_EQ(COMMON_INPUTS[v], i);
}
}
TEST_F(UtilComm, testCompress) {
for (int32_t i = 0; i < 6; i++) {
_cache_range_compare cmpFunc = idxGetCompare((RangeType)(i));
//char[32]a = 0, b = 1;
char a[32] = {0};
char b[32] = {1};
for (int32_t j = 0; j < TSDB_DATA_TYPE_MAX; j++) {
cmpFunc(a, b, j);
}
}
}
TEST_F(UtilComm, testfstDfa) {
{
FstDfaBuilder *builder = dfaBuilderCreate(NULL);
ASSERT_TRUE(builder != NULL);
dfaBuilderDestroy(builder);
}
{
SArray *pInst = taosArrayInit(32, sizeof(uint8_t));
for (int32_t i = 0; i < 26; i++) {
uint8_t v = 'a' + i;
taosArrayPush(pInst, &v);
}
FstDfaBuilder *builder = dfaBuilderCreate(pInst);
FstDfa *dfa = dfaBuilderBuild(builder);
dfaBuilderDestroy(builder);
}
}

2
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -292,6 +292,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowCompactsStmt";
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return "ShowCompactDetailsStmt";
case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT:
return "ShowTransactionDetailsStmt";
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return "ShowGrantsFullStmt";
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:

8
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -715,6 +715,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) {
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
code = makeNode(type, sizeof(SShowCompactDetailsStmt), &pNode);
break;
case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT:
code = makeNode(type, sizeof(SShowTransactionDetailsStmt), &pNode);
break;
case QUERY_NODE_KILL_QUERY_STMT:
code = makeNode(type, sizeof(SKillQueryStmt), &pNode);
break;
@ -1562,6 +1565,11 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pStmt->pCompactId);
break;
}
case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: {
SShowTransactionDetailsStmt* pStmt = (SShowTransactionDetailsStmt*)pNode;
nodesDestroyNode(pStmt->pTransactionId);
break;
}
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg);
break;

1
source/libs/parser/inc/parAst.h
View File

@ -320,6 +320,7 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie
SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView);
SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactIdNode);
SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type);
SNode* createShowTransactionDetailsStmt(SAstCreateContext* pCxt, SNode* pTransactionIdNode);
SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* tsmaName, SNode* pOptions,
SNode* pRealTable, SNode* pInterval);

1
source/libs/parser/inc/sql.y
View File

@ -594,6 +594,7 @@ cmd ::= SHOW BNODES.
cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); }
cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); }
cmd ::= SHOW TRANSACTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); }
cmd ::= SHOW TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createShowTransactionDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); }
cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, A); }
cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); }
cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); }

12
source/libs/parser/src/parAstCreater.c
View File

@ -2931,6 +2931,18 @@ _err:
return NULL;
}
SNode* createShowTransactionDetailsStmt(SAstCreateContext* pCxt, SNode* pTransactionIdNode) {
CHECK_PARSER_STATUS(pCxt);
SShowTransactionDetailsStmt* pStmt = NULL;
pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, (SNode**)&pStmt);
CHECK_MAKE_NODE(pStmt);
pStmt->pTransactionId = pTransactionIdNode;
return (SNode*)pStmt;
_err:
nodesDestroyNode(pTransactionIdNode);
return NULL;
}
static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange) {
int32_t code = TSDB_CODE_SUCCESS;
char* ipCopy = taosStrdup(ipRange);

8
source/libs/parser/src/parAstParser.c
View File

@ -770,6 +770,12 @@ static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SS
return code;
}
static int32_t collectMetaKeyFromShowTransactionDetails(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB,
TSDB_INS_TABLE_TRANSACTION_DETAILS, pCxt->pMetaCache);
return code;
}
static int32_t collectMetaKeyFromShowGrantsFull(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_GRANTS_FULL,
pCxt->pMetaCache);
@ -1094,6 +1100,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowCompacts(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT:
return collectMetaKeyFromShowTransactionDetails(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return collectMetaKeyFromShowGrantsFull(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:

6
source/libs/parser/src/parInsertSql.c
View File

@ -1841,6 +1841,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
}
if (TK_NK_QUESTION == pToken->type) {
if (!pCxt->pComCxt->isStmtBind && i != 0) {
return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values");
}
pCxt->isStmtBind = true;
pStmt->usingTableProcessing = true;
if (pCols->pColIndex[i] == tbnameIdx) {
@ -1874,6 +1877,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
return buildInvalidOperationMsg(&pCxt->msg, "not expected numOfBound");
}
} else {
if (pCxt->pComCxt->isStmtBind) {
return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values");
}
if (pCols->pColIndex[i] < numOfCols) {
const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]];
SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]);

72
source/libs/parser/src/parTranslater.c
View File

@ -370,6 +370,27 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_CREATE_TSMA_STMT,
.pDbName = "",
.pTableName = "",
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_SHOW_CREATE_TSMA_STMT,
.pDbName = "",
.pTableName = "",
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_DROP_TSMA_STMT,
.pDbName = "",
.pTableName = "",
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_SHOW_FILESETS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
@ -377,6 +398,13 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1,
.pShowCols = {"*"}
},
{
.showType = QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_TRANSACTION_DETAILS,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
};
// clang-format on
@ -10086,7 +10114,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
const char* validConfigs[] = {
"encrypt_key",
tsAlterCompactTaskKeywords,
};
if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) {
int32_t klen = strlen(cfgReq.value);
@ -10097,28 +10124,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt
ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN);
}
code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
} else if (0 == strncasecmp(cfgReq.config, validConfigs[1], strlen(validConfigs[1]) + 1)) {
char* endptr = NULL;
int32_t maxCompactTasks = taosStr2Int32(cfgReq.value, &endptr, 10);
int32_t minMaxCompactTasks = MIN_MAX_COMPACT_TASKS;
int32_t maxMaxCompactTasks = MAX_MAX_COMPACT_TASKS;
// check format
if (endptr == cfgReq.value || endptr[0] != '\0') {
tFreeSMCfgDnodeReq(&cfgReq);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS,
"Invalid max compact tasks: %s", cfgReq.value);
}
// check range
if (maxCompactTasks < minMaxCompactTasks || maxCompactTasks > maxMaxCompactTasks) {
tFreeSMCfgDnodeReq(&cfgReq);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS,
"Invalid max compact tasks: %d, valid range [%d,%d]", maxCompactTasks,
minMaxCompactTasks, maxMaxCompactTasks);
}
code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
} else {
code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq);
}
@ -16382,6 +16387,24 @@ static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQ
return code;
}
static int32_t rewriteShowTransactionDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) {
SShowTransactionDetailsStmt* pShow = (SShowTransactionDetailsStmt*)(pQuery->pRoot);
SSelectStmt* pStmt = NULL;
int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, &pStmt);
if (TSDB_CODE_SUCCESS == code) {
if (NULL != pShow->pTransactionId) {
code = createOperatorNode(OP_TYPE_EQUAL, "transaction_id", pShow->pTransactionId, &pStmt->pWhere);
}
}
if (TSDB_CODE_SUCCESS == code) {
pCxt->showRewrite = true;
pQuery->showRewrite = true;
nodesDestroyNode(pQuery->pRoot);
pQuery->pRoot = (SNode*)pStmt;
}
return code;
}
static int32_t createParWhenThenNode(SNode* pWhen, SNode* pThen, SNode** ppResWhenThen) {
SWhenThenNode* pWThen = NULL;
int32_t code = nodesMakeNode(QUERY_NODE_WHEN_THEN, (SNode**)&pWThen);
@ -16957,6 +16980,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
code = rewriteShowCompactDetailsStmt(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT:
code = rewriteShowTransactionDetailsStmt(pCxt, pQuery);
break;
case QUERY_NODE_SHOW_DB_ALIVE_STMT:
case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
code = rewriteShowAliveStmt(pCxt, pQuery);

4
source/libs/scalar/src/filter.c
View File

@ -5128,8 +5128,8 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) {
SOperatorNode *pOper = (SOperatorNode *)pNode;
SValueNode *valNode = (SValueNode *)pOper->pRight;
if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
SExprNode* pLeft = (SExprNode*)pOper->pLeft;
if (IS_NUMERIC_TYPE(pLeft->resType.type) || pLeft->resType.type == TSDB_DATA_TYPE_TIMESTAMP) {
SNode* pLeft = NULL, *pRight = NULL;
int32_t code = nodesCloneNode(pOper->pLeft, &pLeft);
if (TSDB_CODE_SUCCESS != code) {

3
source/libs/stream/inc/streamInt.h
View File

@ -254,6 +254,9 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d
int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList);
int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName);
int32_t uploadCheckpointToS3(const char* id, const char* path);
int32_t deleteCheckpointFile(const char* id, const char* name);
int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock,
int32_t transId);
#ifdef __cplusplus
}

3
source/libs/stream/src/streamCheckpoint.c
View File

@ -19,7 +19,6 @@
#include "tcs.h"
static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName);
static int32_t deleteCheckpointFile(const char* id, const char* name);
static int32_t streamTaskUploadCheckpoint(const char* id, const char* path, int64_t checkpointId);
#ifdef BUILD_NO_CALL
static int32_t deleteCheckpoint(const char* id);
@ -230,7 +229,7 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream
return code;
}
static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock,
int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock,
int32_t transId) {
int32_t code = 0;
int32_t vgId = pTask->pMeta->vgId;

23
source/libs/stream/src/streamDispatch.c
View File

@ -954,7 +954,6 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, void* param, int32_t n
int32_t vgId = pTask->pMeta->vgId;
if (pTmrInfo->launchChkptId != pActiveInfo->activeId) {
streamCleanBeforeQuitTmr(pTmrInfo, param);
stWarn("s-task:%s vgId:%d ready-msg send tmr launched by previous checkpoint procedure, checkpointId:%" PRId64
", quit",
id, vgId, pTmrInfo->launchChkptId);
@ -963,13 +962,11 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, void* param, int32_t n
// active checkpoint info is cleared for now
if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (num == 0) || (pTask->chkInfo.startTs == 0)) {
streamCleanBeforeQuitTmr(pTmrInfo, param);
stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr", id, vgId);
return -1;
}
if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) {
streamCleanBeforeQuitTmr(pTmrInfo, param);
stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr", id,
vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num);
return -1;
@ -998,6 +995,7 @@ static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, in
void* p = taosArrayPush(pTmp, &pInfo->upstreamTaskId);
if (p == NULL) {
stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId);
taosArrayDestroy(pTmp);
return terrno;
} else {
stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, level,
@ -1047,13 +1045,13 @@ static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t
}
}
static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray* pNotRspList) {
static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray** pNotRspList) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr;
SArray* pList = pActiveInfo->pReadyMsgList;
int32_t num = taosArrayGetSize(pList);
int32_t vgId = pTask->pMeta->vgId;
int32_t checkpointId = pActiveInfo->activeId;
int64_t checkpointId = pActiveInfo->activeId;
const char* id = pTask->id.idStr;
int32_t notRsp = 0;
@ -1062,18 +1060,17 @@ static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray*
return code;
}
code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id);
code = doFindNotConfirmUpstream(pNotRspList, pList, num, vgId, pTask->info.taskLevel, id);
if (code) {
streamCleanBeforeQuitTmr(pTmrInfo, param);
stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr", id, tstrerror(code));
return code;
}
notRsp = taosArrayGetSize(pNotRspList);
notRsp = taosArrayGetSize(*pNotRspList);
if (notRsp == 0) {
streamClearChkptReadyMsg(pActiveInfo);
} else {
doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList);
doSendChkptReadyMsg(pTask, *pNotRspList, checkpointId, pList);
}
return code;
@ -1137,10 +1134,12 @@ static void chkptReadyMsgSendMonitorFn(void* param, void* tmrId) {
}
streamMutexLock(&pActiveInfo->lock);
code = chkptReadyMsgSendHelper(pTask, param, pNotRspList);
code = chkptReadyMsgSendHelper(pTask, param, &pNotRspList);
streamMutexUnlock(&pActiveInfo->lock);
if (code != TSDB_CODE_SUCCESS) {
streamCleanBeforeQuitTmr(pTmrInfo, param);
streamMetaReleaseTask(pTask->pMeta, pTask);
taosArrayDestroy(pNotRspList);
return;
@ -1176,7 +1175,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
int32_t num = taosArrayGetSize(pList);
if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) {
stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num,
stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d not send chkpt-ready msg", id, num,
(int32_t)taosArrayGetSize(pTask->upstreamInfo.pList));
streamMutexUnlock(&pActiveInfo->lock);
return TSDB_CODE_STREAM_INTERNAL_ERROR;
@ -1200,7 +1199,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) {
stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id);
}
} else {
stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id);
stError("s-task:%s failed to prepare the checkpoint-ready msg, try next time in 10s", id);
}
}

3
source/libs/stream/src/streamExec.c
View File

@ -915,8 +915,7 @@ int32_t streamResumeTask(SStreamTask* pTask) {
while (1) {
code = doStreamExecTask(pTask);
if (code) {
stError("s-task:%s failed to exec stream task, code:%s", id, tstrerror(code));
return code;
stError("s-task:%s failed to exec stream task, code:%s, continue", id, tstrerror(code));
}
// check if continue
streamMutexLock(&pTask->lock);

6
source/libs/stream/src/streamState.c
View File

@ -546,10 +546,6 @@ void streamStateDestroy(SStreamState* pState, bool remove) {
taosMemoryFreeClear(pState);
}
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) {
return deleteExpiredCheckPoint(pState->pFileState, mark);
}
void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); }
void streamStateCopyBackend(SStreamState* src, SStreamState* dst) {
@ -617,8 +613,6 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void**
void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); }
void streamStateSetFillInfo(SStreamState* pState) { setFillInfo(pState->pFileState); }
int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
return getRowStatePrevRow(pState->pFileState, pKey, pResKey, pVal, pVLen, pWinCode);

22
source/libs/stream/src/tstreamFileState.c
View File

@ -667,18 +667,6 @@ void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLe
}
}
int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) {
int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen);
int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey);
if (pFileState->searchBuff != NULL) {
deleteHashSortRowBuff(pFileState, pKey);
}
if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) {
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_FAILED;
}
static int32_t recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -868,10 +856,6 @@ int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId
return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf);
}
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) {
return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list);
}
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t maxCheckPointId = 0;
@ -1227,10 +1211,6 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState) {
return pFileState->pGroupIdMap;
}
void setFillInfo(SStreamFileState* pFileState) {
pFileState->hasFillCatch = false;
}
void clearExpiredState(SStreamFileState* pFileState) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
@ -1261,6 +1241,7 @@ _end:
}
}
#ifdef BUILD_NO_CALL
int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode) {
int32_t code = TSDB_CODE_SUCCESS;
@ -1328,6 +1309,7 @@ _end:
}
return code;
}
#endif
int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
int32_t* pVLen, int32_t* pWinCode) {

73
source/libs/stream/test/streamCheckPointTest.cpp
View File

@ -390,9 +390,78 @@ TEST(sstreamTaskGetTriggerRecvStatusTest, streamTaskGetTriggerRecvStatusFnTest)
extern int8_t tsS3EpNum;
tsS3EpNum = 1;
code = uploadCheckpointToS3("123", "/tmp/backend5/stream");
EXPECT_EQ(code, TSDB_CODE_SUCCESS);
code = uploadCheckpointToS3("123", "/tmp/backend5/stream/stream");
EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE);
code = downloadCheckpointByNameS3("123", "/root/download", "");
EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE);
code = deleteCheckpointFile("aaa123", "bbb");
EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE);
}
TEST(doCheckBeforeHandleChkptTriggerTest, doCheckBeforeHandleChkptTriggerFnTest) {
SStreamTask* pTask = NULL;
int64_t uid = 2222222222222;
SArray* array = taosArrayInit(4, POINTER_BYTES);
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array,
false, 1, &pTask);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
initTaskLock(pTask);
const char *path = "/tmp/doCheckBeforeHandleChkptTriggerTest/stream";
code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
SStreamState *pState = streamStateOpen((char *)path, pTask, 0, 0);
ASSERT(pState != NULL);
pTask->pBackend = pState->pTdbState->pOwner->pBackend;
code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
pTask->chkInfo.checkpointId = 123;
code = doCheckBeforeHandleChkptTrigger(pTask, 100, NULL, 0);
ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT);
pTask->chkInfo.pActiveInfo->failedId = 223;
code = doCheckBeforeHandleChkptTrigger(pTask, 200, NULL, 0);
ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT);
SStreamDataBlock block;
block.srcTaskId = 456;
SStreamTask upTask;
upTask = *pTask;
upTask.id.taskId = 456;
streamTaskSetUpstreamInfo(pTask, &upTask);
pTask->chkInfo.pActiveInfo->failedId = 23;
code = doCheckBeforeHandleChkptTrigger(pTask, 123, &block, 0);
ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT);
streamTaskSetUpstreamInfo(pTask, &upTask);
streamTaskSetStatusReady(pTask);
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
pTask->chkInfo.pActiveInfo->activeId = 223;
STaskCheckpointReadyInfo readyInfo;
readyInfo.upstreamTaskId = 4567;
block.srcTaskId = 4567;
void* pBuf = rpcMallocCont(sizeof(SMsgHead) + 1);
initRpcMsg(&readyInfo.msg, 0, pBuf, sizeof(SMsgHead) + 1);
taosArrayPush(pTask->chkInfo.pActiveInfo->pReadyMsgList, &readyInfo);
code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0);
ASSERT_NE(code, TSDB_CODE_SUCCESS);
pTask->chkInfo.pActiveInfo->allUpstreamTriggerRecv = 1;
code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0);
ASSERT_NE(code, TSDB_CODE_SUCCESS);
pTask->chkInfo.pActiveInfo->activeId = 1111;
code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0);
ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT);
}

10
source/libs/sync/src/syncMain.c
View File

@ -422,13 +422,11 @@ int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) {
SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) {
SyncIndex minMatchIndex = SYNC_INDEX_INVALID;
if (pSyncNode->peersNum > 0) {
minMatchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[0]));
}
for (int32_t i = 1; i < pSyncNode->peersNum; ++i) {
for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i]));
if (matchIndex < minMatchIndex) {
if (minMatchIndex == SYNC_INDEX_INVALID) {
minMatchIndex = matchIndex;
} else if (matchIndex > 0 && matchIndex < minMatchIndex) {
minMatchIndex = matchIndex;
}
}

177
source/libs/transport/src/transCli.c
View File

@ -151,7 +151,6 @@ typedef struct SCliThrd {
TdThreadMutex msgMtx;
SDelayQueue* delayQueue;
SDelayQueue* timeoutQueue;
SDelayQueue* waitConnQueue;
uint64_t nextTimeout; // next timeout
STrans* pInst; //
@ -159,8 +158,6 @@ typedef struct SCliThrd {
SHashObj* fqdn2ipCache;
SCvtAddr* pCvtAddr;
SHashObj* failFastCache;
SHashObj* batchCache;
SHashObj* connHeapCache;
SCliReq* stopMsg;
@ -224,8 +221,6 @@ static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
static void cliConnCb(uv_connect_t* req, int status);
static void cliAsyncCb(uv_async_t* handle);
SCliBatch* cliGetHeadFromList(SCliBatchList* pList);
static void destroyCliConnQTable(SCliConn* conn);
static void cliHandleException(SCliConn* conn);
@ -1299,8 +1294,8 @@ static void cliHandleException(SCliConn* conn) {
if (conn->registered) {
int8_t ref = transGetRefCount(conn);
if (ref == 0 && !uv_is_closing((uv_handle_t*)conn->stream)) {
// tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd,
// conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds);
// tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd,
// conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds);
uv_close((uv_handle_t*)conn->stream, cliDestroy);
}
}
@ -2124,144 +2119,7 @@ static void cliDoReq(queue* wq, SCliThrd* pThrd) {
tTrace("cli process batch size:%d", count);
}
}
SCliBatch* cliGetHeadFromList(SCliBatchList* pList) {
if (QUEUE_IS_EMPTY(&pList->wq) || pList->connCnt > pList->connMax || pList->sending > pList->connMax) {
return NULL;
}
queue* hr = QUEUE_HEAD(&pList->wq);
QUEUE_REMOVE(hr);
pList->sending += 1;
pList->len -= 1;
SCliBatch* batch = QUEUE_DATA(hr, SCliBatch, listq);
return batch;
}
static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq);
static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port);
static void destroyBatchList(SCliBatchList* pList);
static void cliBuildBatch(SCliReq* pReq, queue* h, SCliThrd* pThrd) {
int32_t code = 0;
STrans* pInst = pThrd->pInst;
SReqCtx* pCtx = pReq->ctx;
char* ip = EPSET_GET_INUSE_IP(pCtx->epSet);
uint32_t port = EPSET_GET_INUSE_PORT(pCtx->epSet);
char key[TSDB_FQDN_LEN + 64] = {0};
CONN_CONSTRUCT_HASH_KEY(key, ip, port);
size_t klen = strlen(key);
SCliBatchList** ppBatchList = taosHashGet(pThrd->batchCache, key, klen);
if (ppBatchList == NULL || *ppBatchList == NULL) {
SCliBatchList* pBatchList = NULL;
code = createBatchList(&pBatchList, key, ip, port);
if (code != 0) {
destroyReq(pReq);
return;
}
pBatchList->batchLenLimit = pInst->shareConnLimit;
SCliBatch* pBatch = NULL;
code = createBatch(&pBatch, pBatchList, pReq);
if (code != 0) {
destroyBatchList(pBatchList);
destroyReq(pReq);
return;
}
code = taosHashPut(pThrd->batchCache, key, klen, &pBatchList, sizeof(void*));
if (code != 0) {
destroyBatchList(pBatchList);
}
} else {
if (QUEUE_IS_EMPTY(&(*ppBatchList)->wq)) {
SCliBatch* pBatch = NULL;
code = createBatch(&pBatch, *ppBatchList, pReq);
if (code != 0) {
destroyReq(pReq);
cliDestroyBatch(pBatch);
}
} else {
queue* hdr = QUEUE_TAIL(&((*ppBatchList)->wq));
SCliBatch* pBatch = QUEUE_DATA(hdr, SCliBatch, listq);
if ((pBatch->shareConnLimit + pReq->msg.contLen) < (*ppBatchList)->batchLenLimit) {
QUEUE_PUSH(&pBatch->wq, h);
pBatch->shareConnLimit += pReq->msg.contLen;
pBatch->wLen += 1;
} else {
SCliBatch* tBatch = NULL;
code = createBatch(&tBatch, *ppBatchList, pReq);
if (code != 0) {
destroyReq(pReq);
}
}
}
}
return;
}
static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port) {
SCliBatchList* pBatchList = taosMemoryCalloc(1, sizeof(SCliBatchList));
if (pBatchList == NULL) {
tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
QUEUE_INIT(&pBatchList->wq);
pBatchList->port = port;
pBatchList->connMax = 1;
pBatchList->connCnt = 0;
pBatchList->batchLenLimit = 0;
pBatchList->len += 1;
pBatchList->ip = taosStrdup(ip);
pBatchList->dst = taosStrdup(key);
if (pBatchList->ip == NULL || pBatchList->dst == NULL) {
taosMemoryFree(pBatchList->ip);
taosMemoryFree(pBatchList->dst);
taosMemoryFree(pBatchList);
tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
*ppBatchList = pBatchList;
return 0;
}
static void destroyBatchList(SCliBatchList* pList) {
if (pList == NULL) {
return;
}
while (!QUEUE_IS_EMPTY(&pList->wq)) {
queue* h = QUEUE_HEAD(&pList->wq);
QUEUE_REMOVE(h);
SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq);
cliDestroyBatch(pBatch);
}
taosMemoryFree(pList->ip);
taosMemoryFree(pList->dst);
taosMemoryFree(pList);
}
static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq) {
SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch));
if (pBatch == NULL) {
tError("failed to create batch since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
QUEUE_INIT(&pBatch->wq);
QUEUE_INIT(&pBatch->listq);
QUEUE_PUSH(&pBatch->wq, &pReq->q);
pBatch->wLen += 1;
pBatch->shareConnLimit = pReq->msg.contLen;
pBatch->pList = pList;
QUEUE_PUSH(&pList->wq, &pBatch->listq);
pList->len += 1;
*ppBatch = pBatch;
return 0;
}
static void cliDoBatchReq(queue* wq, SCliThrd* pThrd) { return cliDoReq(wq, pThrd); }
static void cliAsyncCb(uv_async_t* handle) {
@ -2494,10 +2352,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) {
TAOS_CHECK_GOTO(code, NULL, _end);
}
if ((code = transDQCreate(pThrd->loop, &pThrd->waitConnQueue)) != 0) {
TAOS_CHECK_GOTO(code, NULL, _end);
}
pThrd->destroyAhandleFp = pInst->destroyFp;
pThrd->fqdn2ipCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
@ -2505,11 +2359,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) {
TAOS_CHECK_GOTO(terrno, NULL, _end);
}
pThrd->batchCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (pThrd->batchCache == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _end);
}
pThrd->connHeapCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (pThrd->connHeapCache == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end);
@ -2553,10 +2402,7 @@ _end:
transDQDestroy(pThrd->delayQueue, NULL);
transDQDestroy(pThrd->timeoutQueue, NULL);
transDQDestroy(pThrd->waitConnQueue, NULL);
taosHashCleanup(pThrd->fqdn2ipCache);
taosHashCleanup(pThrd->failFastCache);
taosHashCleanup(pThrd->batchCache);
taosHashCleanup(pThrd->pIdConnTable);
taosArrayDestroy(pThrd->pQIdBuf);
@ -2580,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) {
transDQDestroy(pThrd->delayQueue, destroyReqAndAhanlde);
transDQDestroy(pThrd->timeoutQueue, NULL);
transDQDestroy(pThrd->waitConnQueue, NULL);
tDebug("thread destroy %" PRId64, pThrd->pid);
for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) {
@ -2592,24 +2437,6 @@ static void destroyThrdObj(SCliThrd* pThrd) {
taosMemoryFree(pThrd->loop);
taosHashCleanup(pThrd->fqdn2ipCache);
void** pIter = taosHashIterate(pThrd->batchCache, NULL);
while (pIter != NULL) {
SCliBatchList* pBatchList = (SCliBatchList*)(*pIter);
while (!QUEUE_IS_EMPTY(&pBatchList->wq)) {
queue* h = QUEUE_HEAD(&pBatchList->wq);
QUEUE_REMOVE(h);
SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq);
cliDestroyBatch(pBatch);
}
taosMemoryFree(pBatchList->ip);
taosMemoryFree(pBatchList->dst);
taosMemoryFree(pBatchList);
pIter = (void**)taosHashIterate(pThrd->batchCache, pIter);
}
taosHashCleanup(pThrd->batchCache);
void* pIter2 = taosHashIterate(pThrd->connHeapCache, NULL);
while (pIter2 != NULL) {
SHeap* heap = (SHeap*)(pIter2);

15
source/libs/transport/test/transUT2.cpp
View File

@ -615,6 +615,21 @@ TEST_F(TransEnv, http) {
#endif
}
#if 1
STelemAddrMgmt mgt;
taosTelemetryMgtInit(&mgt, "telemetry.taosdata.com");
int32_t code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, "test", strlen("test"),HTTP_FLAT);
printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr);
taosMsleep(2000);
code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT);
for (int32_t i = 0; i < 1; i++) {
code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT);
printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr);
taosMsleep(2000);
}
taosTelemetryDestroy(&mgt);
#endif
{
STelemAddrMgmt mgt;
taosTelemetryMgtInit(&mgt, "error");

44
source/libs/wal/src/walRead.c
View File

@ -208,28 +208,22 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex));
SWalFileInfo *gloablPRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
if (gloablPRet == NULL) {
SWalFileInfo *globalRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
if (globalRet == NULL) {
wError("failed to find WAL log file with ver:%" PRId64, ver);
TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex));
TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER);
}
SWalFileInfo *pRet = taosMemoryMalloc(sizeof(SWalFileInfo));
if (pRet == NULL) {
wError("failed to allocate memory for localRet");
SWalFileInfo ret;
TAOS_MEMCPY(&ret, globalRet, sizeof(SWalFileInfo));
TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex));
TAOS_RETURN(terrno);
}
TAOS_MEMCPY(pRet, gloablPRet, sizeof(SWalFileInfo));
TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex));
if (pReader->curFileFirstVer != pRet->firstVer) {
if (pReader->curFileFirstVer != ret.firstVer) {
// error code was set inner
TAOS_CHECK_RETURN_WITH_FREE(walReadChangeFile(pReader, pRet->firstVer), pRet);
TAOS_CHECK_RETURN(walReadChangeFile(pReader, ret.firstVer));
}
// error code was set inner
TAOS_CHECK_RETURN_WITH_FREE(walReadSeekFilePos(pReader, pRet->firstVer, ver), pRet);
taosMemoryFree(pRet);
TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, ret.firstVer, ver));
wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver);
pReader->curVersion = ver;
@ -437,15 +431,15 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
seeked = true;
continue;
} else {
wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr());
TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex));
if (contLen < 0) {
TAOS_RETURN(terrno);
code = terrno;
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
code = TSDB_CODE_WAL_FILE_CORRUPTED;
}
wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, tstrerror(code));
TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex));
TAOS_RETURN(code);
}
}
@ -478,15 +472,15 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
}
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != cryptedBodyLen) {
wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, terrstr());
TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex));
if (contLen < 0) {
TAOS_RETURN(terrno);
code = terrno;
} else {
TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED);
code = TSDB_CODE_WAL_FILE_CORRUPTED;
}
wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s",
pReader->pWal->cfg.vgId, ver, tstrerror(code));
TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex));
TAOS_RETURN(code);
}
if (pReader->pHead->head.version != ver) {

6
source/util/src/tanalytics.c
View File

@ -20,7 +20,7 @@
#ifdef USE_ANALYTICS
#include <curl/curl.h>
#define ANAL_ALGO_SPLIT ","
#define ANALYTICS_ALOG_SPLIT_CHAR ","
typedef struct {
int64_t ver;
@ -136,7 +136,7 @@ bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue,
return false;
}
pEnd = strstr(pStart, ANAL_ALGO_SPLIT);
pEnd = strstr(pStart, ANALYTICS_ALOG_SPLIT_CHAR);
if (optMaxLen > 0) {
if (pEnd > pStart) {
int32_t len = (int32_t)(pEnd - pStart);
@ -168,7 +168,7 @@ bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValu
int32_t bufLen = tsnprintf(buf, sizeof(buf), "%s=", optName);
char *pos1 = strstr(option, buf);
char *pos2 = strstr(option, ANAL_ALGO_SPLIT);
char *pos2 = strstr(option, ANALYTICS_ALOG_SPLIT_CHAR);
if (pos1 != NULL) {
*optValue = taosStr2Int64(pos1 + bufLen, NULL, 10);
return true;

14
source/util/src/terror.c
View File

@ -326,6 +326,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While ex
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Wrong transaction execution context")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT_COMPACT, "Transaction not completed due to conflict with compact")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED, "The transaction is not able to be killed")
// mnode-mq
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists")
@ -908,12 +909,6 @@ const char* tstrerror(int32_t err) {
(void)taosThreadOnce(&tsErrorInit, tsSortError);
// this is a system errno
if ((err & 0x00ff0000) == 0x00ff0000) {
int32_t code = err & 0x0000ffff;
// strerror can handle any invalid code
// invalid code return Unknown error
return strerror(code);
}
#ifdef WINDOWS
if ((err & 0x01ff0000) == 0x01ff0000) {
snprintf(WinAPIErrDesc, 256, "windows api error, code: 0x%08x", err & 0x0000ffff);
@ -923,6 +918,13 @@ const char* tstrerror(int32_t err) {
return WinAPIErrDesc;
}
#endif
if ((err & 0x00ff0000) == 0x00ff0000) {
int32_t code = err & 0x0000ffff;
// strerror can handle any invalid code
// invalid code return Unknown error
return strerror(code);
}
int32_t s = 0;
int32_t e = sizeof(errors) / sizeof(errors[0]);

16
tests/develop-test/2-query/table_count_scan.py
View File

@ -68,7 +68,7 @@ class TDTestCase:
for i in range(0, 3):
db_name = tdSql.getData(i, 1)
if db_name == 'information_schema':
tdSql.checkData(i, 0, 36)
tdSql.checkData(i, 0, 37)
tdSql.checkData(i, 2, None)
elif db_name == 'performance_schema':
tdSql.checkData(i, 0, 5)
@ -81,7 +81,7 @@ class TDTestCase:
tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;')
tdSql.checkRows(3)
tdSql.checkData(0, 0, 36)
tdSql.checkData(0, 0, 37)
tdSql.checkData(0, 1, 'information_schema')
tdSql.checkData(0, 2, None)
tdSql.checkData(1, 0, 5)
@ -97,7 +97,7 @@ class TDTestCase:
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(0, 0, 3)
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(2, 0, 36)
tdSql.checkData(2, 0, 37)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -110,7 +110,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 44)
tdSql.checkData(0, 0, 45)
tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);')
@ -193,7 +193,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 36)
tdSql.checkData(3, 0, 37)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -208,7 +208,7 @@ class TDTestCase:
tdSql.checkData(2, 0, 5)
tdSql.checkData(2, 1, 'performance_schema')
tdSql.checkData(2, 2, None)
tdSql.checkData(3, 0, 36)
tdSql.checkData(3, 0, 37)
tdSql.checkData(3, 1, 'information_schema')
tdSql.checkData(3, 2, None)
@ -219,7 +219,7 @@ class TDTestCase:
tdSql.checkData(0, 1, 'tbl_count')
tdSql.checkData(1, 0, 5)
tdSql.checkData(1, 1, 'performance_schema')
tdSql.checkData(2, 0, 36)
tdSql.checkData(2, 0, 37)
tdSql.checkData(2, 1, 'information_schema')
tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'")
@ -232,7 +232,7 @@ class TDTestCase:
tdSql.query('select count(*) from information_schema.ins_tables')
tdSql.checkRows(1)
tdSql.checkData(0, 0, 45)
tdSql.checkData(0, 0, 46)
tdSql.execute('drop database tbl_count')

4
tests/parallel_test/cases.task
View File

@ -250,6 +250,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py
,,n,system-test,python3 ./test.py -f 2-query/large_data.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 2
@ -408,6 +409,9 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_transaction_detail.py -N 3
# ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3
# ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py

38
tests/pytest/util/cases.py
View File

@ -19,6 +19,7 @@ import inspect
import importlib
import traceback
from util.log import *
import platform
class TDCase:
@ -146,5 +147,42 @@ class TDCases:
tdLog.notice("total %d Cluster test case(s) executed" % (runNum))
def getTaosBenchmarkPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
else:
projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def taosBenchmarkExec(self, param):
buildPath = tdCases.getTaosBenchmarkPath()
if (platform.system().lower() == 'windows'):
cmdStr1 = ' mintty -h never %s %s '%(buildPath, param)
tdLog.info(cmdStr1)
os.system(cmdStr1)
else:
cmdStr1 = '%s %s &'%(buildPath, param)
tdLog.info(cmdStr1)
os.system(cmdStr1)
time.sleep(5)
tdCases = TDCases()

2
tests/pytest/util/sql.py
View File

@ -100,7 +100,7 @@ class TDSql:
if drop:
s = f'drop database if exists {dbname}'
self.cursor.execute(s)
s = f'create database {dbname}'
s = f'create database {dbname} stt_trigger 1'
for k, v in kwargs.items():
s += f" {k} {v}"
if "duration" not in kwargs:

6
tests/script/api/stmt2-performance.c
View File

@ -5,8 +5,8 @@
#include <unistd.h>
#include "taos.h"
int CTB_NUMS = 2;
int ROW_NUMS = 2;
int CTB_NUMS = 1;
int ROW_NUMS = 1;
int CYC_NUMS = 2;
void do_query(TAOS* taos, const char* sql) {
@ -217,7 +217,7 @@ int main() {
exit(1);
}
do_stmt(taos, "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)");
do_stmt(taos, "insert into `db`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)");
// do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)");
// do_taosc(taos);
taos_close(taos);

8
tests/script/sh/stop_dnodes.sh
View File

@ -13,7 +13,7 @@ if [ -n "$PID" ]; then
systemctl stop taosd
fi
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
PID=`ps -ef|grep -w taosd | grep -v grep | grep -v taosanode | awk '{print $2}'`
while [ -n "$PID" ]; do
echo kill -9 $PID
#pkill -9 taosd
@ -38,10 +38,10 @@ while [ -n "$PID" ]; do
else
lsof -nti:6030 | xargs kill -9
fi
PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'`
PID=`ps -ef|grep -w taos | grep -v grep |grep -v taosanode| awk '{print $2}'`
done
PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'`
PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode|awk '{print $2}'`
while [ -n "$PID" ]; do
echo kill -9 $PID
#pkill -9 tmq_sim
@ -52,5 +52,5 @@ while [ -n "$PID" ]; do
else
lsof -nti:6030 | xargs kill -9
fi
PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'`
PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode| awk '{print $2}'`
done

111
tests/script/tsim/analytics/basic0.sim
View File

@ -3,7 +3,17 @@ system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== create anode
print =============== failed to create anode on '127.0.0.1:1101'
sql_error create anode '127.0.0.1:1101'
sql show anodes
if $rows != 0 then
return -1
endi
sql_error drop anode 1
print ================ create anode
sql create anode '192.168.1.116:6050'
sql show anodes
@ -30,7 +40,7 @@ print $data00 $data01 $data02
sql use d0
print =============== create super table, include column type for count/sum/min/max/first
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 tinyint, c5 bigint, c6 varchar(12)) tags (t1 int unsigned)
sql show stables
if $rows != 1 then
@ -42,10 +52,11 @@ sql create table ct1 using stb tags(1000)
print ==================== insert data
# input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40]
sql insert into ct1(ts, c1) values(now-1a, 5)(now+1a, 14)(now+2a, 15)(now+3a, 15)(now+4a, 14)
sql insert into ct1(ts, c1) values(now+5a, 19)(now+6a, 17)(now+7a, 16)(now+8a, 20)(now+9a, 22)
sql insert into ct1(ts, c1) values(now+10a, 8)(now+11a, 21)(now+12a, 28)(now+13a, 11)(now+14a, 9)
sql insert into ct1(ts, c1) values(now+15a, 29)(now+16a, 40)
sql insert into ct1(ts, c1, c2, c3, c4, c5, c6) values(now-1a, 5, 5, 5, 5, 5, 'a')(now+1a, 14, 14, 14, 14, 14, 'a')(now+2a, 15, 15, 15, 15, 15, 'a')
sql insert into ct1 values(now+3a, 15, 15, 15, 15, 15, 'a')(now+4a, 14, 14, 14, 14, 14, 'a')(now+5a, 19, 19, 19, 19, 19, 'a')(now+6a, 17, 17, 17, 17, 17, 'a')
sql insert into ct1 values(now+7a, 16, 16, 16, 16, 16, 'a')(now+8a, 20, 20, 20, 20, 20, 'a')(now+9a, 22, 22, 22, 22, 22, 'a')
sql insert into ct1 values(now+10a, 8, 8, 8, 8, 8, 'a')(now+11a, 21, 21, 21, 21, 21, 'a')(now+12a, 28, 28, 28, 28, 28, 'a')(now+13a, 11, 11, 11, 11, 11, 'a')(now+14a, 9, 9, 9, 9, 9, 'a')
sql insert into ct1 values(now+15a, 29, 29, 29, 29, 29, 'a')(now+16a, 40, 40, 40, 40, 40, 'a')
sql select count(*) from ct1
if $data00 != 17 then
@ -58,6 +69,87 @@ if $data00 != 1 then
return -1
endi
print ================= try every loaded anomaly detection algorithm
sql select count(*) from ct1 anomaly_window(c1, 'algo=iqr');
sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma');
sql select count(*) from ct1 anomaly_window(c1, 'algo=lof');
sql select count(*) from ct1 anomaly_window(c1, 'algo=shesd');
sql select count(*) from ct1 anomaly_window(c1, 'algo=grubbs');
print ================= try every column type of column
sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma,k=2');
sql select count(*) from ct1 anomaly_window(c2, 'algo=ksigma,k=2');
sql select count(*) from ct1 anomaly_window(c3, 'algo=ksigma,k=2');
sql select count(*) from ct1 anomaly_window(c4, 'algo=ksigma,k=2');
sql select count(*) from ct1 anomaly_window(c5, 'algo=ksigma,k=2');
print =================== invalid column type
sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2');
sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1
sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1
sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1
sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1
sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=0') from ct1
sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=-10') from ct1
sql_error select forecast(c1, 'conf=50 ,algo = arima, every=0') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters, conf=50 ') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, ' algo=holtwinters , conf=50 ') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, ' algo = holtwinters , conf = 50 ') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1
print =================== valid column type
sql select forecast(c1, 'conf=50 ,algo = arima') from ct1
sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1
sql select forecast(c2, 'conf=50 ,algo = arima, rows=1') from ct1
sql select forecast(c3, 'conf=50 ,algo = arima, rows=1') from ct1
sql select forecast(c4, 'conf=50 ,algo = arima, rows=1') from ct1
sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1
sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1
sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1
if $rows != 10 then
return -1
endi
if $data03 != 28 then
return -1
endi
if $data00 != @23-11-15 06:13:20.000@ then
print expect 23-11-15 06:13:20.000 , actual $data00
return -1
endi
if $data10 != @23-11-15 06:13:20.002@ then
print expect 23-11-15 06:13:20.002 , actual $data10
return -1
endi
if $data20 != @23-11-15 06:13:20.004@ then
return -1
endi
print test the every option and rows option
sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=100,rows=5') from ct1
if $rows != 5 then
return -1
endi
if $data00 != @23-11-15 06:13:20.000@ then
return -1
endi
if $data10 != @23-11-15 06:13:20.100@ then
return -1
endi
sql drop anode 1
sql show anodes
@ -66,6 +158,13 @@ if $rows != 0 then
return -1
endi
sleep 1000
print ===================== query without anodes
sql_error select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1
sql_error select count(*) from ct1 anomaly_window(c1, 'algo=iqr');
_OVER:
system sh/exec.sh -n dnode1 -s stop -x SIGINT
print =============== check

2
tests/script/tsim/query/sys_tbname.sim
View File

@ -58,7 +58,7 @@ endi
sql select tbname from information_schema.ins_tables;
print $rows $data00
if $rows != 45 then
if $rows != 46 then
return -1
endi
if $data00 != @ins_tables@ then

6
tests/script/tsim/query/tableCount.sim
View File

@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou
if $rows != 3 then
return -1
endi
if $data01 != 42 then
if $data01 != 43 then
return -1
endi
if $data11 != 10 then
@ -72,7 +72,7 @@ endi
if $data11 != 5 then
return -1
endi
if $data21 != 36 then
if $data21 != 37 then
return -1
endi
if $data31 != 5 then
@ -97,7 +97,7 @@ endi
if $data42 != 3 then
return -1
endi
if $data52 != 36 then
if $data52 != 37 then
return -1
endi
if $data62 != 5 then

6
tests/system-test/0-others/information_schema.py
View File

@ -61,7 +61,7 @@ class TDTestCase:
self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\
'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\
'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views',
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets"]
'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets", "ins_transaction_details"]
self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps']
def insert_data(self,column_dict,tbname,row_num):
insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str)
@ -222,10 +222,10 @@ class TDTestCase:
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
tdLog.info(len(tdSql.queryResult))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304))
tdSql.checkEqual(True, len(tdSql.queryResult) in range(309, 310))
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
tdSql.checkEqual(60, len(tdSql.queryResult))
tdSql.checkEqual(61, len(tdSql.queryResult))
def ins_dnodes_check(self):
tdSql.execute('drop database if exists db2')

64
tests/system-test/0-others/kill_balance_leader.py Normal file
View File

@ -0,0 +1,64 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.dnodes import *
from util.sql import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
tdLog.debug(f"start to init {__file__}")
self.replicaVar = int(replicaVar)
tdSql.init(conn.cursor(), logSql)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
def run(self):
tdLog.debug(f"start to excute {__file__}")
tdSql.execute('CREATE DATABASE db vgroups 160 replica 3;')
tdSql.execute('balance vgroup leader')
sql ="show transactions;"
rows = tdSql.query(sql)
if rows > 0:
tranId = tdSql.getData(0, 0)
tdLog.info('kill transaction %d'%tranId)
tdSql.execute('kill transaction %d'%tranId, queryTimes=1 )
if self.waitTransactionZero() is False:
tdLog.exit(f"{sql} transaction not finished")
return False
def waitTransactionZero(self, seconds = 300, interval = 1):
# wait end
for i in range(seconds):
sql ="show transactions;"
rows = tdSql.query(sql)
if rows == 0:
tdLog.info("transaction count became zero.")
return True
#tdLog.info(f"i={i} wait ...")
time.sleep(interval)
return False
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

115
tests/system-test/0-others/show_transaction_detail.py Normal file
View File

@ -0,0 +1,115 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from util.log import *
from util.cases import *
from util.dnodes import *
from util.sql import *
from util.cluster import *
import threading
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
tdLog.debug(f"start to init {__file__}")
self.replicaVar = int(replicaVar)
tdSql.init(conn.cursor(), logSql)
self.dnodes = cluster.dnodes
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
def run(self):
tdLog.debug(f"start to excute {__file__}")
tdLog.info("CREATE DATABASE db1 vgroups 16 replica 1;")
tdSql.execute('CREATE DATABASE db1 vgroups 16 replica 1;')
if self.waitTransactionZero() is False:
tdLog.exit(f"{sql} transaction not finished")
return False
newTdSql1=tdCom.newTdSql()
t1 = threading.Thread(target=self.alterDbThread, args=('', newTdSql1))
newTdSql2=tdCom.newTdSql()
t2 = threading.Thread(target=self.createDbThread, args=('', newTdSql2))
t1.start()
t2.start()
#time.sleep(1)
dnode = self.dnodes[2]
# stop dnode
tdLog.info(f"stop dnode 2")
dnode.stoptaosd()
tdLog.info(f"show transactions;")
rows = tdSql.query("show transactions;", queryTimes=1)
tdLog.info(f"rows={rows}")
if rows > 0:
tranId1 = tdSql.getData(0, 0)
oper1 = tdSql.getData(0, 3)
tdLog.info(f"show transaction {tranId1}, {oper1}")
#tranId2 = tdSql.getData(1, 0)
#oper2 = tdSql.getData(1, 3)
#tdLog.info(f"show transaction {tranId2}, {oper2}")
rows = tdSql.query(f"show transaction {tranId1}", queryTimes=1)
if rows != 120 and rows != 176:
tdLog.exit(f"show transaction detial error, rows={rows}")
return False
#rows = tdSql.query(f"show transaction {tranId2}", queryTimes=1)
#if rows != 176 and rows != 120:
# tdLog.exit(f"show transaction detial error, rows={rows}")
# return False
tdLog.info(f"select * from ins_transaction_details")
rows = tdSql.query(f"select * from information_schema.ins_transaction_details", queryTimes=1)
#if rows != 296:
if rows != 176:
tdLog.exit(f"show transaction detial error, rows={rows}")
return False
dnode.starttaosd()
t1.join()
t2.join()
def createDbThread(self, sql, newTdSql):
tdLog.info("CREATE DATABASE db2 vgroups 40 replica 3;")
newTdSql.execute('CREATE DATABASE db2 vgroups 40 replica 3;', queryTimes=1)
def alterDbThread(self, sql, newTdSql):
tdLog.info("alter DATABASE db1 replica 3;")
newTdSql.execute('alter DATABASE db1 replica 3;', queryTimes=1)
def waitTransactionZero(self, seconds = 300, interval = 1):
# wait end
for i in range(seconds):
sql ="show transactions;"
rows = tdSql.query(sql)
if rows == 0:
tdLog.info("transaction count became zero.")
return True
#tdLog.info(f"i={i} wait ...")
time.sleep(interval)
return False
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

25
tests/system-test/1-insert/db_tb_name_check.py
View File

@ -78,9 +78,34 @@ class TDTestCase:
tdSql.error(f'create table {sql} (ts timestamp,c0 int)')
tdSql.execute(f'trim database `{dbname}`')
tdSql.execute(f'drop database `{dbname}`')
def tb_name_len_check(self):
dbname = tdCom.getLongName(10)
tdSql.execute(f'create database if not exists `{dbname}` vgroups 1 replica 1')
tdSql.execute(f'use `{dbname}`')
tdSql.execute(f'CREATE STABLE `test_csv` (`ts` TIMESTAMP, `c1` VARCHAR(2000), `c2` VARCHAR(2000)) TAGS (`c3` VARCHAR(2000))')
tbname = "test_csv_a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012"
tdSql.execute(f"INSERT INTO `{tbname}`\
using `test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\
(`ts`,`c1`,`c2`) values(1591060628000,'a','1');")
tdSql.query(f'select * from {tbname}')
tdSql.checkRows(1)
tdSql.execute(f'drop table {tbname}')
tdSql.execute(f"INSERT INTO `{dbname}`.`{tbname}`\
using `{dbname}`.`test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\
(`ts`,`c1`,`c2`) values(1591060628000,'a','1');")
tdSql.query(f'select * from {tbname}')
tdSql.checkRows(1)
tdSql.execute(f'drop table {tbname}')
tdSql.execute(f'trim database `{dbname}`')
tdSql.execute(f'drop database `{dbname}`')
def run(self):
self.db_name_check()
self.tb_name_check()
self.tb_name_len_check()
def stop(self):
tdSql.close()

56
tests/system-test/2-query/large_data.py Normal file
View File

@ -0,0 +1,56 @@
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
def prepare_data(self):
tdSql.execute("drop database if exists test;")
tdCases.taosBenchmarkExec("-t 2 -n 1000000 -b int,float,nchar -y")
while True:
tdSql.query("select ts from test.d0;")
num1 = tdSql.queryRows
tdSql.query("select ts from test.d1;")
num2 = tdSql.queryRows
if num1 == 1000000 and num2 == 1000000:
break
tdLog.info(f"waiting for data ready, d0: {num1}, d1: {num2}")
time.sleep(1)
def ts5803(self):
tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts;")
num1 = tdSql.queryRows
tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;")
tdSql.checkRows(num1)
tdSql.query("select d0.ts from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;")
tdSql.checkRows(num1)
tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts;")
num1 = tdSql.queryRows
tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;")
tdSql.checkRows(num1)
tdSql.query("select d0.ts from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;")
tdSql.checkRows(num1)
def run(self):
self.prepare_data()
self.ts5803()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

33
tests/system-test/2-query/smaTest.py
View File

@ -39,6 +39,27 @@ class TDTestCase:
self.create_tables();
self.ts = 1500000000000
def test_TD_33336(self):
sql = "flush database db"
tdSql.execute(sql)
time.sleep(5)
sql = f'select last(ts) + 1d, last(ts) - 1d from db.t1'
tdSql.query(sql, queryTimes=1)
lastTs_add1d = tdSql.queryResult[0][0]
lastTs_sub1d = tdSql.queryResult[0][1]
sql = f'select count(*) from db.t1 where ts < "{lastTs_add1d}" and vc1 = 1'
tdSql.query(sql, queryTimes=1)
all_row_count = tdSql.queryResult[0][0]
tdLog.debug(f"all rows: {all_row_count}")
sql = f'select count(*) from db.t1 where ts < "{lastTs_sub1d}" and vc1 = 1'
tdSql.query(sql, queryTimes=1)
row_count_sub1d = tdSql.queryResult[0][0]
tdLog.debug(f"row_count_sub1d: {row_count_sub1d}")
if row_count_sub1d > all_row_count:
tdLog.exit(f' err rows returned for sql: {sql} row_count_sub1d: {row_count_sub1d} > all_row_count: {all_row_count}')
# run case
def run(self):
@ -53,6 +74,8 @@ class TDTestCase:
# self.test_case2()
tdLog.debug(" LIMIT test_case2 ............ [OK]")
self.test_TD_33336()
# stop
def stop(self):
tdSql.close()
@ -65,11 +88,11 @@ class TDTestCase:
# create table
def create_tables(self, dbname="db"):
# super table
tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int) tags(area int)")
tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int, vc1 varchar(255)) tags(area int)")
# child table
tdSql.execute(f"create table {dbname}.t1 using {dbname}.st tags(1)")
tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) ")
tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int, vc1 varchar(255)) tags(area int) sma(i2) ")
tdSql.execute(f"create table {dbname}.t4 using {dbname}.st1 tags(1)")
return
@ -98,7 +121,11 @@ class TDTestCase:
sql = pre_insert
tdLog.debug("insert table %s rows=%d ..." % (tbname, count))
for i in range(count):
sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1)
if random.randint(0, 4) > 2:
tail = ''
else:
tail = 'asd'
sql += " (%d,%d,%d,%s)" % (ts_start + i*1000, i, i+1, '"' + str(random.randint(0,5)) + f'{tail}"')
if i > 0 and i % 20000 == 0:
tdLog.info("%d rows inserted" % i)
tdSql.execute(sql)

2
tests/system-test/2-query/union.py
View File

@ -441,7 +441,7 @@ class TDTestCase:
tdSql.checkRows(2)
sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views"
tdSql.query(sql, queryTimes=1)
tdSql.checkRows(49)
tdSql.checkRows(50)
sql = "select null union select null"
tdSql.query(sql, queryTimes=1)

90
tests/system-test/3-enterprise/restore/kill_restore_dnode.py Normal file
View File

@ -0,0 +1,90 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
sys.path.append("./3-enterprise/restore")
from restoreBasic import *
from util.common import tdCom
import threading
class TDTestCase:
# init
def init(self, conn, logSql, replicaVar=1):
tdLog.debug("start to execute %s" % __file__)
self.basic = RestoreBasic()
self.basic.init(conn, logSql, replicaVar)
# run
def run(self):
self.basic.restore_dnode_prepare(2)
self.execute()
def execute(self):
newTdSql=tdCom.newTdSql()
t0 = threading.Thread(target=self.restoreDnodeThread, args=('', newTdSql))
t0.start()
time.sleep(2)
sql ="show transactions;"
tdLog.info(sql)
rows = tdSql.query(sql)
if rows > 0:
self.basic.stop_dnode(2)
tranId = tdSql.getData(0, 0)
tdLog.info('show transaction %d'%tranId)
rows=tdSql.query('show transaction %d'%tranId, queryTimes=1)
if rows != 11:
tdLog.exit(f"restore transaction detial error, rows={rows}")
return False
tdLog.info('kill transaction %d'%tranId)
tdSql.execute('kill transaction %d'%tranId, queryTimes=1 )
time.sleep(3)
sql ="show transactions;"
tdLog.info(sql)
rows = tdSql.query(sql)
if rows > 0:
tdLog.info(f"{sql} transaction not finished")
return False
self.basic.restore_dnode_exec(2)
else:
tdLog.exit(f"{sql} no transaction exist")
return False
def restoreDnodeThread(self, p, newTdSql):
sleep(1)
sql = f"restore dnode 2"
tdLog.info(sql)
newTdSql.error(sql, expectErrInfo="Wrong transaction execution context")
tdLog.info(f"{sql} finished")
# stop
def stop(self):
self.basic.stop()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

28
tests/system-test/3-enterprise/restore/restoreBasic.py
View File

@ -143,6 +143,34 @@ class RestoreBasic:
tdSql.execute(sql)
self.check_corrent()
def restore_dnode_prepare(self, index):
tdLog.info(f"start restore dnode {index}")
dnode = self.dnodes[index - 1]
# stop dnode
tdLog.info(f"stop dnode {index}")
dnode.stoptaosd()
# remove dnode folder
try:
shutil.rmtree(dnode.dataDir)
tdLog.info(f"delete dir {dnode.dataDir} successful")
except OSError as x:
tdLog.exit(f"remove path {dnode.dataDir} error : {x.strerror}")
dnode.starttaosd()
def restore_dnode_exec(self, index):
# exec restore
sql = f"restore dnode {index}"
tdLog.info(sql)
tdSql.execute(sql)
self.check_corrent()
def stop_dnode(self, index):
dnode = self.dnodes[index - 1]
dnode.starttaosd()
# restore vnode
def restore_vnode(self, index):
tdLog.info(f"start restore vnode on dnode {index}")