diff --git a/docs/en/08-operation/04-maintenance.md b/docs/en/08-operation/04-maintenance.md index 970ee40d18..2f6afbf9df 100644 --- a/docs/en/08-operation/04-maintenance.md +++ b/docs/en/08-operation/04-maintenance.md @@ -17,7 +17,7 @@ TDengine is designed for various writing scenarios, and many of these scenarios ```sql COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']; -SHOW COMPACTS [compact_id]; +SHOW COMPACT [compact_id]; KILL COMPACT compact_id; ``` diff --git a/docs/en/14-reference/01-components/01-taosd.md b/docs/en/14-reference/01-components/01-taosd.md index 1b7f63510b..c86b631df4 100644 --- a/docs/en/14-reference/01-components/01-taosd.md +++ b/docs/en/14-reference/01-components/01-taosd.md @@ -190,7 +190,8 @@ The effective value of charset is UTF-8. |Parameter Name |Supported Version |Dynamic Modification|Description| |-----------------------|-------------------------|--------------------|------------| |supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5| -|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4| +|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-1024, default value 4| +|numOfCompactThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-16, default value 2| |numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| |numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)| |numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)| diff --git a/docs/en/14-reference/03-taos-sql/24-show.md b/docs/en/14-reference/03-taos-sql/24-show.md index 36c20df0b4..b46fb41fa0 100644 --- a/docs/en/14-reference/03-taos-sql/24-show.md +++ b/docs/en/14-reference/03-taos-sql/24-show.md @@ -304,9 +304,10 @@ Displays information about all topics in the current database. ```sql SHOW TRANSACTIONS; +SHOW TRANSACTION [tranaction_id]; ``` -Displays information about transactions currently being executed in the system (these transactions are only for metadata level, not for regular tables). +Displays information about one of or all transaction(s) currently being executed in the system (these transactions are only for metadata level, not for regular tables). ## SHOW USERS diff --git a/docs/zh/08-operation/04-maintenance.md b/docs/zh/08-operation/04-maintenance.md index 9ef165179d..429542485d 100644 --- a/docs/zh/08-operation/04-maintenance.md +++ b/docs/zh/08-operation/04-maintenance.md @@ -19,7 +19,7 @@ TDengine 面向多种写入场景，而很多写入场景下，TDengine 的存 ```SQL COMPACT DATABASE db_name [start with 'XXXX'] [end with 'YYYY']； COMPACT [db_name.]VGROUPS IN (vgroup_id1, vgroup_id2, ...) [start with 'XXXX'] [end with 'YYYY']； -SHOW COMPACTS [compact_id]； +SHOW COMPACT [compact_id]； KILL COMPACT compact_id； ``` diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index 7a0a306a41..74e6f485ff 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -185,7 +185,8 @@ charset 的有效值是 UTF-8。 |参数名称|支持版本|动态修改|参数含义| |--------------------------|----------|-------------------------|-| |supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| -|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4| +|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-1024，默认值为 4| +|numOfCompactThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-16，默认值为 2| |numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| |numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| |numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index ca2170db8b..cf29ef1451 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -6,7 +6,7 @@ description: 对表的各种管理操作 ## 创建表 -`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表。 +`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表（也可以通过指定 TAGS 字段创建超级表）。 ```sql CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] diff --git a/docs/zh/14-reference/03-taos-sql/24-show.md b/docs/zh/14-reference/03-taos-sql/24-show.md index 4596a03281..3898920e65 100644 --- a/docs/zh/14-reference/03-taos-sql/24-show.md +++ b/docs/zh/14-reference/03-taos-sql/24-show.md @@ -306,9 +306,10 @@ SHOW TOPICS; ```sql SHOW TRANSACTIONS; +SHOW TRANSACTION [tranaction_id]; ``` -显示当前系统中正在执行的事务的信息(该事务仅针对除普通表以外的元数据级别) +显示当前系统中正在执行的所有或者某一个事务的信息(该事务仅针对除普通表以外的元数据级别) ## SHOW USERS diff --git a/include/common/systable.h b/include/common/systable.h index bd8ba76f4f..fe867c9ad0 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -63,6 +63,7 @@ extern "C" { #define TSDB_INS_TABLE_TSMAS "ins_tsmas" #define TSDB_INS_DISK_USAGE "ins_disk_usage" #define TSDB_INS_TABLE_FILESETS "ins_filesets" +#define TSDB_INS_TABLE_TRANSACTION_DETAILS "ins_transaction_details" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFS_TABLE_SMAS "perf_smas" diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 0b34e882c8..292e7f561a 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -182,7 +182,7 @@ void tColDataClear(SColData *pColData); void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward); -void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); +int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 501f1cabc1..ec130ad6b5 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -112,9 +112,8 @@ extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsQueueMemoryAllowed; extern int32_t tsRetentionSpeedLimitMB; -extern const char *tsAlterCompactTaskKeywords; -extern int32_t tsNumOfCompactThreads; -extern int32_t tsNumOfRetentionThreads; +extern int32_t tsNumOfCompactThreads; +extern int32_t tsNumOfRetentionThreads; // sync raft extern int32_t tsElectInterval; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0b6a8b3f1b..9488e54615 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -163,6 +163,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_ANODE_FULL, TSDB_MGMT_TABLE_USAGE, TSDB_MGMT_TABLE_FILESETS, + TSDB_MGMT_TABLE_TRANSACTION_DETAIL, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -405,6 +406,7 @@ typedef enum ENodeType { QUERY_NODE_SHOW_CREATE_TSMA_STMT, QUERY_NODE_DROP_TSMA_STMT, QUERY_NODE_SHOW_FILESETS_STMT, + QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index a2be706dc0..7a4401827c 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -389,7 +389,6 @@ typedef struct SStateStore { int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - void (*streamStateSetFillInfo)(SStreamState* pState); void (*streamStateClearExpiredState)(SStreamState* pState); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, @@ -455,7 +454,6 @@ typedef struct SStateStore { int32_t (*streamStateBegin)(SStreamState* pState); void (*streamStateCommit)(SStreamState* pState); void (*streamStateDestroy)(SStreamState* pState, bool remove); - int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst); } SStateStore; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8eb30b8184..0f736e7068 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -439,6 +439,11 @@ typedef struct SShowCompactDetailsStmt { SNode* pCompactId; } SShowCompactDetailsStmt; +typedef struct SShowTransactionDetailsStmt { + ENodeType type; + SNode* pTransactionId; +} SShowTransactionDetailsStmt; + typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef struct SIndexOptions { diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 2179547352..b4e0087b1a 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -34,7 +34,6 @@ void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); void streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); -int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); @@ -108,7 +107,6 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); // twa -void streamStateSetFillInfo(SStreamState* pState); void streamStateClearExpiredState(SStreamState* pState); void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index f1f5b00e38..f47c308e18 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -67,7 +67,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); -int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState); void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e317fdd65a..890758762a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -411,6 +411,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8) #define TSDB_CODE_MND_TRANS_CONFLICT_COMPACT TAOS_DEF_ERROR_CODE(0, 0x03D9) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) +#define TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED TAOS_DEF_ERROR_CODE(0, 0x03D2) // mnode-mq #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E0) diff --git a/include/util/tdef.h b/include/util/tdef.h index 2ee84b42bd..2b0aa00b1a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -329,7 +329,11 @@ typedef enum ELogicConditionType { #define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 512 +#define TSDB_TRANS_ERROR_LEN 512 +#define TSDB_TRANS_OBJTYPE_LEN 40 +#define TSDB_TRANS_RESULT_LEN 100 +#define TSDB_TRANS_TARGET_LEN 300 +#define TSDB_TRANS_DETAIL_LEN 100 #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index acd118acc9..d48142811c 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -848,6 +848,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) { STMT_DLOG("start to set dbName: %s", dbName); pStmt->db = taosStrdup(dbName); + (void)strdequote(pStmt->db); STMT_ERR_RET(stmtCreateRequest(pStmt)); // The SQL statement specifies a database name, overriding the previously specified database diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 3e4667fbe7..60f0a72e39 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -126,6 +126,10 @@ void fetchCallback(void* param, void* res, int32_t numOfRow) { void queryCallback(void* param, void* res, int32_t code) { if (code != TSDB_CODE_SUCCESS) { (void)printf("failed to execute, reason:%s\n", taos_errstr(res)); + taos_free_result(res); + tsem_t *sem = (tsem_t *)param; + tsem_post(sem); + return; } (void)printf("start to fetch data\n"); taos_fetch_raw_block_a(res, fetchCallback, param); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cb08046399..38dce105bb 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -314,6 +314,8 @@ static const SSysDbTableSchema transSchema[] = { {.name = "oper", .bytes = TSDB_TRANS_OPER_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "stable", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "killable", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + //{.name = "kill_mnode", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "failed_times", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "last_action_info", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -403,6 +405,15 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { {.name = "remain_time(s)", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; +static const SSysDbTableSchema userTransactionDetailSchema[] = { + {.name = "transaction_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "action", .bytes = 30 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "obj_type", .bytes = TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "result", .bytes = TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "target", .bytes = TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "detail", .bytes = TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, +}; + static const SSysDbTableSchema anodesSchema[] = { {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "url", .bytes = TSDB_ANALYTIC_ANODE_URL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -521,6 +532,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_ANODES_FULL, anodesFullSchema, tListLen(anodesFullSchema), true}, {TSDB_INS_DISK_USAGE, diskUsageSchema, tListLen(diskUsageSchema), false}, {TSDB_INS_TABLE_FILESETS, filesetsFullSchema, tListLen(filesetsFullSchema), false}, + {TSDB_INS_TABLE_TRANSACTION_DETAILS, userTransactionDetailSchema, tListLen(userTransactionDetailSchema), false}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f1aacfed15..3efccf23b4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2671,8 +2671,13 @@ static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal tColDataGetValue6, // HAS_VALUE | HAS_NULL tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE }; -void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { +int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { + if (iVal < 0 || iVal >= pColData->nVal || + (pColData->flag <= 0 || pColData->flag >= sizeof(tColDataGetValueImpl)/POINTER_BYTES)){ + return TSDB_CODE_INVALID_PARA; + } tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); + return TSDB_CODE_SUCCESS; } uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { @@ -3436,7 +3441,10 @@ static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, S for (int32_t i = 0; i < nColData; i++) { SColVal cv = {0}; - tColDataGetValue(&aFromColData[i], iFromRow, &cv); + code = tColDataGetValue(&aFromColData[i], iFromRow, &cv); + if (code != TSDB_CODE_SUCCESS) { + return code; + } code = tColDataAppendValue(&aToColData[i], &cv); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3575,7 +3583,10 @@ static int32_t tColDataMerge(SArray **colArr) { SColData *dstCol = taosArrayGet(dst, j); SColVal cv; - tColDataGetValue(srcCol, i, &cv); + code = tColDataGetValue(srcCol, i, &cv); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } code = tColDataAppendValue(dstCol, &cv); if (code) { goto _exit; @@ -3588,7 +3599,10 @@ static int32_t tColDataMerge(SArray **colArr) { SColData *dstCol = taosArrayGet(dst, j); SColVal cv; - tColDataGetValue(srcCol, i, &cv); + code = tColDataGetValue(srcCol, i, &cv); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } code = tColDataUpdateValue(dstCol, &cv, true); if (code) { goto _exit; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c58ad32a18..b298cac725 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -102,9 +102,8 @@ int32_t tsMaxStreamBackendCache = 128; // M int32_t tsPQSortMemThreshold = 16; // M int32_t tsRetentionSpeedLimitMB = 0; // unlimited -const char *tsAlterCompactTaskKeywords = "max_compact_tasks"; -int32_t tsNumOfCompactThreads = 2; -int32_t tsNumOfRetentionThreads = 1; +int32_t tsNumOfCompactThreads = 2; +int32_t tsNumOfRetentionThreads = 1; // sync raft int32_t tsElectInterval = 25 * 1000; @@ -745,8 +744,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); - - TAOS_CHECK_RETURN(cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + + TAOS_CHECK_RETURN( + cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -795,9 +795,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - tsNumOfCompactThreads = tsNumOfCommitThreads; - tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); - tsNumOfSupportVnodes = tsNumOfCores * 2 + 5; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); @@ -842,7 +839,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); - TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCompactThreads", tsNumOfCompactThreads, 1, 16, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0); @@ -1040,10 +1037,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(pCfg, "maxCompactConcurrency"); + pItem = cfgGetItem(pCfg, "numOfCompactThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfCompactThreads = numOfCores / 2; - tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4); pItem->i32 = tsNumOfCompactThreads; pItem->stype = stype; } @@ -1548,7 +1543,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads"); tsNumOfCommitThreads = pItem->i32; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency"); + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCompactThreads"); tsNumOfCompactThreads = pItem->i32; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB"); @@ -2354,6 +2349,8 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, TAOS_RETURN(code); } +extern void tsdbAlterNumCompactThreads(); + static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = -1; @@ -2404,6 +2401,17 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) { goto _exit; } + if (strcasecmp(name, "numOfCompactThreads") == 0) { +#ifdef TD_ENTERPRISE + tsNumOfCompactThreads = pItem->i32; + code = TSDB_CODE_SUCCESS; + // tsdbAlterNumCompactThreads(); +#else + code = TSDB_CODE_INVALID_CFG; +#endif + goto _exit; + } + { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index ebf91025bb..10c1077697 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -449,6 +449,20 @@ static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) { checkSColVal(data[i], &colVal, pCol->type); } } +#ifndef WINDOWS +TEST(testCase, tColDataGetValue) { + SColData pColData = {0}; + SColVal pColVal = {0}; + ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0); + + pColData = {.flag = 8}; + pColVal = {0}; + ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0); + + pColData = {.nVal = 1, .flag = 8}; + ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0); +} +#endif TEST(testCase, AllNormTest) { int16_t nCols = 14; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 8139e4aa98..423268a8cc 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -475,27 +475,6 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } -extern void tsdbAlterNumCompactThreads(); -static int32_t dmAlterMaxCompactTask(const char *value) { - int32_t max_compact_tasks; - char *endptr = NULL; - - max_compact_tasks = taosStr2Int32(value, &endptr, 10); - if (endptr == value || endptr[0] != '\0') { - return TSDB_CODE_INVALID_MSG; - } - - if (max_compact_tasks != tsNumOfCompactThreads) { - dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks); - tsNumOfCompactThreads = max_compact_tasks; -#ifdef TD_ENTERPRISE - (void)tsdbAlterNumCompactThreads(); -#endif - } - - return TSDB_CODE_SUCCESS; -} - int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t code = 0; SDCfgDnodeReq cfgReq = {0}; @@ -509,10 +488,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return taosUpdateTfsItemDisable(pCfg, cfgReq.value, pMgmt->pTfs); } - if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) { - return dmAlterMaxCompactTask(cfgReq.value); - } - dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true); @@ -706,6 +681,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { return TSDB_CODE_INVALID_MSG; } + dInfo("retrieve table:%s, user:%s, compactId:%" PRId64, retrieveReq.tb, retrieveReq.user, retrieveReq.compactId); #if 0 if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { code = TSDB_CODE_MND_NO_RIGHTS; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 8e595f76c9..165437ed28 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -129,7 +129,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e3d2ad6d34..b32274e0a7 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -133,6 +133,12 @@ typedef enum { TRN_EXEC_SERIAL = 1, } ETrnExec; +typedef enum { + TRN_KILL_MODE_SKIP = 0, + TRN_KILL_MODE_INTERUPT = 1, + //TRN_KILL_MODE_ROLLBACK = 2, +} ETrnKillMode; + typedef enum { DND_REASON_ONLINE = 0, DND_REASON_STATUS_MSG_TIMEOUT, @@ -201,6 +207,8 @@ typedef struct { SRWLatch lockRpcArray; int64_t mTraceId; TdThreadMutex mutex; + bool ableToBeKilled; + ETrnKillMode killMode; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 7f039bc21f..05280d0d68 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -54,6 +54,8 @@ typedef struct { SSdbRaw *pRaw; int64_t mTraceId; + int64_t startTime; + int64_t endTime; } STransAction; typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); @@ -80,6 +82,8 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, voi void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId); void mndTransSetSerial(STrans *pTrans); +void mndTransSetBeKilled(STrans *pTrans, bool ableToBeKilled); +void mndTransSetKillMode(STrans *pTrans, ETrnKillMode killMode); void mndTransSetParallel(STrans *pTrans); void mndTransSetChangeless(STrans *pTrans); void mndTransSetOper(STrans *pTrans, EOperType oper); diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index 9a053066b2..0052f5de56 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -45,6 +45,8 @@ int32_t mndRetrieveCompactDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char *sep = NULL; SDbObj *pDb = NULL; + mInfo("retrieve compact detail"); + if (strlen(pShow->db) > 0) { sep = strchr(pShow->db, '.'); if (sep && diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c28fd343f0..1efbff9d53 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1270,6 +1270,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p TAOS_RETURN(code); } mInfo("trans:%d, used to alter db:%s", pTrans->id, pOld->name); + mInfo("trans:%d, used to alter db, ableToBeKilled:%d, killMode:%d", pTrans->id, pTrans->ableToBeKilled, pTrans->killMode); mndTransSetDbName(pTrans, pOld->name, NULL); TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER); @@ -1278,6 +1279,8 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p TAOS_CHECK_GOTO(mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew), NULL, _OVER); TAOS_CHECK_GOTO(mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew), NULL, _OVER); TAOS_CHECK_GOTO(mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew), NULL, _OVER); + + mInfo("trans:%d, used to alter db, ableToBeKilled:%d, killMode:%d", pTrans->id, pTrans->ableToBeKilled, pTrans->killMode); TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER); code = 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 49dc62d471..f19eabd885 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -132,6 +132,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_COMPACT; } else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) { type = TSDB_MGMT_TABLE_COMPACT_DETAIL; + } else if (strncasecmp(name, TSDB_INS_TABLE_TRANSACTION_DETAILS, len) == 0) { + type = TSDB_MGMT_TABLE_TRANSACTION_DETAIL; } else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_FULL, len) == 0) { type = TSDB_MGMT_TABLE_GRANTS_FULL; } else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_LOGS, len) == 0) { @@ -236,7 +238,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { SRetrieveTableReq retrieveReq = {0}; TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq)); - mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb); + mDebug("process to retrieve systable req db:%s, tb:%s, compactId:%" PRId64, retrieveReq.db, retrieveReq.tb, + retrieveReq.compactId); if (retrieveReq.showId == 0) { STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 718b7d0df6..83fad45919 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -14,19 +14,21 @@ */ #define _DEFAULT_SOURCE +#include "mndTrans.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" #include "mndSubscribe.h" #include "mndSync.h" -#include "mndTrans.h" #include "mndUser.h" +#include "mndVgroup.h" +#include "osTime.h" #define TRANS_VER1_NUMBER 1 #define TRANS_VER2_NUMBER 2 #define TRANS_ARRAY_SIZE 8 -#define TRANS_RESERVE_SIZE 44 +#define TRANS_RESERVE_SIZE 42 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); @@ -70,7 +72,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter); - +static int32_t mndRetrieveTransDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t tsMaxTransId = 0; int32_t mndInitTrans(SMnode *pMnode) { @@ -89,6 +91,7 @@ int32_t mndInitTrans(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndCancelGetNextTrans); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANSACTION_DETAIL, mndRetrieveTransDetail); return sdbSetTable(pMnode->pSdb, table); } @@ -156,7 +159,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans) { int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_INVALID_MSG; - int8_t sver = taosArrayGetSize(pTrans->prepareActions) ? TRANS_VER2_NUMBER : TRANS_VER1_NUMBER; + int8_t sver = TRANS_VER2_NUMBER; int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen; rawDataLen += mndTransGetActionsSize(pTrans->prepareActions); @@ -220,6 +223,11 @@ SSdbRaw *mndTransEncode(STrans *pTrans) { pIter = taosHashIterate(pTrans->arbGroupIds, pIter); } + if (sver > TRANS_VER1_NUMBER) { + SDB_SET_INT8(pRaw, dataPos, pTrans->ableToBeKilled, _OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->killMode, _OVER) + } + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -310,7 +318,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TRANS_VER1_NUMBER && sver != TRANS_VER2_NUMBER) { + if (sver > TRANS_VER2_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -389,6 +397,13 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { if ((terrno = taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0)) != 0) goto _OVER; } + int8_t ableKill = 0; + int8_t killMode = 0; + SDB_GET_INT8(pRaw, dataPos, &ableKill, _OVER) + SDB_GET_INT8(pRaw, dataPos, &killMode, _OVER) + pTrans->ableToBeKilled = ableKill; + pTrans->killMode = killMode; + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; @@ -430,12 +445,25 @@ static const char *mndTransStr(ETrnStage stage) { } } +static const char *mndTransTypeStr(ETrnAct actionType) { + switch (actionType) { + case TRANS_ACTION_MSG: + return "msg"; + case TRANS_ACTION_RAW: + return "sdb"; + default: + return "invalid"; + } +} + static void mndSetTransLastAction(STrans *pTrans, STransAction *pAction) { if (pAction != NULL) { - pTrans->lastAction = pAction->id; - pTrans->lastMsgType = pAction->msgType; - pTrans->lastEpset = pAction->epSet; - pTrans->lastErrorNo = pAction->errCode; + if (pAction->errCode != TSDB_CODE_ACTION_IN_PROGRESS) { + pTrans->lastAction = pAction->id; + pTrans->lastMsgType = pAction->msgType; + pTrans->lastEpset = pAction->epSet; + pTrans->lastErrorNo = pAction->errCode; + } } else { pTrans->lastAction = 0; pTrans->lastMsgType = 0; @@ -636,6 +664,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->policy = policy; pTrans->conflict = conflict; pTrans->exec = TRN_EXEC_PARALLEL; + pTrans->ableToBeKilled = false; pTrans->createdTime = taosGetTimestampMs(); pTrans->prepareActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); @@ -804,6 +833,13 @@ void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) { void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } +void mndTransSetBeKilled(STrans *pTrans, bool ableToBeKilled) { pTrans->ableToBeKilled = ableToBeKilled; } + +void mndTransSetKillMode(STrans *pTrans, ETrnKillMode killMode) { + pTrans->ableToBeKilled = true; + pTrans->killMode = killMode; +} + void mndTransSetParallel(STrans *pTrans) { pTrans->exec = TRN_EXEC_PARALLEL; } void mndTransSetChangeless(STrans *pTrans) { pTrans->changeless = true; } @@ -1043,6 +1079,39 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return TSDB_CODE_INVALID_PARA; } + mInfo("trans:%d, action list:", pTrans->id); + int32_t index = 0; + for (int32_t i = 0; i < taosArrayGetSize(pTrans->prepareActions); ++i, ++index) { + STransAction *pAction = taosArrayGet(pTrans->prepareActions, i); + mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index, + mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); + } + + for (int32_t i = 0; i < taosArrayGetSize(pTrans->redoActions); ++i, ++index) { + STransAction *pAction = taosArrayGet(pTrans->redoActions, i); + mInfo("trans:%d, action:%d, %s:%d msgType:%s", pTrans->id, index, + mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType));; + } + + for (int32_t i = 0; i < taosArrayGetSize(pTrans->commitActions); ++i, ++index) { + STransAction *pAction = taosArrayGet(pTrans->commitActions, i); + mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index, + mndTransStr(pAction->stage), i, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); + } + + for (int32_t i = 0; i < taosArrayGetSize(pTrans->undoActions); ++i, ++index) { + STransAction *pAction = taosArrayGet(pTrans->undoActions, i); + if(pAction->actionType == TRANS_ACTION_MSG){ + mInfo("trans:%d, action:%d, %s:%d msgType:%s", pTrans->id, index, + mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType));; + } + else{ + mInfo("trans:%d, action:%d, %s:%d sdbType:%s, sdbStatus:%s", pTrans->id, index, + mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), sdbStatusName(pAction->pRaw->status)); + } + } + + TAOS_CHECK_RETURN(mndTransCheckConflict(pMnode, pTrans)); TAOS_CHECK_RETURN(mndTransCheckParallelActions(pMnode, pTrans)); @@ -1260,7 +1329,10 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { if (pAction != NULL) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; - pTrans->lastErrorNo = pRsp->code; + pAction->endTime = taosGetTimestampMs(); + + // pTrans->lastErrorNo = pRsp->code; + mndSetTransLastAction(pTrans, pAction); mInfo("trans:%d, %s:%d response is received, received code:0x%x(%s), accept:0x%x(%s) retry:0x%x(%s)", transId, mndTransStr(pAction->stage), action, pRsp->code, tstrerror(pRsp->code), pAction->acceptableCode, @@ -1374,6 +1446,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio pAction->msgSent = 1; // pAction->msgReceived = 0; pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS; + pAction->startTime = taosGetTimestampMs(); + pAction->endTime = 0; mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail); mndSetTransLastAction(pTrans, pAction); @@ -1527,8 +1601,9 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pActions, action); - mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d", pTrans->id, pTrans->actionPos, - mndTransStr(pAction->stage), pAction->actionType); + mInfo("trans:%d, current action:%d, stage:%s, actionType(1:msg,2:log):%d, msgSent:%d, msgReceived:%d", + pTrans->id, pTrans->actionPos, mndTransStr(pAction->stage), pAction->actionType, pAction->msgSent, + pAction->msgReceived); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { @@ -1924,13 +1999,25 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { TAOS_RETURN(TSDB_CODE_MND_TRANS_INVALID_STAGE); } - for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - STransAction *pAction = taosArrayGet(pArray, i); - mInfo("trans:%d, %s:%d set processed for kill msg received, errCode from %s to success", pTrans->id, - mndTransStr(pAction->stage), i, tstrerror(pAction->errCode)); - pAction->msgSent = 1; - pAction->msgReceived = 1; - pAction->errCode = 0; + if(pTrans->ableToBeKilled == false){ + return TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED; + } + + if(pTrans->killMode == TRN_KILL_MODE_SKIP){ + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + STransAction *pAction = taosArrayGet(pArray, i); + mInfo("trans:%d, %s:%d set processed for kill msg received, errCode from %s to success", pTrans->id, + mndTransStr(pAction->stage), i, tstrerror(pAction->errCode)); + pAction->msgSent = 1; + pAction->msgReceived = 1; + pAction->errCode = 0; + } + } + else if(pTrans->killMode == TRN_KILL_MODE_INTERUPT){ + pTrans->stage = TRN_STAGE_PRE_FINISH; + } + else{ + return TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED; } mndTransExecute(pMnode, pTrans); @@ -2002,6 +2089,114 @@ void mndTransPullup(SMnode *pMnode) { taosArrayDestroy(pArray); } +static char *formatTimestamp(char *buf, int64_t val, int precision) { + time_t tt; + if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + } + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + } else { + tt = (time_t)(val / 1000); + } + + struct tm tm; + if (taosLocalTime(&tt, &tm, NULL, 0, NULL) == NULL) { + mError("failed to get local time"); + return NULL; + } + size_t pos = taosStrfTime(buf, 32, "%Y-%m-%d %H:%M:%S", &tm); + + if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + } else if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", (int)(val % 1000000000)); + } else { + sprintf(buf + pos, ".%03d", (int)(val % 1000)); + } + + return buf; +} + +static void mndTransLogAction(STrans *pTrans) { + char detail[512] = {0}; + int32_t len = 0; + int32_t index = 0; + + if (pTrans->stage == TRN_STAGE_PREPARE) { + for (int32_t i = 0; i < taosArrayGetSize(pTrans->prepareActions); ++i, ++index) { + len = 0; + STransAction *pAction = taosArrayGet(pTrans->prepareActions, i); + len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index, + mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), + sdbStatusName(pAction->pRaw->status)); + mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail); + } + } + + if (pTrans->stage == TRN_STAGE_REDO_ACTION) { + for (int32_t i = 0; i < taosArrayGetSize(pTrans->redoActions); ++i, ++index) { + len = 0; + STransAction *pAction = taosArrayGet(pTrans->redoActions, i); + if (pAction->actionType == TRANS_ACTION_MSG) { + char bufStart[40] = {0}; + (void)formatTimestamp(bufStart, pAction->startTime, TSDB_TIME_PRECISION_MILLI); + + char endStart[40] = {0}; + (void)formatTimestamp(endStart, pAction->endTime, TSDB_TIME_PRECISION_MILLI); + len += snprintf(detail + len, sizeof(detail) - len, + "action:%d, %s:%d msgType:%s," + "sent:%d, received:%d, startTime:%s, endTime:%s, ", + index, mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType), pAction->msgSent, + pAction->msgReceived, bufStart, endStart); + + SEpSet epset = pAction->epSet; + if (epset.numOfEps > 0) { + len += snprintf(detail + len, sizeof(detail) - len, "numOfEps:%d inUse:%d ", epset.numOfEps, epset.inUse); + for (int32_t i = 0; i < epset.numOfEps; ++i) { + len += + snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); + } + } + + len += snprintf(detail + len, sizeof(detail) - len, ", errCode:0x%x(%s)\n", pAction->errCode & 0xFFFF, + tstrerror(pAction->errCode)); + } else { + len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s, written:%d\n", + index, mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), + sdbStatusName(pAction->pRaw->status), pAction->rawWritten); + } + mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail); + } + } + + if (pTrans->stage == TRN_STAGE_COMMIT_ACTION) { + for (int32_t i = 0; i < taosArrayGetSize(pTrans->commitActions); ++i, ++index) { + len = 0; + STransAction *pAction = taosArrayGet(pTrans->commitActions, i); + len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index, + mndTransStr(pAction->stage), i, sdbTableName(pAction->pRaw->type), + sdbStatusName(pAction->pRaw->status)); + mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail); + } + + for (int32_t i = 0; i < taosArrayGetSize(pTrans->undoActions); ++i, ++index) { + len = 0; + STransAction *pAction = taosArrayGet(pTrans->undoActions, i); + if (pAction->actionType == TRANS_ACTION_MSG) { + len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d msgType:%s\n", index, + mndTransStr(pAction->stage), pAction->id, TMSG_INFO(pAction->msgType)); + ; + } else { + len += snprintf(detail + len, sizeof(detail) - len, "action:%d, %s:%d sdbType:%s, sdbStatus:%s\n", index, + mndTransStr(pAction->stage), pAction->id, sdbTableName(pAction->pRaw->type), + sdbStatusName(pAction->pRaw->status)); + } + mDebug("trans:%d, show tran action, detail:%s", pTrans->id, detail); + } + } +} + static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -2044,6 +2239,20 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)stbname, false), pTrans, &lino, _OVER); + const char *killableStr = pTrans->ableToBeKilled ? "yes" : "no"; + char killableVstr[10 + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(killableVstr, killableStr, 10 + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)killableVstr, false), pTrans, &lino, _OVER); + + /* + const char *killModeStr = pTrans->killMode == TRN_KILL_MODE_SKIP ? "skip" : "interrupt"; + char killModeVstr[10 + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(killModeVstr, killModeStr, 24); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)killModeVstr, false), pTrans, &lino, _OVER); + */ + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&pTrans->failedTimes, false), pTrans, &lino, _OVER); @@ -2061,13 +2270,15 @@ static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl len += tsnprintf(detail + len, sizeof(detail) - len, "msgType:%s numOfEps:%d inUse:%d ", TMSG_INFO(pTrans->lastMsgType), epset.numOfEps, epset.inUse); for (int32_t i = 0; i < pTrans->lastEpset.numOfEps; ++i) { - len += tsnprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); + len += snprintf(detail + len, sizeof(detail) - len, "ep:%d-%s:%u ", i, epset.eps[i].fqdn, epset.eps[i].port); } } STR_WITH_MAXSIZE_TO_VARSTR(lastInfo, detail, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); RETRIEVE_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)lastInfo, false), pTrans, &lino, _OVER); + mndTransLogAction(pTrans); + numOfRows++; sdbRelease(pSdb, pTrans); } @@ -2078,6 +2289,239 @@ _OVER: return numOfRows; } +static int32_t mndShowTransCommonColumns(SShowObj *pShow, SSDataBlock *pBlock, STransAction *pAction, + int32_t transactionId, int32_t curActionId, int32_t numOfRows, int32_t *cols) { + int32_t code = 0; + int32_t lino = 0; + int32_t len = 0; + + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, (*cols)++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)&transactionId, false), &lino, _OVER); + + char action[30 + 1] = {0}; + if (curActionId == pAction->id) { + len += snprintf(action + len, sizeof(action) - len, "%s:%d(%s)<-last", mndTransStr(pAction->stage), pAction->id, + mndTransTypeStr(pAction->actionType)); + } else { + len += snprintf(action + len, sizeof(action) - len, "%s:%d(%s)", mndTransStr(pAction->stage), pAction->id, + mndTransTypeStr(pAction->actionType)); + } + char actionVStr[30 + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(actionVStr, action, pShow->pMeta->pSchemas[*cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, (*cols)++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)actionVStr, false), &lino, _OVER); +_OVER: + if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code)); + return code; +} + +static void mndShowTransAction(SShowObj *pShow, SSDataBlock *pBlock, STransAction *pAction, int32_t transactionId, + int32_t curActionId, int32_t rows, int32_t numOfRows) { + int32_t code = 0; + int32_t lino = 0; + int32_t len = 0; + int32_t cols = 0; + + cols = 0; + + if (mndShowTransCommonColumns(pShow, pBlock, pAction, transactionId, curActionId, numOfRows, &cols) != 0) return; + + if (pAction->actionType == TRANS_ACTION_MSG) { + int32_t len = 0; + + char objType[TSDB_TRANS_OBJTYPE_LEN + 1] = {0}; + len += snprintf(objType + len, sizeof(objType) - len, "%s(s:%d,r:%d)", TMSG_INFO(pAction->msgType), + pAction->msgSent, pAction->msgReceived); + char objTypeVStr[TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(objTypeVStr, objType, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)objTypeVStr, false), &lino, _OVER); + + char result[TSDB_TRANS_RESULT_LEN + 1] = {0}; + len = 0; + len += snprintf(result + len, sizeof(result) - len, "errCode:0x%x(%s)", pAction->errCode & 0xFFFF, + tstrerror(pAction->errCode)); + char resultVStr[TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(resultVStr, result, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)resultVStr, false), &lino, _OVER); + + char target[TSDB_TRANS_TARGET_LEN] = {0}; + len = 0; + SEpSet epset = pAction->epSet; + if (epset.numOfEps > 0) { + for (int32_t i = 0; i < epset.numOfEps; ++i) { + len += snprintf(target + len, sizeof(target) - len, "ep:%d-%s:%u,", i, epset.eps[i].fqdn, epset.eps[i].port); + } + len += snprintf(target + len, sizeof(target) - len, "(%d:%d) ", epset.numOfEps, epset.inUse); + } + char targetVStr[TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(targetVStr, target, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)targetVStr, false), &lino, _OVER); + + char detail[TSDB_TRANS_DETAIL_LEN] = {0}; + len = 0; + char bufStart[40] = {0}; + if (pAction->startTime > 0) (void)formatTimestamp(bufStart, pAction->startTime, TSDB_TIME_PRECISION_MILLI); + char bufEnd[40] = {0}; + if (pAction->endTime > 0) (void)formatTimestamp(bufEnd, pAction->endTime, TSDB_TIME_PRECISION_MILLI); + len += snprintf(detail + len, sizeof(detail) - len, "startTime:%s, endTime:%s, ", bufStart, bufEnd); + char detailVStr[TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(detailVStr, detail, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)detailVStr, false), &lino, _OVER); + + } else { + int32_t len = 0; + + char objType[TSDB_TRANS_OBJTYPE_LEN + 1] = {0}; + if (pAction->pRaw->type == SDB_VGROUP) { + SSdbRow *pRow = mndVgroupActionDecode(pAction->pRaw); + SVgObj *pVgroup = sdbGetRowObj(pRow); + len += snprintf(objType + len, sizeof(objType) - len, "%s(%d)", sdbTableName(pAction->pRaw->type), pVgroup->vgId); + taosMemoryFreeClear(pRow); + } else { + strcpy(objType, sdbTableName(pAction->pRaw->type)); + } + char objTypeVStr[TSDB_TRANS_OBJTYPE_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(objTypeVStr, objType, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)objTypeVStr, false), &lino, _OVER); + + char result[TSDB_TRANS_RESULT_LEN + 1] = {0}; + len = 0; + len += snprintf(result + len, sizeof(result) - len, "rawWritten:%d", pAction->rawWritten); + char resultVStr[TSDB_TRANS_RESULT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(resultVStr, result, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)resultVStr, false), &lino, _OVER); + + char target[TSDB_TRANS_TARGET_LEN] = ""; + char targetVStr[TSDB_TRANS_TARGET_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(targetVStr, target, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)targetVStr, false), &lino, _OVER); + + char detail[TSDB_TRANS_DETAIL_LEN] = {0}; + len = 0; + len += snprintf(detail + len, sizeof(detail) - len, "sdbStatus:%s", sdbStatusName(pAction->pRaw->status)); + char detailVStr[TSDB_TRANS_DETAIL_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(detailVStr, detail, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, (const char *)detailVStr, false), &lino, _OVER); + } + +_OVER: + if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code)); +} + +static SArray *mndTransGetAction(STrans *pTrans, ETrnStage stage) { + if (stage == TRN_STAGE_PREPARE) { + return pTrans->prepareActions; + } + if (stage == TRN_STAGE_REDO_ACTION) { + return pTrans->redoActions; + } + if (stage == TRN_STAGE_COMMIT_ACTION) { + return pTrans->commitActions; + } + if (stage == TRN_STAGE_UNDO_ACTION) { + return pTrans->undoActions; + } + return NULL; +} + +typedef struct STransDetailIter { + void *pIter; + STrans *pTrans; + ETrnStage stage; + int32_t num; +} STransDetailIter; + +static void mndTransShowActions(SSdb *pSdb, STransDetailIter *pShowIter, SShowObj *pShow, SSDataBlock *pBlock, + int32_t rows, int32_t *numOfRows, SArray *pActions, int32_t end, int32_t start) { + int32_t actionNum = taosArrayGetSize(pActions); + mInfo("stage:%s, Actions num:%d", mndTransStr(pShowIter->stage), actionNum); + + for (int32_t i = start; i < actionNum; ++i) { + STransAction *pAction = taosArrayGet(pShowIter->pTrans->redoActions, i); + mndShowTransAction(pShow, pBlock, pAction, pShowIter->pTrans->id, pShowIter->pTrans->lastAction, rows, *numOfRows); + (*numOfRows)++; + if (*numOfRows >= rows) break; + } + + if (*numOfRows == end) { + sdbRelease(pSdb, pShowIter->pTrans); + pShowIter->pTrans = NULL; + pShowIter->num = 0; + } else { + pShowIter->pTrans = pShowIter->pTrans; + pShowIter->stage = pShowIter->pTrans->stage; + pShowIter->num += (*numOfRows); + } +} + +static int32_t mndRetrieveTransDetail(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + + int32_t code = 0; + int32_t lino = 0; + + mInfo("start to mndRetrieveTransDetail, rows:%d, pShow->numOfRows:%d, pShow->pIter:%p", rows, pShow->numOfRows, + pShow->pIter); + + if (pShow->pIter == NULL) { + pShow->pIter = taosMemoryMalloc(sizeof(STransDetailIter)); + if (pShow->pIter == NULL) { + mError("failed to malloc for pShow->pIter"); + return 0; + } + memset(pShow->pIter, 0, sizeof(STransDetailIter)); + } + + STransDetailIter *pShowIter = (STransDetailIter *)pShow->pIter; + + while (numOfRows < rows) { + if (pShowIter->pTrans == NULL) { + pShowIter->pIter = sdbFetch(pSdb, SDB_TRANS, pShowIter->pIter, (void **)&(pShowIter->pTrans)); + mDebug("retrieve trans detail from fetch, pShow->pIter:%p, pTrans:%p", pShowIter->pIter, pShowIter->pTrans); + if (pShowIter->pIter == NULL) break; + mInfo("retrieve trans detail from fetch, id:%d, trans stage:%d, IterNum:%d", pShowIter->pTrans->id, + pShowIter->pTrans->stage, pShowIter->num); + + SArray *pActions = mndTransGetAction(pShowIter->pTrans, pShowIter->pTrans->stage); + + mndTransShowActions(pSdb, pShowIter, pShow, pBlock, rows, &numOfRows, pActions, taosArrayGetSize(pActions), 0); + break; + } else { + mInfo("retrieve trans detail from iter, id:%d, iterStage:%d, IterNum:%d", pShowIter->pTrans->id, pShowIter->stage, + pShowIter->num); + SArray *pActions = mndTransGetAction(pShowIter->pTrans, pShowIter->stage); + + mndTransShowActions(pSdb, pShowIter, pShow, pBlock, rows, &numOfRows, pActions, + taosArrayGetSize(pActions) - pShowIter->num, pShowIter->num); + break; + } + } + +_OVER: + pShow->numOfRows += numOfRows; + + if (code != 0) { + mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code)); + } else { + mInfo("retrieve trans detail, numOfRows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows) + } + if (numOfRows == 0) { + taosMemoryFree(pShow->pIter); + pShow->pIter = NULL; + } + return numOfRows; +} + static void mndCancelGetNextTrans(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetchByType(pSdb, pIter, SDB_TRANS); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0bce21290b..e20afb7201 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2766,12 +2766,14 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mndTransSetSerial(pTrans); - if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 3) { + if (pNewDb->cfg.replications == 3) { mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); // add second - TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); + if (pNewVgroup->replica == 1){ + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); + } // learner stage pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; @@ -2790,7 +2792,9 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); // add third - TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); + if (pNewVgroup->replica == 2){ + TAOS_CHECK_RETURN (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); + } pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; pNewVgroup->vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; @@ -2802,7 +2806,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb TAOS_CHECK_RETURN(mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &pNewVgroup->vnodeGid[2])); TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); - } else if (pNewVgroup->replica == 3 && pNewDb->cfg.replications == 1) { + } else if (pNewDb->cfg.replications == 1) { mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId, pVgroup->vnodeGid[1].dnodeId, pVgroup->vnodeGid[2].dnodeId); @@ -2819,9 +2823,9 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2)); TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true)); TAOS_CHECK_RETURN( - mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); - } else if (pNewVgroup->replica == 1 && pNewDb->cfg.replications == 2) { + } else if (pNewDb->cfg.replications == 2) { mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 4fe4333534..68dc981338 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -68,7 +68,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur; pStore->streamStateGetKVByCur = streamStateGetKVByCur; - pStore->streamStateSetFillInfo = streamStateSetFillInfo; pStore->streamStateClearExpiredState = streamStateClearExpiredState; pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; @@ -117,7 +116,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy = streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateCopyBackend = streamStateCopyBackend; } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 30a20cd68d..8fe7d3823a 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -325,7 +325,6 @@ int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8 ctx->suidInfo = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (ctx->suidInfo == NULL) { return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); - ; } taosHashSetFreeFp(ctx->suidInfo, destroySTableInfoForChildTable); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index c0c4c4a5a3..a2b6194375 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -796,7 +796,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* sourceIdx++; } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { - tColDataGetValue(pCol, i, &colVal); + code = tColDataGetValue(pCol, i, &colVal); + TSDB_CHECK_CODE(code, line, END); code = doSetVal(pColData, i, &colVal); TSDB_CHECK_CODE(code, line, END); } @@ -937,7 +938,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData pCol = taosArrayGet(pCols, j); TQ_NULL_GO_TO_END(pCol); SColVal colVal = {0}; - tColDataGetValue(pCol, i, &colVal); + TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal)); PROCESS_VAL } @@ -961,7 +962,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); TQ_NULL_GO_TO_END(pColData); SColVal colVal = {0}; - tColDataGetValue(pCol, i, &colVal); + TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal)); SET_DATA } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 5151ea3958..4dff1d08d9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1715,7 +1715,7 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow); if (colType == 2) { SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type); - tColDataGetValue(pColData, tRow.iRow, &colVal); + TAOS_CHECK_GOTO(tColDataGetValue(pColData, tRow.iRow, &colVal), &lino, _exit); SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal}; if (!taosArrayPush(ctxArray, &updateCtx)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index a9f3893b96..4ca140c03e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -123,7 +123,8 @@ static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int pColData = &pBlock->aColData[slotId]; - tColDataGetValue(pColData, irow, &cv); + code = tColDataGetValue(pColData, irow, &cv); + TSDB_CHECK_CODE(code, lino, _end); pKey->numOfPKs = 1; pKey->pks[0].type = cv.value.type; @@ -1603,7 +1604,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro TSDB_CHECK_CODE(code, lino, _end); } else { // varchar/nchar type for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { - tColDataGetValue(pData, j, &cv); + code = tColDataGetValue(pData, j, &cv); + TSDB_CHECK_CODE(code, lino, _end); code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); TSDB_CHECK_CODE(code, lino, _end); } @@ -5282,7 +5284,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->cid == pSupInfo->colId[i]) { - tColDataGetValue(pData, rowIndex, &cv); + code = tColDataGetValue(pData, rowIndex, &cv); + TSDB_CHECK_CODE(code, lino, _end); code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); TSDB_CHECK_CODE(code, lino, _end); j += 1; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 16f6777765..88c6ac3d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -622,7 +622,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId); if (pColData) { - tColDataGetValue(pColData, pRow->iRow, pColVal); + if (tColDataGetValue(pColData, pRow->iRow, pColVal) != 0){ + tsdbError("failed to tColDataGetValue"); + } } else { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } @@ -645,7 +647,9 @@ void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) { SColData *pColData = &pBlock->aColData[i]; if (pColData->cflag & COL_IS_KEY) { SColVal cv; - tColDataGetValue(pColData, irow, &cv); + if (tColDataGetValue(pColData, irow, &cv) != 0){ + break; + } key->pks[key->numOfPKs] = cv.value; key->numOfPKs++; } else { @@ -719,7 +723,9 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { } if (pIter->iColData <= pIter->pRow->pBlockData->nColData) { - tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv); + if (tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv) != 0){ + return NULL; + } ++pIter->iColData; return &pIter->cv; } else { @@ -1251,7 +1257,8 @@ static int32_t tBlockDataUpsertBlockRow(SBlockData *pBlockData, SBlockData *pBlo cv = COL_VAL_NONE(pColDataTo->cid, pColDataTo->type); if (flag == 0 && (code = tColDataAppendValue(pColDataTo, &cv))) goto _exit; } else { - tColDataGetValue(pColDataFrom, iRow, &cv); + code = tColDataGetValue(pColDataFrom, iRow, &cv); + if (code) goto _exit; if (flag) { code = tColDataUpdateValue(pColDataTo, &cv, flag > 0); diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index df6fb17730..41e6c6c2c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -191,7 +191,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur; pStore->streamStateGetKVByCur = streamStateGetKVByCur; - pStore->streamStateSetFillInfo = streamStateSetFillInfo; pStore->streamStateClearExpiredState = streamStateClearExpiredState; pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist; @@ -243,7 +242,6 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; pStore->streamStateDestroy = streamStateDestroy; - pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; pStore->streamStateReloadInfo = streamStateReloadInfo; pStore->streamStateCopyBackend = streamStateCopyBackend; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f52a0c3aba..723fd14145 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -243,7 +243,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { code = TSDB_CODE_VND_HASH_MISMATCH; goto _exit; } else if (mer1.me.type == TSDB_CHILD_TABLE) { - metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); + metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN); @@ -279,7 +279,8 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { } } else { vError("vnodeGetTableCfg get invalid table type:%d", mer1.me.type); - return TSDB_CODE_APP_ERROR; + code = TSDB_CODE_APP_ERROR; + goto _exit; } cfgRsp.numOfTags = schemaTag.nCols; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bc1462176a..2e16b0968a 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1986,10 +1986,19 @@ void catalogDestroy(void) { } if (gCtgMgmt.cacheTimer) { - if (taosTmrStop(gCtgMgmt.cacheTimer)) { - qTrace("stop catalog cache timer may failed"); + if (!taosTmrStop(gCtgMgmt.cacheTimer)) { +/* + qDebug("catalog cacheTimer %" PRIuPTR " not stopped", (uintptr_t)gCtgMgmt.cacheTimer); + + while (!taosTmrIsStopped(&gCtgMgmt.cacheTimer)) { + taosMsleep(1); + } +*/ } + + qDebug("catalog cacheTimer %" PRIuPTR " is stopped", (uintptr_t)gCtgMgmt.cacheTimer); gCtgMgmt.cacheTimer = NULL; + taosTmrCleanUp(gCtgMgmt.timer); gCtgMgmt.timer = NULL; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index f3a0b04457..a1512f8fd9 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -480,6 +480,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { dbCache = (SCtgDBCache *)pIter; + CTG_LOCK(CTG_READ, &dbCache->dbLock); + dbFName = taosHashGetKey(pIter, &len); int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; @@ -509,6 +511,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { hashMethod, hashPrefix, hashSuffix, vgNum); if (dbCache->vgCache.vgInfo) { + CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); + int32_t i = 0; void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL); while (pVgIter) { @@ -524,6 +528,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter); } + + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } if (dbCache->cfgCache.cfgInfo) { @@ -544,6 +550,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { pCfg->schemaless, pCfg->sstTrigger); } + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); + ++i; pIter = taosHashIterate(dbHash, pIter); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index c2889f096b..06eabc09da 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -149,12 +149,13 @@ void ctgTestInitLogFile() { return; } - const char *defaultLogFileNamePrefix = "taoslog"; + const char *defaultLogFileNamePrefix = "catalogTest"; const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; tmrDebugFlag = 159; + tsNumOfLogLines = 1000000000; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); (void)ctgdEnableDebug("api", true); @@ -1839,7 +1840,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint64_t n = 0; ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); - if (n != 3) { + if (n < 3) { taosMsleep(50); } else { break; diff --git a/source/libs/executor/src/anomalywindowoperator.c b/source/libs/executor/src/anomalywindowoperator.c index dd1a52022e..eb72edb964 100644 --- a/source/libs/executor/src/anomalywindowoperator.c +++ b/source/libs/executor/src/anomalywindowoperator.c @@ -61,9 +61,13 @@ static int32_t anomalyCacheBlock(SAnomalyWindowOperatorInfo* pInfo, SSDataBlock* int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t keyBufSize = 0; + int32_t num = 0; + SExprInfo* pExprInfo = NULL; + const char* id = GET_TASKID(pTaskInfo); - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; SAnomalyWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAnomalyWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SAnomalyWindowPhysiNode* pAnomalyNode = (SAnomalyWindowPhysiNode*)physiNode; @@ -74,13 +78,13 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p } if (!taosAnalGetOptStr(pAnomalyNode->anomalyOpt, "algo", pInfo->algoName, sizeof(pInfo->algoName))) { - qError("failed to get anomaly_window algorithm name from %s", pAnomalyNode->anomalyOpt); + qError("%s failed to get anomaly_window algorithm name from %s", id, pAnomalyNode->anomalyOpt); code = TSDB_CODE_ANA_ALGO_NOT_FOUND; goto _error; } if (taosAnalGetAlgoUrl(pInfo->algoName, ANAL_ALGO_TYPE_ANOMALY_DETECT, pInfo->algoUrl, sizeof(pInfo->algoUrl)) != 0) { - qError("failed to get anomaly_window algorithm url from %s", pInfo->algoName); + qError("%s failed to get anomaly_window algorithm url from %s", id, pInfo->algoName); code = TSDB_CODE_ANA_ALGO_NOT_LOAD; goto _error; } @@ -94,20 +98,18 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p SExprInfo* pScalarExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr); QUERY_CHECK_CODE(code, lino, _error); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); } - size_t keyBufSize = 0; - int32_t num = 0; - SExprInfo* pExprInfo = NULL; code = createExprInfo(pAnomalyNode->window.pFuncs, NULL, &pExprInfo, &num); QUERY_CHECK_CODE(code, lino, _error); initResultSizeInfo(&pOperator->resultInfo, 4096); - code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, id, pTaskInfo->streamInfo.pState, + &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); SSDataBlock* pResBlock = createDataBlockFromDescNode(pAnomalyNode->window.node.pOutputDataBlockDesc); @@ -124,27 +126,19 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p pInfo->anomalyCol = extractColumnFromColumnNode(pColNode); pInfo->anomalyKey.type = pInfo->anomalyCol.type; pInfo->anomalyKey.bytes = pInfo->anomalyCol.bytes; + pInfo->anomalyKey.pData = taosMemoryCalloc(1, pInfo->anomalyCol.bytes); - if (pInfo->anomalyKey.pData == NULL) { - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalyKey.pData, code, lino, _error, terrno) int32_t itemSize = sizeof(int32_t) + pInfo->aggSup.resultRowSize + pInfo->anomalyKey.bytes; pInfo->anomalySup.pResultRow = taosMemoryCalloc(1, itemSize); - if (pInfo->anomalySup.pResultRow == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.pResultRow, code, lino, _error, terrno) + pInfo->anomalySup.blocks = taosArrayInit(16, sizeof(SSDataBlock*)); - if (pInfo->anomalySup.blocks == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.blocks, code, lino, _error, terrno) + pInfo->anomalySup.windows = taosArrayInit(16, sizeof(STimeWindow)); - if (pInfo->anomalySup.windows == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->anomalySup.windows, code, lino, _error, terrno) code = filterInitFromNode((SNode*)pAnomalyNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); QUERY_CHECK_CODE(code, lino, _error); @@ -162,18 +156,21 @@ int32_t createAnomalywindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* p *pOptrInfo = pOperator; - qDebug("anomaly_window operator is created, algo:%s url:%s opt:%s", pInfo->algoName, pInfo->algoUrl, + qDebug("%s anomaly_window operator is created, algo:%s url:%s opt:%s", id, pInfo->algoName, pInfo->algoUrl, pInfo->anomalyOpt); return TSDB_CODE_SUCCESS; _error: + qError("%s failed to create anomaly_window operator, line:%d algo:%s code:%s", id, lino, pAnomalyNode->anomalyOpt, + tstrerror(code)); + if (pInfo != NULL) { anomalyDestroyOperatorInfo(pInfo); } destroyOperatorAndDownstreams(pOperator, &downstream, 1); pTaskInfo->code = code; - qError("failed to create anomaly_window operator, algo:%s code:0x%x", pInfo->algoName, code); + return code; } diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 61809b99e0..adf1b4f0d1 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -46,7 +46,7 @@ static FORCE_INLINE bool mJoinBlkReachThreshold(SMJoinOperatorInfo* pInfo, int64 return blkRows >= pInfo->ctx.mergeCtx.blkThreshold; } - return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit; + return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit || blkRows >= pInfo->ctx.mergeCtx.blkThreshold; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 13ae220116..7862b36f27 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1441,7 +1441,7 @@ static int32_t doSetUserTableMetaInfo(SStoreMetaReader* pMetaReaderFn, SStoreMet SMetaReader mr1 = {0}; pMetaReaderFn->initReader(&mr1, pVnode, META_READER_NOLOCK, pMetaFn); - + int64_t suid = pMReader->me.ctbEntry.suid; code = pMetaReaderFn->getTableEntryByUid(&mr1, suid); if (code != TSDB_CODE_SUCCESS) { @@ -1752,7 +1752,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_NOLOCK, &pAPI->metaFn); - + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; code = pAPI->metaReaderFn.getTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { @@ -2284,7 +2284,7 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { // db_name pColInfoData = taosArrayGet(p->pDataBlock, index++); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); - code = colDataSetVal(pColInfoData, numOfRows, db, false); + code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); // vgroup_id @@ -2543,7 +2543,8 @@ static int32_t doSysTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) if (pInfo->showRewrite) { getDBNameFromCondition(pInfo->pCondition, dbName); if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 && - strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) { + strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0 && + strncasecmp(name, TSDB_INS_TABLE_TRANSACTION_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) { TAOS_UNUSED(tsnprintf(pInfo->req.db, sizeof(pInfo->req.db), "%d.%s", pInfo->accountId, dbName)); } } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) { diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 2814b5dc16..a6954afab6 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -28,7 +28,7 @@ extern "C" { #endif -typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; +typedef enum { INS_MATCH, INS_JUMP, INS_SPLIT, INS_RANGE } InstType; typedef struct MatchValue { #ifdef WINDOWS diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 3b0014f16a..fa7dbb5f1f 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -159,14 +159,14 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r if (false == sparSetGet(set, i, &ip)) continue; Inst *inst = taosArrayGet(builder->dfa->insts, ip); - if (inst->ty == JUMP || inst->ty == SPLIT) { + if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) { continue; - } else if (inst->ty == RANGE) { + } else if (inst->ty == INS_RANGE) { if (taosArrayPush(tinsts, &ip) == NULL) { code = terrno; goto _exception; } - } else if (inst->ty == MATCH) { + } else if (inst->ty == INS_MATCH) { isMatch = true; if (taosArrayPush(tinsts, &ip) == NULL) { code = terrno; @@ -234,11 +234,11 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { } bool succ = sparSetAdd(set, ip, NULL); Inst *inst = taosArrayGet(dfa->insts, ip); - if (inst->ty == MATCH || inst->ty == RANGE) { + if (inst->ty == INS_MATCH || inst->ty == INS_RANGE) { // do nothing - } else if (inst->ty == JUMP) { + } else if (inst->ty == INS_JUMP) { dfaAdd(dfa, set, inst->jv.step); - } else if (inst->ty == SPLIT) { + } else if (inst->ty == INS_SPLIT) { dfaAdd(dfa, set, inst->sv.len1); dfaAdd(dfa, set, inst->sv.len2); } @@ -253,11 +253,11 @@ bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) { if (false == sparSetGet(from, i, &ip)) continue; Inst *inst = taosArrayGet(dfa->insts, ip); - if (inst->ty == JUMP || inst->ty == SPLIT) { + if (inst->ty == INS_JUMP || inst->ty == INS_SPLIT) { continue; - } else if (inst->ty == MATCH) { + } else if (inst->ty == INS_MATCH) { isMatch = true; - } else if (inst->ty == RANGE) { + } else if (inst->ty == INS_RANGE) { if (inst->rv.start <= byte && byte <= inst->rv.end) { dfaAdd(dfa, to, ip + 1); } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e2693ff4a4..c6c091c630 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -44,10 +44,10 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval); static int tfileWriteFooter(TFileWriter* write); // handle file corrupt later -static int tfileReaderLoadHeader(TFileReader* reader); +static int tfileReaderLoadHeader(TFileReader* reader); static int32_t tfileReaderLoadFst(TFileReader* reader); -static int tfileReaderVerify(TFileReader* reader); -static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static int tfileReaderVerify(TFileReader* reader); +static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); static int32_t tfileGetFileList(const char* path, SArray** pResult); static int tfileRmExpireFile(SArray* result); @@ -106,6 +106,11 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { SArray* files = NULL; int32_t code = tfileGetFileList(path, &files); + if (code != 0) { + indexError("failed to get file list since %s", tstrerror(code)); + goto End; + } + for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); @@ -1182,7 +1187,6 @@ _exception: TAOS_UNUSED(taosCloseDir(&pDir)); if (files != NULL) { taosArrayDestroyEx(files, tfileDestroyFileName); - taosArrayDestroy(files); } return code; } diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc index 299b62b6fb..5b22b51172 100644 --- a/source/libs/index/test/utilUT.cc +++ b/source/libs/index/test/utilUT.cc @@ -17,6 +17,7 @@ #include "tglobal.h" #include "tskiplist.h" #include "tutil.h" +#include "indexFstDfa.h" class UtilEnv : public ::testing::Test { protected: @@ -41,6 +42,29 @@ class UtilEnv : public ::testing::Test { SArray *rslt; }; +class UtilComm : public ::testing::Test { + protected: + virtual void SetUp() { + // src = (SArray *)taosArrayInit(2, sizeof(void *)); + // for (int i = 0; i < 3; i++) { + // SArray *m = taosArrayInit(10, sizeof(uint64_t)); + // taosArrayPush(src, &m); + // } + + // rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + // for (int i = 0; i < taosArrayGetSize(src); i++) { + // SArray *m = (SArray *)taosArrayGetP(src, i); + // taosArrayDestroy(m); + // } + // taosArrayDestroy(src); + } + // SArray *src; + // SArray *rslt; + +}; + static void clearSourceArray(SArray *p) { for (int i = 0; i < taosArrayGetSize(p); i++) { SArray *m = (SArray *)taosArrayGetP(p, i); @@ -369,3 +393,35 @@ TEST_F(UtilEnv, testDictComm) { EXPECT_EQ(COMMON_INPUTS[v], i); } } + +TEST_F(UtilComm, testCompress) { + for (int32_t i = 0; i < 6; i++) { + _cache_range_compare cmpFunc = idxGetCompare((RangeType)(i)); + //char[32]a = 0, b = 1; + char a[32] = {0}; + char b[32] = {1}; + for (int32_t j = 0; j < TSDB_DATA_TYPE_MAX; j++) { + cmpFunc(a, b, j); + } + } +} +TEST_F(UtilComm, testfstDfa) { + { + FstDfaBuilder *builder = dfaBuilderCreate(NULL); + ASSERT_TRUE(builder != NULL); + dfaBuilderDestroy(builder); + } + { + SArray *pInst = taosArrayInit(32, sizeof(uint8_t)); + for (int32_t i = 0; i < 26; i++) { + uint8_t v = 'a' + i; + taosArrayPush(pInst, &v); + } + FstDfaBuilder *builder = dfaBuilderCreate(pInst); + FstDfa *dfa = dfaBuilderBuild(builder); + dfaBuilderDestroy(builder); + } +} + + + diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6d4d89607f..c2290b9eac 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -292,6 +292,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowCompactsStmt"; case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: return "ShowCompactDetailsStmt"; + case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: + return "ShowTransactionDetailsStmt"; case QUERY_NODE_SHOW_GRANTS_FULL_STMT: return "ShowGrantsFullStmt"; case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7beaeaa46c..5bfc35aa9c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -714,6 +714,9 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { break; case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: code = makeNode(type, sizeof(SShowCompactDetailsStmt), &pNode); + break; + case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: + code = makeNode(type, sizeof(SShowTransactionDetailsStmt), &pNode); break; case QUERY_NODE_KILL_QUERY_STMT: code = makeNode(type, sizeof(SKillQueryStmt), &pNode); @@ -1562,6 +1565,11 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pCompactId); break; } + case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: { + SShowTransactionDetailsStmt* pStmt = (SShowTransactionDetailsStmt*)pNode; + nodesDestroyNode(pStmt->pTransactionId); + break; + } case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg); break; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e69a3da4a9..8b2a98baf2 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -320,6 +320,7 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView); SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactIdNode); SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type); +SNode* createShowTransactionDetailsStmt(SAstCreateContext* pCxt, SNode* pTransactionIdNode); SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* tsmaName, SNode* pOptions, SNode* pRealTable, SNode* pInterval); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 63eb09d509..38e93a7ce3 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -594,6 +594,7 @@ cmd ::= SHOW BNODES. cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } cmd ::= SHOW TRANSACTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } +cmd ::= SHOW TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createShowTransactionDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); } cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, A); } cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index fa656667af..5fc5f5fafe 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -2931,6 +2931,18 @@ _err: return NULL; } +SNode* createShowTransactionDetailsStmt(SAstCreateContext* pCxt, SNode* pTransactionIdNode) { + CHECK_PARSER_STATUS(pCxt); + SShowTransactionDetailsStmt* pStmt = NULL; + pCxt->errCode = nodesMakeNode(QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, (SNode**)&pStmt); + CHECK_MAKE_NODE(pStmt); + pStmt->pTransactionId = pTransactionIdNode; + return (SNode*)pStmt; +_err: + nodesDestroyNode(pTransactionIdNode); + return NULL; +} + static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange) { int32_t code = TSDB_CODE_SUCCESS; char* ipCopy = taosStrdup(ipRange); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 1687916cb0..e876575f48 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -770,6 +770,12 @@ static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SS return code; } +static int32_t collectMetaKeyFromShowTransactionDetails(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, + TSDB_INS_TABLE_TRANSACTION_DETAILS, pCxt->pMetaCache); + return code; +} + static int32_t collectMetaKeyFromShowGrantsFull(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_GRANTS_FULL, pCxt->pMetaCache); @@ -1094,6 +1100,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowCompacts(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: + return collectMetaKeyFromShowTransactionDetails(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_GRANTS_FULL_STMT: return collectMetaKeyFromShowGrantsFull(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_GRANTS_LOGS_STMT: diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index e2069deefc..128fb50b8f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1841,6 +1841,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } if (TK_NK_QUESTION == pToken->type) { + if (!pCxt->pComCxt->isStmtBind && i != 0) { + return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values"); + } pCxt->isStmtBind = true; pStmt->usingTableProcessing = true; if (pCols->pColIndex[i] == tbnameIdx) { @@ -1874,6 +1877,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return buildInvalidOperationMsg(&pCxt->msg, "not expected numOfBound"); } } else { + if (pCxt->pComCxt->isStmtBind) { + return buildInvalidOperationMsg(&pCxt->msg, "not support mixed bind and non-bind values"); + } if (pCols->pColIndex[i] < numOfCols) { const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 342bd6d66e..8f3cbd6f8b 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -371,12 +371,40 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { .pShowCols = {"*"} }, { + .showType = QUERY_NODE_CREATE_TSMA_STMT, + .pDbName = "", + .pTableName = "", + .numOfShowCols = 1, + .pShowCols = {"*"} + }, + { + .showType = QUERY_NODE_SHOW_CREATE_TSMA_STMT, + .pDbName = "", + .pTableName = "", + .numOfShowCols = 1, + .pShowCols = {"*"} + }, + { + .showType = QUERY_NODE_DROP_TSMA_STMT, + .pDbName = "", + .pTableName = "", + .numOfShowCols = 1, + .pShowCols = {"*"} + }, + { .showType = QUERY_NODE_SHOW_FILESETS_STMT, .pDbName = TSDB_INFORMATION_SCHEMA_DB, .pTableName = TSDB_INS_TABLE_FILESETS, .numOfShowCols = 1, .pShowCols = {"*"} - }, + }, + { + .showType = QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, + .pDbName = TSDB_INFORMATION_SCHEMA_DB, + .pTableName = TSDB_INS_TABLE_TRANSACTION_DETAILS, + .numOfShowCols = 1, + .pShowCols = {"*"} + }, }; // clang-format on @@ -10086,7 +10114,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt const char* validConfigs[] = { "encrypt_key", - tsAlterCompactTaskKeywords, }; if (0 == strncasecmp(cfgReq.config, validConfigs[0], strlen(validConfigs[0]) + 1)) { int32_t klen = strlen(cfgReq.value); @@ -10097,28 +10124,6 @@ static int32_t translateAlterDnode(STranslateContext* pCxt, SAlterDnodeStmt* pSt ENCRYPT_KEY_LEN_MIN, ENCRYPT_KEY_LEN); } code = buildCmdMsg(pCxt, TDMT_MND_CREATE_ENCRYPT_KEY, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); - } else if (0 == strncasecmp(cfgReq.config, validConfigs[1], strlen(validConfigs[1]) + 1)) { - char* endptr = NULL; - int32_t maxCompactTasks = taosStr2Int32(cfgReq.value, &endptr, 10); - int32_t minMaxCompactTasks = MIN_MAX_COMPACT_TASKS; - int32_t maxMaxCompactTasks = MAX_MAX_COMPACT_TASKS; - - // check format - if (endptr == cfgReq.value || endptr[0] != '\0') { - tFreeSMCfgDnodeReq(&cfgReq); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, - "Invalid max compact tasks: %s", cfgReq.value); - } - - // check range - if (maxCompactTasks < minMaxCompactTasks || maxCompactTasks > maxMaxCompactTasks) { - tFreeSMCfgDnodeReq(&cfgReq); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_DNODE_INVALID_COMPACT_TASKS, - "Invalid max compact tasks: %d, valid range [%d,%d]", maxCompactTasks, - minMaxCompactTasks, maxMaxCompactTasks); - } - - code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } else { code = buildCmdMsg(pCxt, TDMT_MND_CONFIG_DNODE, (FSerializeFunc)tSerializeSMCfgDnodeReq, &cfgReq); } @@ -16382,6 +16387,24 @@ static int32_t rewriteShowCompactDetailsStmt(STranslateContext* pCxt, SQuery* pQ return code; } +static int32_t rewriteShowTransactionDetailsStmt(STranslateContext* pCxt, SQuery* pQuery) { + SShowTransactionDetailsStmt* pShow = (SShowTransactionDetailsStmt*)(pQuery->pRoot); + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT, &pStmt); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pShow->pTransactionId) { + code = createOperatorNode(OP_TYPE_EQUAL, "transaction_id", pShow->pTransactionId, &pStmt->pWhere); + } + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; + pQuery->showRewrite = true; + nodesDestroyNode(pQuery->pRoot); + pQuery->pRoot = (SNode*)pStmt; + } + return code; +} + static int32_t createParWhenThenNode(SNode* pWhen, SNode* pThen, SNode** ppResWhenThen) { SWhenThenNode* pWThen = NULL; int32_t code = nodesMakeNode(QUERY_NODE_WHEN_THEN, (SNode**)&pWThen); @@ -16957,6 +16980,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT: code = rewriteShowCompactDetailsStmt(pCxt, pQuery); break; + case QUERY_NODE_SHOW_TRANSACTION_DETAILS_STMT: + code = rewriteShowTransactionDetailsStmt(pCxt, pQuery); + break; case QUERY_NODE_SHOW_DB_ALIVE_STMT: case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: code = rewriteShowAliveStmt(pCxt, pQuery); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b329bbbd44..6689e12edd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -5128,8 +5128,8 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SOperatorNode *pOper = (SOperatorNode *)pNode; - SValueNode *valNode = (SValueNode *)pOper->pRight; - if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SExprNode* pLeft = (SExprNode*)pOper->pLeft; + if (IS_NUMERIC_TYPE(pLeft->resType.type) || pLeft->resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SNode* pLeft = NULL, *pRight = NULL; int32_t code = nodesCloneNode(pOper->pLeft, &pLeft); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 7af64c041d..f922a5e03e 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -254,6 +254,9 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList); int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); int32_t uploadCheckpointToS3(const char* id, const char* path); +int32_t deleteCheckpointFile(const char* id, const char* name); +int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index ebde9fe50e..d3eba382c9 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -19,7 +19,6 @@ #include "tcs.h" static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName); -static int32_t deleteCheckpointFile(const char* id, const char* name); static int32_t streamTaskUploadCheckpoint(const char* id, const char* path, int64_t checkpointId); #ifdef BUILD_NO_CALL static int32_t deleteCheckpoint(const char* id); @@ -230,8 +229,8 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream return code; } -static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, - int32_t transId) { +int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t checkpointId, SStreamDataBlock* pBlock, + int32_t transId) { int32_t code = 0; int32_t vgId = pTask->pMeta->vgId; int32_t taskLevel = pTask->info.taskLevel; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index fa16cace25..42d7f44b62 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -954,7 +954,6 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, void* param, int32_t n int32_t vgId = pTask->pMeta->vgId; if (pTmrInfo->launchChkptId != pActiveInfo->activeId) { - streamCleanBeforeQuitTmr(pTmrInfo, param); stWarn("s-task:%s vgId:%d ready-msg send tmr launched by previous checkpoint procedure, checkpointId:%" PRId64 ", quit", id, vgId, pTmrInfo->launchChkptId); @@ -963,13 +962,11 @@ static int32_t doTaskChkptStatusCheck(SStreamTask* pTask, void* param, int32_t n // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) || (pActiveInfo->transId == 0) || (num == 0) || (pTask->chkInfo.startTs == 0)) { - streamCleanBeforeQuitTmr(pTmrInfo, param); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr", id, vgId); return -1; } if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { - streamCleanBeforeQuitTmr(pTmrInfo, param); stWarn("s-task:%s vgId:%d upstream number:%d not equals sent readyMsg:%d, quit from readyMsg send tmr", id, vgId, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList), num); return -1; @@ -998,6 +995,7 @@ static int32_t doFindNotConfirmUpstream(SArray** ppNotRspList, SArray* pList, in void* p = taosArrayPush(pTmp, &pInfo->upstreamTaskId); if (p == NULL) { stError("s-task:%s vgId:%d failed to record not rsp task, code: out of memory", id, vgId); + taosArrayDestroy(pTmp); return terrno; } else { stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, level, @@ -1047,13 +1045,13 @@ static void doSendChkptReadyMsg(SStreamTask* pTask, SArray* pNotRspList, int64_t } } -static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray* pNotRspList) { +static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray** pNotRspList) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptReadyMsgTmr; SArray* pList = pActiveInfo->pReadyMsgList; int32_t num = taosArrayGetSize(pList); int32_t vgId = pTask->pMeta->vgId; - int32_t checkpointId = pActiveInfo->activeId; + int64_t checkpointId = pActiveInfo->activeId; const char* id = pTask->id.idStr; int32_t notRsp = 0; @@ -1062,18 +1060,17 @@ static int32_t chkptReadyMsgSendHelper(SStreamTask* pTask, void* param, SArray* return code; } - code = doFindNotConfirmUpstream(&pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); + code = doFindNotConfirmUpstream(pNotRspList, pList, num, vgId, pTask->info.taskLevel, id); if (code) { - streamCleanBeforeQuitTmr(pTmrInfo, param); stError("s-task:%s failed to find not rsp checkpoint-ready downstream, code:%s, out of tmr", id, tstrerror(code)); return code; } - notRsp = taosArrayGetSize(pNotRspList); + notRsp = taosArrayGetSize(*pNotRspList); if (notRsp == 0) { streamClearChkptReadyMsg(pActiveInfo); } else { - doSendChkptReadyMsg(pTask, pNotRspList, checkpointId, pList); + doSendChkptReadyMsg(pTask, *pNotRspList, checkpointId, pList); } return code; @@ -1137,10 +1134,12 @@ static void chkptReadyMsgSendMonitorFn(void* param, void* tmrId) { } streamMutexLock(&pActiveInfo->lock); - code = chkptReadyMsgSendHelper(pTask, param, pNotRspList); + code = chkptReadyMsgSendHelper(pTask, param, &pNotRspList); streamMutexUnlock(&pActiveInfo->lock); if (code != TSDB_CODE_SUCCESS) { + streamCleanBeforeQuitTmr(pTmrInfo, param); + streamMetaReleaseTask(pTask->pMeta, pTask); taosArrayDestroy(pNotRspList); return; @@ -1176,7 +1175,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { int32_t num = taosArrayGetSize(pList); if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { - stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num, + stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d not send chkpt-ready msg", id, num, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList)); streamMutexUnlock(&pActiveInfo->lock); return TSDB_CODE_STREAM_INTERNAL_ERROR; @@ -1200,7 +1199,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); } } else { - stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); + stError("s-task:%s failed to prepare the checkpoint-ready msg, try next time in 10s", id); } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 85f287f301..89cb4153fe 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -915,8 +915,7 @@ int32_t streamResumeTask(SStreamTask* pTask) { while (1) { code = doStreamExecTask(pTask); if (code) { - stError("s-task:%s failed to exec stream task, code:%s", id, tstrerror(code)); - return code; + stError("s-task:%s failed to exec stream task, code:%s, continue", id, tstrerror(code)); } // check if continue streamMutexLock(&pTask->lock); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 89f0ea9e1f..7259c0e49a 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -546,10 +546,6 @@ void streamStateDestroy(SStreamState* pState, bool remove) { taosMemoryFreeClear(pState); } -int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { - return deleteExpiredCheckPoint(pState->pFileState, mark); -} - void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } void streamStateCopyBackend(SStreamState* src, SStreamState* dst) { @@ -617,8 +613,6 @@ int32_t streamStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** void streamStateClearExpiredState(SStreamState* pState) { clearExpiredState(pState->pFileState); } -void streamStateSetFillInfo(SStreamState* pState) { setFillInfo(pState->pFileState); } - int32_t streamStateGetPrev(SStreamState* pState, const SWinKey* pKey, SWinKey* pResKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { return getRowStatePrevRow(pState->pFileState, pKey, pResKey, pVal, pVLen, pWinCode); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 05edad0f5f..aaff58d1b4 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -667,18 +667,6 @@ void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLe } } -int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { - int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); - int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); - if (pFileState->searchBuff != NULL) { - deleteHashSortRowBuff(pFileState, pKey); - } - if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) { - return TSDB_CODE_SUCCESS; - } - return TSDB_CODE_FAILED; -} - static int32_t recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -868,10 +856,6 @@ int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf); } -int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) { - return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list); -} - int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t code = TSDB_CODE_SUCCESS; int64_t maxCheckPointId = 0; @@ -1227,10 +1211,6 @@ SSHashObj* getGroupIdCache(SStreamFileState* pFileState) { return pFileState->pGroupIdMap; } -void setFillInfo(SStreamFileState* pFileState) { - pFileState->hasFillCatch = false; -} - void clearExpiredState(SStreamFileState* pFileState) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; @@ -1261,6 +1241,7 @@ _end: } } +#ifdef BUILD_NO_CALL int32_t getStateSearchRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; @@ -1328,6 +1309,7 @@ _end: } return code; } +#endif int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal, int32_t* pVLen, int32_t* pWinCode) { diff --git a/source/libs/stream/test/streamCheckPointTest.cpp b/source/libs/stream/test/streamCheckPointTest.cpp index c8297d56b7..c993743dc3 100644 --- a/source/libs/stream/test/streamCheckPointTest.cpp +++ b/source/libs/stream/test/streamCheckPointTest.cpp @@ -390,9 +390,78 @@ TEST(sstreamTaskGetTriggerRecvStatusTest, streamTaskGetTriggerRecvStatusFnTest) extern int8_t tsS3EpNum; tsS3EpNum = 1; - code = uploadCheckpointToS3("123", "/tmp/backend5/stream"); - EXPECT_EQ(code, TSDB_CODE_SUCCESS); + code = uploadCheckpointToS3("123", "/tmp/backend5/stream/stream"); + EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); code = downloadCheckpointByNameS3("123", "/root/download", ""); EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); + + code = deleteCheckpointFile("aaa123", "bbb"); + EXPECT_NE(code, TSDB_CODE_OUT_OF_RANGE); } + +TEST(doCheckBeforeHandleChkptTriggerTest, doCheckBeforeHandleChkptTriggerFnTest) { + SStreamTask* pTask = NULL; + int64_t uid = 2222222222222; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + const char *path = "/tmp/doCheckBeforeHandleChkptTriggerTest/stream"; + code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SStreamState *pState = streamStateOpen((char *)path, pTask, 0, 0); + ASSERT(pState != NULL); + + pTask->pBackend = pState->pTdbState->pOwner->pBackend; + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.checkpointId = 123; + code = doCheckBeforeHandleChkptTrigger(pTask, 100, NULL, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + pTask->chkInfo.pActiveInfo->failedId = 223; + code = doCheckBeforeHandleChkptTrigger(pTask, 200, NULL, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + SStreamDataBlock block; + block.srcTaskId = 456; + SStreamTask upTask; + upTask = *pTask; + upTask.id.taskId = 456; + streamTaskSetUpstreamInfo(pTask, &upTask); + pTask->chkInfo.pActiveInfo->failedId = 23; + code = doCheckBeforeHandleChkptTrigger(pTask, 123, &block, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); + + streamTaskSetUpstreamInfo(pTask, &upTask); + streamTaskSetStatusReady(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 223; + + STaskCheckpointReadyInfo readyInfo; + readyInfo.upstreamTaskId = 4567; + block.srcTaskId = 4567; + void* pBuf = rpcMallocCont(sizeof(SMsgHead) + 1); + + initRpcMsg(&readyInfo.msg, 0, pBuf, sizeof(SMsgHead) + 1); + taosArrayPush(pTask->chkInfo.pActiveInfo->pReadyMsgList, &readyInfo); + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_NE(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->allUpstreamTriggerRecv = 1; + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_NE(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 1111; + code = doCheckBeforeHandleChkptTrigger(pTask, 223, &block, 0); + ASSERT_EQ(code, TSDB_CODE_STREAM_INVLD_CHKPT); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbde104f4e..4862a4b963 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -422,13 +422,11 @@ int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) { SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) { SyncIndex minMatchIndex = SYNC_INDEX_INVALID; - if (pSyncNode->peersNum > 0) { - minMatchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[0])); - } - - for (int32_t i = 1; i < pSyncNode->peersNum; ++i) { + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { SyncIndex matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i])); - if (matchIndex < minMatchIndex) { + if (minMatchIndex == SYNC_INDEX_INVALID) { + minMatchIndex = matchIndex; + } else if (matchIndex > 0 && matchIndex < minMatchIndex) { minMatchIndex = matchIndex; } } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 468d9d9b50..3b84fc4574 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -151,7 +151,6 @@ typedef struct SCliThrd { TdThreadMutex msgMtx; SDelayQueue* delayQueue; SDelayQueue* timeoutQueue; - SDelayQueue* waitConnQueue; uint64_t nextTimeout; // next timeout STrans* pInst; // @@ -159,8 +158,6 @@ typedef struct SCliThrd { SHashObj* fqdn2ipCache; SCvtAddr* pCvtAddr; - SHashObj* failFastCache; - SHashObj* batchCache; SHashObj* connHeapCache; SCliReq* stopMsg; @@ -224,8 +221,6 @@ static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); -SCliBatch* cliGetHeadFromList(SCliBatchList* pList); - static void destroyCliConnQTable(SCliConn* conn); static void cliHandleException(SCliConn* conn); @@ -1299,8 +1294,8 @@ static void cliHandleException(SCliConn* conn) { if (conn->registered) { int8_t ref = transGetRefCount(conn); if (ref == 0 && !uv_is_closing((uv_handle_t*)conn->stream)) { -// tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd, -// conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds); + // tTrace("%s conn %p fd %d,%d,%d,%p uv_closed", CONN_GET_INST_LABEL(conn), conn, conn->stream->u.fd, + // conn->stream->io_watcher.fd, conn->stream->accepted_fd, conn->stream->queued_fds); uv_close((uv_handle_t*)conn->stream, cliDestroy); } } @@ -2124,144 +2119,7 @@ static void cliDoReq(queue* wq, SCliThrd* pThrd) { tTrace("cli process batch size:%d", count); } } -SCliBatch* cliGetHeadFromList(SCliBatchList* pList) { - if (QUEUE_IS_EMPTY(&pList->wq) || pList->connCnt > pList->connMax || pList->sending > pList->connMax) { - return NULL; - } - queue* hr = QUEUE_HEAD(&pList->wq); - QUEUE_REMOVE(hr); - pList->sending += 1; - pList->len -= 1; - - SCliBatch* batch = QUEUE_DATA(hr, SCliBatch, listq); - return batch; -} -static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq); - -static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port); - -static void destroyBatchList(SCliBatchList* pList); -static void cliBuildBatch(SCliReq* pReq, queue* h, SCliThrd* pThrd) { - int32_t code = 0; - STrans* pInst = pThrd->pInst; - SReqCtx* pCtx = pReq->ctx; - - char* ip = EPSET_GET_INUSE_IP(pCtx->epSet); - uint32_t port = EPSET_GET_INUSE_PORT(pCtx->epSet); - char key[TSDB_FQDN_LEN + 64] = {0}; - CONN_CONSTRUCT_HASH_KEY(key, ip, port); - size_t klen = strlen(key); - SCliBatchList** ppBatchList = taosHashGet(pThrd->batchCache, key, klen); - if (ppBatchList == NULL || *ppBatchList == NULL) { - SCliBatchList* pBatchList = NULL; - code = createBatchList(&pBatchList, key, ip, port); - if (code != 0) { - destroyReq(pReq); - return; - } - - pBatchList->batchLenLimit = pInst->shareConnLimit; - - SCliBatch* pBatch = NULL; - code = createBatch(&pBatch, pBatchList, pReq); - if (code != 0) { - destroyBatchList(pBatchList); - destroyReq(pReq); - return; - } - - code = taosHashPut(pThrd->batchCache, key, klen, &pBatchList, sizeof(void*)); - if (code != 0) { - destroyBatchList(pBatchList); - } - } else { - if (QUEUE_IS_EMPTY(&(*ppBatchList)->wq)) { - SCliBatch* pBatch = NULL; - code = createBatch(&pBatch, *ppBatchList, pReq); - if (code != 0) { - destroyReq(pReq); - cliDestroyBatch(pBatch); - } - } else { - queue* hdr = QUEUE_TAIL(&((*ppBatchList)->wq)); - SCliBatch* pBatch = QUEUE_DATA(hdr, SCliBatch, listq); - if ((pBatch->shareConnLimit + pReq->msg.contLen) < (*ppBatchList)->batchLenLimit) { - QUEUE_PUSH(&pBatch->wq, h); - pBatch->shareConnLimit += pReq->msg.contLen; - pBatch->wLen += 1; - } else { - SCliBatch* tBatch = NULL; - code = createBatch(&tBatch, *ppBatchList, pReq); - if (code != 0) { - destroyReq(pReq); - } - } - } - } - return; -} -static int32_t createBatchList(SCliBatchList** ppBatchList, char* key, char* ip, uint32_t port) { - SCliBatchList* pBatchList = taosMemoryCalloc(1, sizeof(SCliBatchList)); - if (pBatchList == NULL) { - tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - QUEUE_INIT(&pBatchList->wq); - pBatchList->port = port; - pBatchList->connMax = 1; - pBatchList->connCnt = 0; - pBatchList->batchLenLimit = 0; - pBatchList->len += 1; - - pBatchList->ip = taosStrdup(ip); - pBatchList->dst = taosStrdup(key); - if (pBatchList->ip == NULL || pBatchList->dst == NULL) { - taosMemoryFree(pBatchList->ip); - taosMemoryFree(pBatchList->dst); - taosMemoryFree(pBatchList); - tError("failed to create batch list since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - *ppBatchList = pBatchList; - return 0; -} -static void destroyBatchList(SCliBatchList* pList) { - if (pList == NULL) { - return; - } - while (!QUEUE_IS_EMPTY(&pList->wq)) { - queue* h = QUEUE_HEAD(&pList->wq); - QUEUE_REMOVE(h); - - SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq); - cliDestroyBatch(pBatch); - } - taosMemoryFree(pList->ip); - taosMemoryFree(pList->dst); - taosMemoryFree(pList); -} -static int32_t createBatch(SCliBatch** ppBatch, SCliBatchList* pList, SCliReq* pReq) { - SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch)); - if (pBatch == NULL) { - tError("failed to create batch since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return terrno; - } - - QUEUE_INIT(&pBatch->wq); - QUEUE_INIT(&pBatch->listq); - - QUEUE_PUSH(&pBatch->wq, &pReq->q); - pBatch->wLen += 1; - pBatch->shareConnLimit = pReq->msg.contLen; - pBatch->pList = pList; - - QUEUE_PUSH(&pList->wq, &pBatch->listq); - pList->len += 1; - - *ppBatch = pBatch; - return 0; -} static void cliDoBatchReq(queue* wq, SCliThrd* pThrd) { return cliDoReq(wq, pThrd); } static void cliAsyncCb(uv_async_t* handle) { @@ -2494,10 +2352,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(code, NULL, _end); } - if ((code = transDQCreate(pThrd->loop, &pThrd->waitConnQueue)) != 0) { - TAOS_CHECK_GOTO(code, NULL, _end); - } - pThrd->destroyAhandleFp = pInst->destroyFp; pThrd->fqdn2ipCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -2505,11 +2359,6 @@ static int32_t createThrdObj(void* trans, SCliThrd** ppThrd) { TAOS_CHECK_GOTO(terrno, NULL, _end); } - pThrd->batchCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pThrd->batchCache == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _end); - } - pThrd->connHeapCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (pThrd->connHeapCache == NULL) { TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); @@ -2553,10 +2402,7 @@ _end: transDQDestroy(pThrd->delayQueue, NULL); transDQDestroy(pThrd->timeoutQueue, NULL); - transDQDestroy(pThrd->waitConnQueue, NULL); taosHashCleanup(pThrd->fqdn2ipCache); - taosHashCleanup(pThrd->failFastCache); - taosHashCleanup(pThrd->batchCache); taosHashCleanup(pThrd->pIdConnTable); taosArrayDestroy(pThrd->pQIdBuf); @@ -2580,7 +2426,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { transDQDestroy(pThrd->delayQueue, destroyReqAndAhanlde); transDQDestroy(pThrd->timeoutQueue, NULL); - transDQDestroy(pThrd->waitConnQueue, NULL); tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { @@ -2592,24 +2437,6 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); - void** pIter = taosHashIterate(pThrd->batchCache, NULL); - while (pIter != NULL) { - SCliBatchList* pBatchList = (SCliBatchList*)(*pIter); - while (!QUEUE_IS_EMPTY(&pBatchList->wq)) { - queue* h = QUEUE_HEAD(&pBatchList->wq); - QUEUE_REMOVE(h); - - SCliBatch* pBatch = QUEUE_DATA(h, SCliBatch, listq); - cliDestroyBatch(pBatch); - } - taosMemoryFree(pBatchList->ip); - taosMemoryFree(pBatchList->dst); - taosMemoryFree(pBatchList); - - pIter = (void**)taosHashIterate(pThrd->batchCache, pIter); - } - taosHashCleanup(pThrd->batchCache); - void* pIter2 = taosHashIterate(pThrd->connHeapCache, NULL); while (pIter2 != NULL) { SHeap* heap = (SHeap*)(pIter2); diff --git a/source/libs/transport/test/transUT2.cpp b/source/libs/transport/test/transUT2.cpp index 6dfb5e503a..fabe9e9c4f 100644 --- a/source/libs/transport/test/transUT2.cpp +++ b/source/libs/transport/test/transUT2.cpp @@ -615,6 +615,21 @@ TEST_F(TransEnv, http) { #endif } +#if 1 + STelemAddrMgmt mgt; + taosTelemetryMgtInit(&mgt, "telemetry.taosdata.com"); + int32_t code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, "test", strlen("test"),HTTP_FLAT); + printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr); + + taosMsleep(2000); + code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT); + for (int32_t i = 0; i < 1; i++) { + code = taosSendTelemReport(&mgt,tsTelemUri, tsTelemPort, pCont, len,HTTP_FLAT); + printf("old addr:%s new addr:%s\n",mgt.defaultAddr, mgt.cachedAddr); + taosMsleep(2000); + } + taosTelemetryDestroy(&mgt); +#endif { STelemAddrMgmt mgt; taosTelemetryMgtInit(&mgt, "error"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 43a2ff6a23..4d47de98b4 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -208,28 +208,22 @@ static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; TAOS_UNUSED(taosThreadRwlockRdlock(&pWal->mutex)); - SWalFileInfo *gloablPRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - if (gloablPRet == NULL) { + SWalFileInfo *globalRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + if (globalRet == NULL) { wError("failed to find WAL log file with ver:%" PRId64, ver); TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); TAOS_RETURN(TSDB_CODE_WAL_INVALID_VER); } - SWalFileInfo *pRet = taosMemoryMalloc(sizeof(SWalFileInfo)); - if (pRet == NULL) { - wError("failed to allocate memory for localRet"); - TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); - TAOS_RETURN(terrno); - } - TAOS_MEMCPY(pRet, gloablPRet, sizeof(SWalFileInfo)); + SWalFileInfo ret; + TAOS_MEMCPY(&ret, globalRet, sizeof(SWalFileInfo)); TAOS_UNUSED(taosThreadRwlockUnlock(&pWal->mutex)); - if (pReader->curFileFirstVer != pRet->firstVer) { + if (pReader->curFileFirstVer != ret.firstVer) { // error code was set inner - TAOS_CHECK_RETURN_WITH_FREE(walReadChangeFile(pReader, pRet->firstVer), pRet); + TAOS_CHECK_RETURN(walReadChangeFile(pReader, ret.firstVer)); } // error code was set inner - TAOS_CHECK_RETURN_WITH_FREE(walReadSeekFilePos(pReader, pRet->firstVer, ver), pRet); - taosMemoryFree(pRet); + TAOS_CHECK_RETURN(walReadSeekFilePos(pReader, ret.firstVer, ver)); wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver); pReader->curVersion = ver; @@ -437,15 +431,15 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { seeked = true; continue; } else { - wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", - pReader->pWal->cfg.vgId, ver, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); - if (contLen < 0) { - TAOS_RETURN(terrno); + code = terrno; } else { - TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); + code = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, tstrerror(code)); + TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); + TAOS_RETURN(code); } } @@ -478,15 +472,15 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != cryptedBodyLen) { - wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", - pReader->pWal->cfg.vgId, ver, terrstr()); - TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); - if (contLen < 0) { - TAOS_RETURN(terrno); + code = terrno; } else { - TAOS_RETURN(TSDB_CODE_WAL_FILE_CORRUPTED); + code = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, tstrerror(code)); + TAOS_UNUSED(taosThreadMutexUnlock(&pReader->mutex)); + TAOS_RETURN(code); } if (pReader->pHead->head.version != ver) { diff --git a/source/util/src/tanalytics.c b/source/util/src/tanalytics.c index e68edd4b76..bf2cb4fd07 100644 --- a/source/util/src/tanalytics.c +++ b/source/util/src/tanalytics.c @@ -20,7 +20,7 @@ #ifdef USE_ANALYTICS #include -#define ANAL_ALGO_SPLIT "," +#define ANALYTICS_ALOG_SPLIT_CHAR "," typedef struct { int64_t ver; @@ -136,7 +136,7 @@ bool taosAnalGetOptStr(const char *option, const char *optName, char *optValue, return false; } - pEnd = strstr(pStart, ANAL_ALGO_SPLIT); + pEnd = strstr(pStart, ANALYTICS_ALOG_SPLIT_CHAR); if (optMaxLen > 0) { if (pEnd > pStart) { int32_t len = (int32_t)(pEnd - pStart); @@ -168,7 +168,7 @@ bool taosAnalGetOptInt(const char *option, const char *optName, int64_t *optValu int32_t bufLen = tsnprintf(buf, sizeof(buf), "%s=", optName); char *pos1 = strstr(option, buf); - char *pos2 = strstr(option, ANAL_ALGO_SPLIT); + char *pos2 = strstr(option, ANALYTICS_ALOG_SPLIT_CHAR); if (pos1 != NULL) { *optValue = taosStr2Int64(pos1 + bufLen, NULL, 10); return true; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d2d551a539..feed297f71 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -326,6 +326,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While ex TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Wrong transaction execution context") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT_COMPACT, "Transaction not completed due to conflict with compact") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_ABLE_TO_kILLED, "The transaction is not able to be killed") // mnode-mq TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") @@ -908,12 +909,6 @@ const char* tstrerror(int32_t err) { (void)taosThreadOnce(&tsErrorInit, tsSortError); // this is a system errno - if ((err & 0x00ff0000) == 0x00ff0000) { - int32_t code = err & 0x0000ffff; - // strerror can handle any invalid code - // invalid code return Unknown error - return strerror(code); - } #ifdef WINDOWS if ((err & 0x01ff0000) == 0x01ff0000) { snprintf(WinAPIErrDesc, 256, "windows api error, code: 0x%08x", err & 0x0000ffff); @@ -923,6 +918,13 @@ const char* tstrerror(int32_t err) { return WinAPIErrDesc; } #endif + if ((err & 0x00ff0000) == 0x00ff0000) { + int32_t code = err & 0x0000ffff; + // strerror can handle any invalid code + // invalid code return Unknown error + return strerror(code); + } + int32_t s = 0; int32_t e = sizeof(errors) / sizeof(errors[0]); diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py index 9725c36bee..b5c6140481 100644 --- a/tests/develop-test/2-query/table_count_scan.py +++ b/tests/develop-test/2-query/table_count_scan.py @@ -68,7 +68,7 @@ class TDTestCase: for i in range(0, 3): db_name = tdSql.getData(i, 1) if db_name == 'information_schema': - tdSql.checkData(i, 0, 36) + tdSql.checkData(i, 0, 37) tdSql.checkData(i, 2, None) elif db_name == 'performance_schema': tdSql.checkData(i, 0, 5) @@ -81,7 +81,7 @@ class TDTestCase: tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;') tdSql.checkRows(3) - tdSql.checkData(0, 0, 36) + tdSql.checkData(0, 0, 37) tdSql.checkData(0, 1, 'information_schema') tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 5) @@ -97,7 +97,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'performance_schema') tdSql.checkData(0, 0, 3) tdSql.checkData(0, 1, 'tbl_count') - tdSql.checkData(2, 0, 36) + tdSql.checkData(2, 0, 37) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -110,7 +110,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 44) + tdSql.checkData(0, 0, 45) tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -193,7 +193,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 36) + tdSql.checkData(3, 0, 37) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -208,7 +208,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 36) + tdSql.checkData(3, 0, 37) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -219,7 +219,7 @@ class TDTestCase: tdSql.checkData(0, 1, 'tbl_count') tdSql.checkData(1, 0, 5) tdSql.checkData(1, 1, 'performance_schema') - tdSql.checkData(2, 0, 36) + tdSql.checkData(2, 0, 37) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -232,7 +232,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 45) + tdSql.checkData(0, 0, 46) tdSql.execute('drop database tbl_count') diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 2035d80705..327def87ea 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -250,6 +250,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py +,,n,system-test,python3 ./test.py -f 2-query/large_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 2 @@ -408,6 +409,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_transaction_detail.py -N 3 +# ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/kill_balance_leader.py -N 3 +# ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/kill_restore_dnode.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 536b8f30d3..eee8809ad0 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -19,6 +19,7 @@ import inspect import importlib import traceback from util.log import * +import platform class TDCase: @@ -146,5 +147,42 @@ class TDCases: tdLog.notice("total %d Cluster test case(s) executed" % (runNum)) + def getTaosBenchmarkPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + else: + projPath = selfPath[: selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmarkExec(self, param): + buildPath = tdCases.getTaosBenchmarkPath() + + if (platform.system().lower() == 'windows'): + cmdStr1 = ' mintty -h never %s %s '%(buildPath, param) + tdLog.info(cmdStr1) + os.system(cmdStr1) + else: + cmdStr1 = '%s %s &'%(buildPath, param) + tdLog.info(cmdStr1) + os.system(cmdStr1) + + time.sleep(5) + tdCases = TDCases() diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bb7b8411f9..cdfe3ce8a0 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -100,7 +100,7 @@ class TDSql: if drop: s = f'drop database if exists {dbname}' self.cursor.execute(s) - s = f'create database {dbname}' + s = f'create database {dbname} stt_trigger 1' for k, v in kwargs.items(): s += f" {k} {v}" if "duration" not in kwargs: diff --git a/tests/script/api/stmt2-performance.c b/tests/script/api/stmt2-performance.c index a539affaf1..7ab20d873f 100644 --- a/tests/script/api/stmt2-performance.c +++ b/tests/script/api/stmt2-performance.c @@ -5,8 +5,8 @@ #include #include "taos.h" -int CTB_NUMS = 2; -int ROW_NUMS = 2; +int CTB_NUMS = 1; +int ROW_NUMS = 1; int CYC_NUMS = 2; void do_query(TAOS* taos, const char* sql) { @@ -217,7 +217,7 @@ int main() { exit(1); } - do_stmt(taos, "insert into db.stb(tbname,ts,b,t1,t2) values(?,?,?,?,?)"); + do_stmt(taos, "insert into `db`.`stb` (tbname,ts,b,t1,t2) values(?,?,?,?,?)"); // do_stmt(taos, "insert into db.? using db.stb tags(?,?)values(?,?)"); // do_taosc(taos); taos_close(taos); diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index 8923804547..da2083b013 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -13,7 +13,7 @@ if [ -n "$PID" ]; then systemctl stop taosd fi -PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w taosd | grep -v grep | grep -v taosanode | awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 taosd @@ -38,10 +38,10 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w taos | grep -v grep |grep -v taosanode| awk '{print $2}'` done -PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` +PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode|awk '{print $2}'` while [ -n "$PID" ]; do echo kill -9 $PID #pkill -9 tmq_sim @@ -52,5 +52,5 @@ while [ -n "$PID" ]; do else lsof -nti:6030 | xargs kill -9 fi - PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'` + PID=`ps -ef|grep -w tmq_sim | grep -v grep | grep -v taosanode| awk '{print $2}'` done \ No newline at end of file diff --git a/tests/script/tsim/analytics/basic0.sim b/tests/script/tsim/analytics/basic0.sim index 77c9184e8f..3ac49b1fc3 100644 --- a/tests/script/tsim/analytics/basic0.sim +++ b/tests/script/tsim/analytics/basic0.sim @@ -3,7 +3,17 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print =============== create anode +print =============== failed to create anode on '127.0.0.1:1101' +sql_error create anode '127.0.0.1:1101' + +sql show anodes +if $rows != 0 then + return -1 +endi + +sql_error drop anode 1 + +print ================ create anode sql create anode '192.168.1.116:6050' sql show anodes @@ -30,7 +40,7 @@ print $data00 $data01 $data02 sql use d0 print =============== create super table, include column type for count/sum/min/max/first -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 tinyint, c5 bigint, c6 varchar(12)) tags (t1 int unsigned) sql show stables if $rows != 1 then @@ -42,10 +52,11 @@ sql create table ct1 using stb tags(1000) print ==================== insert data # input_list = [5, 14, 15, 15, 14, 19, 17, 16, 20, 22, 8, 21, 28, 11, 9, 29, 40] -sql insert into ct1(ts, c1) values(now-1a, 5)(now+1a, 14)(now+2a, 15)(now+3a, 15)(now+4a, 14) -sql insert into ct1(ts, c1) values(now+5a, 19)(now+6a, 17)(now+7a, 16)(now+8a, 20)(now+9a, 22) -sql insert into ct1(ts, c1) values(now+10a, 8)(now+11a, 21)(now+12a, 28)(now+13a, 11)(now+14a, 9) -sql insert into ct1(ts, c1) values(now+15a, 29)(now+16a, 40) +sql insert into ct1(ts, c1, c2, c3, c4, c5, c6) values(now-1a, 5, 5, 5, 5, 5, 'a')(now+1a, 14, 14, 14, 14, 14, 'a')(now+2a, 15, 15, 15, 15, 15, 'a') +sql insert into ct1 values(now+3a, 15, 15, 15, 15, 15, 'a')(now+4a, 14, 14, 14, 14, 14, 'a')(now+5a, 19, 19, 19, 19, 19, 'a')(now+6a, 17, 17, 17, 17, 17, 'a') +sql insert into ct1 values(now+7a, 16, 16, 16, 16, 16, 'a')(now+8a, 20, 20, 20, 20, 20, 'a')(now+9a, 22, 22, 22, 22, 22, 'a') +sql insert into ct1 values(now+10a, 8, 8, 8, 8, 8, 'a')(now+11a, 21, 21, 21, 21, 21, 'a')(now+12a, 28, 28, 28, 28, 28, 'a')(now+13a, 11, 11, 11, 11, 11, 'a')(now+14a, 9, 9, 9, 9, 9, 'a') +sql insert into ct1 values(now+15a, 29, 29, 29, 29, 29, 'a')(now+16a, 40, 40, 40, 40, 40, 'a') sql select count(*) from ct1 if $data00 != 17 then @@ -58,6 +69,87 @@ if $data00 != 1 then return -1 endi +print ================= try every loaded anomaly detection algorithm +sql select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=lof'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=shesd'); +sql select count(*) from ct1 anomaly_window(c1, 'algo=grubbs'); + +print ================= try every column type of column +sql select count(*) from ct1 anomaly_window(c1, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c2, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c3, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c4, 'algo=ksigma,k=2'); +sql select count(*) from ct1 anomaly_window(c5, 'algo=ksigma,k=2'); + +print =================== invalid column type +sql_error select count(*) from ct1 anomaly_window(c6, 'algo=ksigma,k=2'); +sql_error select forecast(c6, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 + + +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=119,wncheck=1,period=0') from ct1 +sql_error select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters1,conf=0.5,wncheck=1,period=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=0') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, rows=-10') from ct1 +sql_error select forecast(c1, 'conf=50 ,algo = arima, every=0') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters, conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo=holtwinters , conf=50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, ' algo = holtwinters , conf = 50 ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, , ,') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, a =') from ct1 +sql select _frowts, _flow, _fhigh, forecast(c1, 'conf=50 ,algo = holtwinters, = a ,') from ct1 + +print =================== valid column type +sql select forecast(c1, 'conf=50 ,algo = arima') from ct1 +sql select forecast(c1, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c2, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c3, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c4, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=2') from ct1 +if $rows != 10 then + return -1 +endi + +if $data03 != 28 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + print expect 23-11-15 06:13:20.000 , actual $data00 + return -1 +endi + +if $data10 != @23-11-15 06:13:20.002@ then + print expect 23-11-15 06:13:20.002 , actual $data10 + return -1 +endi + +if $data20 != @23-11-15 06:13:20.004@ then + return -1 +endi + +print test the every option and rows option + +sql select _frowts, _flow, _fhigh, forecast(c1, 'algo=holtwinters,conf=50,wncheck=1,period=0,start=1700000000000,every=100,rows=5') from ct1 +if $rows != 5 then + return -1 +endi + +if $data00 != @23-11-15 06:13:20.000@ then + return -1 +endi + +if $data10 != @23-11-15 06:13:20.100@ then + return -1 +endi sql drop anode 1 sql show anodes @@ -66,6 +158,13 @@ if $rows != 0 then return -1 endi +sleep 1000 + +print ===================== query without anodes +sql_error select forecast(c5, 'conf=50 ,algo = arima, rows=1') from ct1 +sql_error select count(*) from ct1 anomaly_window(c1, 'algo=iqr'); + + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index f8c1d6a986..c6c979e97c 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -58,7 +58,7 @@ endi sql select tbname from information_schema.ins_tables; print $rows $data00 -if $rows != 45 then +if $rows != 46 then return -1 endi if $data00 != @ins_tables@ then diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim index 49e9126361..19568b42b6 100644 --- a/tests/script/tsim/query/tableCount.sim +++ b/tests/script/tsim/query/tableCount.sim @@ -53,7 +53,7 @@ sql select stable_name,count(table_name) from information_schema.ins_tables grou if $rows != 3 then return -1 endi -if $data01 != 42 then +if $data01 != 43 then return -1 endi if $data11 != 10 then @@ -72,7 +72,7 @@ endi if $data11 != 5 then return -1 endi -if $data21 != 36 then +if $data21 != 37 then return -1 endi if $data31 != 5 then @@ -97,7 +97,7 @@ endi if $data42 != 3 then return -1 endi -if $data52 != 36 then +if $data52 != 37 then return -1 endi if $data62 != 5 then diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index af0dd6d949..8c25b70b76 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -61,7 +61,7 @@ class TDTestCase: self.ins_list = ['ins_dnodes','ins_mnodes','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges','ins_views', - 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets"] + 'ins_compacts', 'ins_compact_details', 'ins_grants_full','ins_grants_logs', 'ins_machines', 'ins_arbgroups', 'ins_tsmas', "ins_encryptions", "ins_anodes", "ins_anodes_full", "ins_disk_usagea", "ins_filesets", "ins_transaction_details"] self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] def insert_data(self,column_dict,tbname,row_num): insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) @@ -222,10 +222,10 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(303, 304)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(309, 310)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") - tdSql.checkEqual(60, len(tdSql.queryResult)) + tdSql.checkEqual(61, len(tdSql.queryResult)) def ins_dnodes_check(self): tdSql.execute('drop database if exists db2') diff --git a/tests/system-test/0-others/kill_balance_leader.py b/tests/system-test/0-others/kill_balance_leader.py new file mode 100644 index 0000000000..be86336661 --- /dev/null +++ b/tests/system-test/0-others/kill_balance_leader.py @@ -0,0 +1,64 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import * +from util.cases import * +from util.dnodes import * +from util.sql import * + + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to init {__file__}") + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), logSql) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + def run(self): + tdLog.debug(f"start to excute {__file__}") + + tdSql.execute('CREATE DATABASE db vgroups 160 replica 3;') + + tdSql.execute('balance vgroup leader') + + sql ="show transactions;" + rows = tdSql.query(sql) + + if rows > 0: + tranId = tdSql.getData(0, 0) + tdLog.info('kill transaction %d'%tranId) + tdSql.execute('kill transaction %d'%tranId, queryTimes=1 ) + + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + + def waitTransactionZero(self, seconds = 300, interval = 1): + # wait end + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("transaction count became zero.") + return True + #tdLog.info(f"i={i} wait ...") + time.sleep(interval) + + return False + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/show_transaction_detail.py b/tests/system-test/0-others/show_transaction_detail.py new file mode 100644 index 0000000000..ba588f8f01 --- /dev/null +++ b/tests/system-test/0-others/show_transaction_detail.py @@ -0,0 +1,115 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.log import * +from util.cases import * +from util.dnodes import * +from util.sql import * +from util.cluster import * +import threading + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to init {__file__}") + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), logSql) + self.dnodes = cluster.dnodes + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + def run(self): + tdLog.debug(f"start to excute {__file__}") + + tdLog.info("CREATE DATABASE db1 vgroups 16 replica 1;") + tdSql.execute('CREATE DATABASE db1 vgroups 16 replica 1;') + + if self.waitTransactionZero() is False: + tdLog.exit(f"{sql} transaction not finished") + return False + + newTdSql1=tdCom.newTdSql() + t1 = threading.Thread(target=self.alterDbThread, args=('', newTdSql1)) + + newTdSql2=tdCom.newTdSql() + t2 = threading.Thread(target=self.createDbThread, args=('', newTdSql2)) + + t1.start() + t2.start() + + #time.sleep(1) + + dnode = self.dnodes[2] + + # stop dnode + tdLog.info(f"stop dnode 2") + dnode.stoptaosd() + + tdLog.info(f"show transactions;") + rows = tdSql.query("show transactions;", queryTimes=1) + tdLog.info(f"rows={rows}") + if rows > 0: + tranId1 = tdSql.getData(0, 0) + oper1 = tdSql.getData(0, 3) + tdLog.info(f"show transaction {tranId1}, {oper1}") + + #tranId2 = tdSql.getData(1, 0) + #oper2 = tdSql.getData(1, 3) + #tdLog.info(f"show transaction {tranId2}, {oper2}") + + rows = tdSql.query(f"show transaction {tranId1}", queryTimes=1) + if rows != 120 and rows != 176: + tdLog.exit(f"show transaction detial error, rows={rows}") + return False + + #rows = tdSql.query(f"show transaction {tranId2}", queryTimes=1) + #if rows != 176 and rows != 120: + # tdLog.exit(f"show transaction detial error, rows={rows}") + # return False + + tdLog.info(f"select * from ins_transaction_details") + rows = tdSql.query(f"select * from information_schema.ins_transaction_details", queryTimes=1) + + #if rows != 296: + if rows != 176: + tdLog.exit(f"show transaction detial error, rows={rows}") + return False + + dnode.starttaosd() + + t1.join() + t2.join() + + def createDbThread(self, sql, newTdSql): + tdLog.info("CREATE DATABASE db2 vgroups 40 replica 3;") + newTdSql.execute('CREATE DATABASE db2 vgroups 40 replica 3;', queryTimes=1) + + def alterDbThread(self, sql, newTdSql): + tdLog.info("alter DATABASE db1 replica 3;") + newTdSql.execute('alter DATABASE db1 replica 3;', queryTimes=1) + + def waitTransactionZero(self, seconds = 300, interval = 1): + # wait end + for i in range(seconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + tdLog.info("transaction count became zero.") + return True + #tdLog.info(f"i={i} wait ...") + time.sleep(interval) + + return False + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index e217bbe183..2f7afef6ec 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -78,9 +78,34 @@ class TDTestCase: tdSql.error(f'create table {sql} (ts timestamp,c0 int)') tdSql.execute(f'trim database `{dbname}`') tdSql.execute(f'drop database `{dbname}`') + + def tb_name_len_check(self): + dbname = tdCom.getLongName(10) + tdSql.execute(f'create database if not exists `{dbname}` vgroups 1 replica 1') + tdSql.execute(f'use `{dbname}`') + tdSql.execute(f'CREATE STABLE `test_csv` (`ts` TIMESTAMP, `c1` VARCHAR(2000), `c2` VARCHAR(2000)) TAGS (`c3` VARCHAR(2000))') + tbname = "test_csv_a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" + tdSql.execute(f"INSERT INTO `{tbname}`\ + using `test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\ + (`ts`,`c1`,`c2`) values(1591060628000,'a','1');") + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {tbname}') + + tdSql.execute(f"INSERT INTO `{dbname}`.`{tbname}`\ + using `{dbname}`.`test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\ + (`ts`,`c1`,`c2`) values(1591060628000,'a','1');") + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {tbname}') + + tdSql.execute(f'trim database `{dbname}`') + tdSql.execute(f'drop database `{dbname}`') + def run(self): self.db_name_check() self.tb_name_check() + self.tb_name_len_check() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/large_data.py b/tests/system-test/2-query/large_data.py new file mode 100644 index 0000000000..2279c3ce70 --- /dev/null +++ b/tests/system-test/2-query/large_data.py @@ -0,0 +1,56 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def prepare_data(self): + tdSql.execute("drop database if exists test;") + + tdCases.taosBenchmarkExec("-t 2 -n 1000000 -b int,float,nchar -y") + + while True: + tdSql.query("select ts from test.d0;") + num1 = tdSql.queryRows + tdSql.query("select ts from test.d1;") + num2 = tdSql.queryRows + if num1 == 1000000 and num2 == 1000000: + break + tdLog.info(f"waiting for data ready, d0: {num1}, d1: {num2}") + time.sleep(1) + + def ts5803(self): + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts;") + num1 = tdSql.queryRows + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts;") + num1 = tdSql.queryRows + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + def run(self): + self.prepare_data() + self.ts5803() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 04fb893e75..355ac04707 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -39,6 +39,27 @@ class TDTestCase: self.create_tables(); self.ts = 1500000000000 + def test_TD_33336(self): + sql = "flush database db" + tdSql.execute(sql) + time.sleep(5) + sql = f'select last(ts) + 1d, last(ts) - 1d from db.t1' + tdSql.query(sql, queryTimes=1) + lastTs_add1d = tdSql.queryResult[0][0] + lastTs_sub1d = tdSql.queryResult[0][1] + + sql = f'select count(*) from db.t1 where ts < "{lastTs_add1d}" and vc1 = 1' + tdSql.query(sql, queryTimes=1) + all_row_count = tdSql.queryResult[0][0] + tdLog.debug(f"all rows: {all_row_count}") + + sql = f'select count(*) from db.t1 where ts < "{lastTs_sub1d}" and vc1 = 1' + tdSql.query(sql, queryTimes=1) + row_count_sub1d = tdSql.queryResult[0][0] + tdLog.debug(f"row_count_sub1d: {row_count_sub1d}") + + if row_count_sub1d > all_row_count: + tdLog.exit(f' err rows returned for sql: {sql} row_count_sub1d: {row_count_sub1d} > all_row_count: {all_row_count}') # run case def run(self): @@ -53,6 +74,8 @@ class TDTestCase: # self.test_case2() tdLog.debug(" LIMIT test_case2 ............ [OK]") + self.test_TD_33336() + # stop def stop(self): tdSql.close() @@ -65,11 +88,11 @@ class TDTestCase: # create table def create_tables(self, dbname="db"): # super table - tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int) tags(area int)") + tdSql.execute(f"create table {dbname}.st(ts timestamp, i1 int,i2 int, vc1 varchar(255)) tags(area int)") # child table tdSql.execute(f"create table {dbname}.t1 using {dbname}.st tags(1)") - tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int) tags(area int) sma(i2) ") + tdSql.execute(f"create table {dbname}.st1(ts timestamp, i1 int ,i2 int, vc1 varchar(255)) tags(area int) sma(i2) ") tdSql.execute(f"create table {dbname}.t4 using {dbname}.st1 tags(1)") return @@ -98,7 +121,11 @@ class TDTestCase: sql = pre_insert tdLog.debug("insert table %s rows=%d ..." % (tbname, count)) for i in range(count): - sql += " (%d,%d,%d)" % (ts_start + i*1000, i, i+1) + if random.randint(0, 4) > 2: + tail = '' + else: + tail = 'asd' + sql += " (%d,%d,%d,%s)" % (ts_start + i*1000, i, i+1, '"' + str(random.randint(0,5)) + f'{tail}"') if i > 0 and i % 20000 == 0: tdLog.info("%d rows inserted" % i) tdSql.execute(sql) @@ -125,4 +152,4 @@ class TDTestCase: # add case with filename # tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 380b7879c4..1abe2b1a65 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -441,7 +441,7 @@ class TDTestCase: tdSql.checkRows(2) sql = "select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, stable_name `TABLE_NAME`, 'TABLE' `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_stables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, table_name `TABLE_NAME`, case when `type`='SYSTEM_TABLE' then 'TABLE' when `type`='NORMAL_TABLE' then 'TABLE' when `type`='CHILD_TABLE' then 'TABLE' else 'UNKNOWN' end `TABLE_TYPE`, table_comment `REMARKS` from information_schema.ins_tables union all select db_name `TABLE_CAT`, '' `TABLE_SCHEM`, view_name `TABLE_NAME`, 'VIEW' `TABLE_TYPE`, NULL `REMARKS` from information_schema.ins_views" tdSql.query(sql, queryTimes=1) - tdSql.checkRows(49) + tdSql.checkRows(50) sql = "select null union select null" tdSql.query(sql, queryTimes=1) diff --git a/tests/system-test/3-enterprise/restore/kill_restore_dnode.py b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py new file mode 100644 index 0000000000..d29a11f159 --- /dev/null +++ b/tests/system-test/3-enterprise/restore/kill_restore_dnode.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +sys.path.append("./3-enterprise/restore") +from restoreBasic import * +from util.common import tdCom +import threading + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + self.basic = RestoreBasic() + self.basic.init(conn, logSql, replicaVar) + + # run + def run(self): + self.basic.restore_dnode_prepare(2) + + self.execute() + + def execute(self): + newTdSql=tdCom.newTdSql() + t0 = threading.Thread(target=self.restoreDnodeThread, args=('', newTdSql)) + t0.start() + + time.sleep(2) + sql ="show transactions;" + tdLog.info(sql) + rows = tdSql.query(sql) + + if rows > 0: + self.basic.stop_dnode(2) + + tranId = tdSql.getData(0, 0) + + tdLog.info('show transaction %d'%tranId) + rows=tdSql.query('show transaction %d'%tranId, queryTimes=1) + if rows != 11: + tdLog.exit(f"restore transaction detial error, rows={rows}") + return False + + tdLog.info('kill transaction %d'%tranId) + tdSql.execute('kill transaction %d'%tranId, queryTimes=1 ) + + time.sleep(3) + sql ="show transactions;" + tdLog.info(sql) + rows = tdSql.query(sql) + if rows > 0: + tdLog.info(f"{sql} transaction not finished") + return False + + self.basic.restore_dnode_exec(2) + else: + tdLog.exit(f"{sql} no transaction exist") + return False + + def restoreDnodeThread(self, p, newTdSql): + sleep(1) + + sql = f"restore dnode 2" + tdLog.info(sql) + newTdSql.error(sql, expectErrInfo="Wrong transaction execution context") + tdLog.info(f"{sql} finished") + + # stop + def stop(self): + self.basic.stop() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/3-enterprise/restore/restoreBasic.py b/tests/system-test/3-enterprise/restore/restoreBasic.py index 77fa606b9c..74cf572018 100644 --- a/tests/system-test/3-enterprise/restore/restoreBasic.py +++ b/tests/system-test/3-enterprise/restore/restoreBasic.py @@ -143,6 +143,34 @@ class RestoreBasic: tdSql.execute(sql) self.check_corrent() + def restore_dnode_prepare(self, index): + tdLog.info(f"start restore dnode {index}") + dnode = self.dnodes[index - 1] + + # stop dnode + tdLog.info(f"stop dnode {index}") + dnode.stoptaosd() + + # remove dnode folder + try: + shutil.rmtree(dnode.dataDir) + tdLog.info(f"delete dir {dnode.dataDir} successful") + except OSError as x: + tdLog.exit(f"remove path {dnode.dataDir} error : {x.strerror}") + + dnode.starttaosd() + + def restore_dnode_exec(self, index): + # exec restore + sql = f"restore dnode {index}" + tdLog.info(sql) + tdSql.execute(sql) + self.check_corrent() + + def stop_dnode(self, index): + dnode = self.dnodes[index - 1] + + dnode.starttaosd() # restore vnode def restore_vnode(self, index): tdLog.info(f"start restore vnode on dnode {index}")