diff --git a/Jenkinsfile2 b/Jenkinsfile2 index e25be162e8..9e89e5a2f8 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -175,6 +175,7 @@ def pre_test_build_mac() { cd ${WK}/debug cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false make -j10 + ctest -j10 || exit 7 ''' sh ''' date diff --git a/cmake/cmake.version b/cmake/cmake.version index 1a83126cd7..92f7f8b895 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.1.5") + SET(TD_VER_NUMBER "3.0.1.6") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 5a2c8ed6ee..6d12b8c730 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -7,12 +7,12 @@ title: Table You create standard tables and subtables with the `CREATE TABLE` statement. ```sql -CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) [table_options] +CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] CREATE TABLE create_subtable_clause -CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) - [TAGS (create_definition [, create_definitionn] ...)] +CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) + [TAGS (create_definition [, create_definition] ...)] [table_options] create_subtable_clause: { @@ -195,4 +195,4 @@ This command is useful in migrating data from one TDengine cluster to another be ``` DESCRIBE [db_name.]tb_name; -``` \ No newline at end of file +``` diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index a20e33163b..8a7c713f8c 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -6,7 +6,7 @@ title: Supertable ## Create a Supertable ```sql -CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definitionn] ...) TAGS (create_definition [, create_definition] ...) [table_options] +CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definition] ...) TAGS (create_definition [, create_definition] ...) [table_options] create_definition: col_name column_definition diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index b114f92a41..d21f1829b0 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -374,7 +374,11 @@ The configuration parameters for specifying super table tag columns and data col ### Query scenario configuration parameters -`filetype` must be set to `query` in the query scenario. See [General Configuration Parameters](#General Configuration Parameters) for details of this parameter and other general parameters +`filetype` must be set to `query` in the query scenario. + +To control the query scenario by setting `kill_slow_query_threshold` and `kill_slow_query_interval` parameters to kill the execution of slow query statements. Threshold controls exec_usec of query command will be killed by taosBenchmark after the specified time, in seconds; interval controls sleep time to avoid continuous querying of slow queries consuming CPU in seconds. + +See [General Configuration Parameters](#General Configuration Parameters) for details of other general parameters. #### Configuration parameters for executing the specified query statement diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index f6790e3c69..045abcde95 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -9,12 +9,12 @@ description: 对表的各种管理操作 `CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表。 ```sql -CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) [table_options] +CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] CREATE TABLE create_subtable_clause -CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) - [TAGS (create_definition [, create_definitionn] ...)] +CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) + [TAGS (create_definition [, create_definition] ...)] [table_options] create_subtable_clause: { diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 7dff30e443..bd32da68ae 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -7,7 +7,7 @@ description: 对超级表的各种管理操作 ## 创建超级表 ```sql -CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definitionn] ...) TAGS (create_definition [, create_definition] ...) [table_options] +CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definition] ...) TAGS (create_definition [, create_definition] ...) [table_options] create_definition: col_name column_definition diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 3ffc081cd7..3cd68e9b95 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -224,7 +224,7 @@ GROUP BY 子句中的表达式可以包含表或视图中的任何列，这些 该子句对行进行分组，但不保证结果集的顺序。若要对分组进行排序，请使用 ORDER BY 子句 -## PARTITON BY +## PARTITION BY PARTITION BY 子句是 TDengine 特色语法，按 part_list 对数据进行切分，在每个切分的分片中进行计算。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index afc050298f..13530923b8 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -374,7 +374,11 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) ### 查询场景配置参数 -查询场景下 `filetype` 必须设置为 `query`，该参数及其它通用参数详见[通用配置参数](#通用配置参数) +查询场景下 `filetype` 必须设置为 `query`。 + +查询场景可以通过设置 `kill_slow_query_threshold` 和 `kill_slow_query_interval` 参数来控制杀掉慢查询语句的执行，threshold 控制如果 exec_usec 超过指定时间的查询将被 taosBenchmark 杀掉，单位为秒；interval 控制休眠时间，避免持续查询慢查询消耗 CPU ，单位为秒。 + +其它通用参数详见[通用配置参数](#通用配置参数)。 #### 执行指定查询语句的配置参数 diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 76b13579c1..7bc56daab0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -297,7 +297,6 @@ typedef struct { typedef struct { int32_t code; - int8_t hashMeta; int64_t uid; char* tblFName; int32_t numOfRows; @@ -1128,6 +1127,7 @@ typedef struct { SQnodeLoad qload; SClusterCfg clusterCfg; SArray* pVloads; // array of SVnodeLoad + int32_t statusSeq; } SStatusReq; int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); @@ -1149,6 +1149,7 @@ typedef struct { int64_t dnodeVer; SDnodeCfg dnodeCfg; SArray* pDnodeEps; // Array of SDnodeEp + int32_t statusSeq; } SStatusRsp; int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 36966779de..58e7b71bec 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -273,6 +273,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT, "sync-pre-snapshot", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT_REPLY, "sync-pre-snapshot-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b957be4267..b310ec8080 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -203,13 +203,11 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); -int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - STableMeta** pTableMeta); +int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - SVgroupInfo* pVgroup, bool* exists); +int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); /** * Force refresh DB's local cached vgroup info. @@ -309,7 +307,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, +int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); @@ -326,6 +324,10 @@ SMetaData* catalogCloneMetaData(SMetaData* pData); void catalogFreeMetaData(SMetaData* pData); +int32_t ctgdEnableDebug(char *option, bool enable); + +int32_t ctgdHandleDbgCommand(char *command); + /** * Destroy catalog and relase all resources */ diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 7d95e840e9..651b379851 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -242,13 +242,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ - ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ + ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_TDB_TABLE_NOT_EXIST || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) -#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) +#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER) #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index dbe8a347da..694a6ef62b 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -25,8 +25,6 @@ extern "C" { #include "tlrucache.h" #include "tmsgcb.h" -extern bool gRaftDetailLog; - #define SYNC_RESP_TTL_MS 10000000 #define SYNC_SPEED_UP_HB_TIMER 400 #define SYNC_SPEED_UP_AFTER_MS (1000 * 20) @@ -39,6 +37,7 @@ extern bool gRaftDetailLog; #define SYNC_ADD_QUORUM_COUNT 3 #define SYNC_MNODE_LOG_RETENTION 10000 #define SYNC_VNODE_LOG_RETENTION 100 +#define SNAPSHOT_MAX_CLOCK_SKEW_MS 1000 * 10 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 @@ -132,7 +131,7 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - void (*FpCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta *pMeta); + void (*FpCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); @@ -202,37 +201,27 @@ typedef struct SSyncInfo { int32_t (*syncEqCtrlMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg); } SSyncInfo; -int32_t syncInit(); -void syncCleanUp(); -bool syncIsInit(); -int64_t syncOpen(SSyncInfo* pSyncInfo); -void syncStart(int64_t rid); -void syncStop(int64_t rid); -ESyncState syncGetMyRole(int64_t rid); -bool syncIsReady(int64_t rid); -const char* syncGetMyRoleStr(int64_t rid); -bool syncRestoreFinish(int64_t rid); -SyncTerm syncGetMyTerm(int64_t rid); -SyncIndex syncGetLastIndex(int64_t rid); -SyncIndex syncGetCommitIndex(int64_t rid); -SyncGroupId syncGetVgId(int64_t rid); -void syncGetEpSet(int64_t rid, SEpSet* pEpSet); -void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); -// int32_t syncProposeBatch(int64_t rid, SRpcMsg** pMsgPArr, bool* pIsWeakArr, int32_t arrSize); -const char* syncStr(ESyncState state); -bool syncIsRestoreFinish(int64_t rid); -int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); +typedef struct SSyncState { + ESyncState state; + bool restored; +} SSyncState; +int32_t syncInit(); +void syncCleanUp(); +int64_t syncOpen(SSyncInfo* pSyncInfo); +void syncStart(int64_t rid); +void syncStop(int64_t rid); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); +int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); -int32_t syncLeaderTransfer(int64_t rid); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); int32_t syncEndSnapshot(int64_t rid); +int32_t syncLeaderTransfer(int64_t rid); int32_t syncStepDown(int64_t rid, SyncTerm newTerm); -int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); - -const char* syncUtilState2String(ESyncState state); +SSyncState syncGetState(int64_t rid); +void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); +const char* syncStr(ESyncState state); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2365469066..092ede2281 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -91,6 +91,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0128) #define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0129) #define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x012A) +#define TSDB_CODE_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012B) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -106,7 +107,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) #define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) #define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) -#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) #define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) #define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) #define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) @@ -305,34 +305,14 @@ int32_t* taosGetErrno(); #define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A) // vnode -#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) -#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) -#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) #define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) -#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) -#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) -#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) -#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) -#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) -#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) -#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) -#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) -#define TSDB_CODE_VND_IS_UPDATING TAOS_DEF_ERROR_CODE(0, 0x050E) -#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) -#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) #define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) -#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) -#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) -#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0520) -#define TSDB_CODE_VND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0521) #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0522) -#define TSDB_CODE_VND_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0523) #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0524) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0525) -#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0526) +#define TSDB_CODE_VND_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0526) #define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527) -#define TSDB_CODE_VND_INVALID_CFG_FILE TAOS_DEF_ERROR_CODE(0, 0x0528) -#define TSDB_CODE_VND_INVALID_TERM_FILE TAOS_DEF_ERROR_CODE(0, 0x0529) +#define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -341,7 +321,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) #define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) #define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) -#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) #define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) #define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) #define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) @@ -361,13 +340,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0618) #define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) -#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061B) +#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) -#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) #define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) #define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) #define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) @@ -430,7 +408,6 @@ int32_t* taosGetErrno(); // tq #define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) -#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) #define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) #define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) #define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 20f93ecaec..b19d8257a0 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -211,7 +211,8 @@ elif [[ ${packgeName} =~ "tar" ]];then if [ ${diffNumbers} != 0 ];then echoColor R "The number and names of files is different from the previous installation package" - echoColor Y `cat ${installPath}/diffFile.log` + diffLog=`cat ${installPath}/diffFile.log` + echoColor Y "${diffLog}" exit -1 else echoColor G "The number and names of files are the same as previous installation packages" diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cf8b0babdd..33af862528 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -318,6 +318,7 @@ void* createTscObj(const char* user, const char* auth, const char* db, int32_ void destroyTscObj(void* pObj); STscObj* acquireTscObj(int64_t rid); int32_t releaseTscObj(int64_t rid); +void destroyAppInst(SAppInstInfo *pAppInfo); uint64_t generateRequestId(); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 2261a6becb..ef4c05afae 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -39,6 +39,7 @@ typedef enum { STMT_BIND_COL, STMT_ADD_BATCH, STMT_EXECUTE, + STMT_MAX, } STMT_STATUS; typedef struct SStmtTableCache { @@ -94,12 +95,18 @@ typedef struct STscStmt { STscObj *taos; SCatalog *pCatalog; int32_t affectedRows; + uint32_t seqId; + uint32_t seqIds[STMT_MAX]; SStmtSQLInfo sql; SStmtExecInfo exec; SStmtBindInfo bInfo; } STscStmt; +extern char *gStmtStatusStr[]; + +#define STMT_LOG_SEQ(n) do { (pStmt)->seqId++; (pStmt)->seqIds[n]++; STMT_DLOG("the %dth:%d %s", (pStmt)->seqIds[n], (pStmt)->seqId, gStmtStatusStr[n]); } while (0) + #define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S) #define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S) @@ -128,6 +135,12 @@ typedef struct STscStmt { } \ } while (0) +#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) +#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) + +#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt) +#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt) + TAOS_STMT *stmtInit(STscObj *taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c12d78c18..e9e30a0be2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -133,6 +133,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas taosThreadMutexInit(&p->qnodeMutex, NULL); p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); + if (NULL == p->pAppHbMgr) { + destroyAppInst(p); + taosThreadMutexUnlock(&appInfo.mutex); + taosMemoryFreeClear(key); + return NULL; + } taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); p->instKey = key; key = NULL; @@ -1266,7 +1272,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType); - pMsgSendInfo->param = pRequest; + pMsgSendInfo->param = taosMemoryCalloc(1, sizeof(pRequest->self)); + + *(int64_t*)pMsgSendInfo->param = pRequest->self; SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b03576ff01..a08eab3a29 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -286,7 +286,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } else { // assert to avoid un-initialization error - ASSERT(0); + tscError("invalid result passed to taos_fetch_row"); + return NULL; } return NULL; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e586018c19..938cc4e41d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -45,8 +45,13 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) { } int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { - SRequestObj* pRequest = param; - + SRequestObj *pRequest = acquireRequest(*(int64_t*)param); + if (NULL == pRequest) { + setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); + tsem_post(&pRequest->body.rspSem); + goto End; + } + if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); @@ -55,6 +60,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { STscObj* pTscObj = pRequest->pTscObj; + if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) { + setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); + tsem_post(&pRequest->body.rspSem); + goto End; + } + SConnectRsp connectRsp = {0}; if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) { code = TSDB_CODE_TSC_INVALID_VERSION; @@ -115,10 +126,15 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); - + tsem_post(&pRequest->body.rspSem); End: + if (pRequest) { + releaseRequest(pRequest->self); + } + + taosMemoryFree(param); taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pData); return code; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f05d37765d..4326ab3519 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1181,7 +1181,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { pVgData = NULL; pArray = NULL; code = pRequest->code; - if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { + if (code == TSDB_CODE_TDB_TABLE_NOT_EXIST) { code = TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 81d0d616c9..7f8d857a0f 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -5,6 +5,8 @@ #include "clientStmt.h" +char *gStmtStatusStr[] = {"unknown", "init", "prepare", "settbname", "settags", "fetchFields", "bind", "bindCol", "addBatch", "exec"}; + static int32_t stmtCreateRequest(STscStmt* pStmt) { int32_t code = 0; @@ -21,6 +23,10 @@ static int32_t stmtCreateRequest(STscStmt* pStmt) { int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { int32_t code = 0; + if (newStatus >= STMT_INIT && newStatus < STMT_MAX) { + STMT_LOG_SEQ(newStatus); + } + switch (newStatus) { case STMT_PREPARE: break; @@ -528,13 +534,17 @@ TAOS_STMT* stmtInit(STscObj* taos) { pStmt->bInfo.needParse = true; pStmt->sql.status = STMT_INIT; + STMT_LOG_SEQ(STMT_INIT); + + tscDebug("stmt:%p initialized", pStmt); + return pStmt; } int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { STscStmt* pStmt = (STscStmt*)stmt; - tscDebug("stmt start to prepare"); + STMT_DLOG_E("start to prepare"); if (pStmt->sql.status >= STMT_PREPARE) { STMT_ERR_RET(stmtResetStmt(pStmt)); @@ -555,7 +565,7 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STscStmt* pStmt = (STscStmt*)stmt; - tscDebug("stmt start to set tbName: %s", tbName); + STMT_DLOG("start to set tbName: %s", tbName); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); @@ -587,7 +597,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { STscStmt* pStmt = (STscStmt*)stmt; - tscDebug("stmt start to set tbTags"); + STMT_DLOG_E("start to set tbTags"); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); @@ -649,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; - tscDebug("start to bind stmt data, colIdx: %d", colIdx); + STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); @@ -743,7 +753,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { int stmtAddBatch(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; - tscDebug("stmt start to add batch"); + STMT_DLOG_E("start to add batch"); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH)); @@ -756,8 +766,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks); if (pRsp->nBlocks <= 0) { - tscError("invalid submit resp block number %d", pRsp->nBlocks); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + return TSDB_CODE_SUCCESS; } size_t keyLen = 0; @@ -810,7 +819,7 @@ int stmtExec(TAOS_STMT* stmt) { SSubmitRsp* pRsp = NULL; bool autoCreateTbl = pStmt->exec.autoCreateTbl; - tscDebug("stmt start to exec"); + STMT_DLOG_E("start to exec"); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); @@ -885,6 +894,8 @@ int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->exec.affec int stmtIsInsert(TAOS_STMT* stmt, int* insert) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start is insert"); + if (pStmt->sql.type) { *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type); } else { @@ -897,6 +908,8 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to get tag fields"); + if (STMT_TYPE_QUERY == pStmt->sql.type) { STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); } @@ -927,6 +940,8 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to get col fields"); + if (STMT_TYPE_QUERY == pStmt->sql.type) { STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); } @@ -957,6 +972,8 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to get param num"); + STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && @@ -986,6 +1003,8 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to get param"); + if (STMT_TYPE_QUERY == pStmt->sql.type) { STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); } @@ -1028,6 +1047,8 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) { TAOS_RES* stmtUseResult(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to use result"); + if (STMT_TYPE_QUERY != pStmt->sql.type) { tscError("useResult only for query statement"); return NULL; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b8436237c0..0b27d9b527 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -285,7 +285,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int // ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pColumnInfoData->pData = tmp; @@ -1913,6 +1913,22 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); if (len >= size - 1) return dumpBuf; break; + case TSDB_DATA_TYPE_TINYINT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(int8_t*)var); + if (len >= size - 1) return dumpBuf; + break; + case TSDB_DATA_TYPE_UTINYINT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint8_t*)var); + if (len >= size - 1) return dumpBuf; + break; + case TSDB_DATA_TYPE_SMALLINT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(int16_t*)var); + if (len >= size - 1) return dumpBuf; + break; + case TSDB_DATA_TYPE_USMALLINT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(uint16_t*)var); + if (len >= size - 1) return dumpBuf; + break; case TSDB_DATA_TYPE_INT: len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); if (len >= size - 1) return dumpBuf; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 073e6bc8c6..4eaa934676 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1020,6 +1020,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, pReq->qload.timeInQueryQueue) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1; + if (tEncodeI32(&encoder, pReq->statusSeq) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1095,6 +1096,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &pReq->qload.timeInQueryQueue) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->statusSeq) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1126,6 +1128,7 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { if (tEncodeU16(&encoder, pDnodeEp->ep.port) < 0) return -1; } + if (tEncodeI32(&encoder, pRsp->statusSeq) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1167,6 +1170,7 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { } } + if (tDecodeI32(&decoder, &pRsp->statusSeq) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -5432,9 +5436,12 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1; - if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1; if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1; - if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; + if (pBlock->tblFName) { + if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1; if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1; if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1; @@ -5451,7 +5458,6 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; - if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1; if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); if (NULL == pBlock->tblFName) return -1; diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index fe5c599d53..3a6bc097c1 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -31,7 +31,7 @@ static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); */ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_TSC_NO_DISKSPACE; + terrno = TSDB_CODE_NO_DISKSPACE; // tscError("tmp file created failed since %s", terrstr()); return NULL; } diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index dc4412b77b..c776beb3f0 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -36,6 +36,7 @@ typedef struct SDnodeMgmt { GetVnodeLoadsFp getVnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; + int32_t statusSeq; } SDnodeMgmt; // dmHandle.c diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index f12dce5149..85a09b79fd 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -32,9 +32,13 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { } static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { + const STraceId *trace = &pRsp->info.traceId; + dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code); + if (pRsp->code != 0) { if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) { - dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->pData->dnodeId); + dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId, + pMgmt->statusSeq); pMgmt->pData->dropped = 1; dmWriteEps(pMgmt->pData); } @@ -42,9 +46,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { SStatusRsp statusRsp = {0}; if (pRsp->pCont != NULL && pRsp->contLen > 0 && tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { - dTrace("status msg received from mnode, dnodeVer:%" PRId64 " saved:%" PRId64, statusRsp.dnodeVer, - pMgmt->pData->dnodeVer); if (pMgmt->pData->dnodeVer != statusRsp.dnodeVer) { + dGInfo("status rsp received from mnode, statusSeq:%d:%d dnodeVer:%" PRId64 ":%" PRId64, pMgmt->statusSeq, + statusRsp.statusSeq, pMgmt->pData->dnodeVer, statusRsp.dnodeVer); pMgmt->pData->dnodeVer = statusRsp.dnodeVer; dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); @@ -91,6 +95,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (*pMgmt->getQnodeLoadsFp)(&req.qload); + pMgmt->statusSeq++; + req.statusSeq = pMgmt->statusSeq; + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); tSerializeSStatusReq(pHead, contLen, &req); @@ -99,13 +106,13 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcRsp = {0}; - dTrace("send status msg to mnode, dnodeVer:%" PRId64, req.dnodeVer); + dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { - dError("failed to send status msg since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps, + dError("failed to send status req since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("index:%d, mnode ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 2c4a8e9b70..976f2cfd8b 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -195,6 +195,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 6faa3354ee..51213a6ab3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -437,6 +437,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 3dfcb8a708..f1e867a78a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -175,7 +175,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp break; case WRITE_QUEUE: if (!osDataSpaceAvailable()) { - terrno = TSDB_CODE_VND_NO_DISKSPACE; + terrno = TSDB_CODE_NO_DISKSPACE; code = terrno; dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); } else if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index a222ad3f7d..421d723202 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -71,17 +71,17 @@ static bool dmCheckDiskSpace() { bool ret = true; if (!osDataSpaceAvailable()) { dError("data disk space unavailable, i.e. %s", tsDataDir); - terrno = TSDB_CODE_VND_NO_DISKSPACE; + terrno = TSDB_CODE_NO_DISKSPACE; ret = false; } if (!osLogSpaceAvailable()) { dError("log disk space unavailable, i.e. %s", tsLogDir); - terrno = TSDB_CODE_VND_NO_DISKSPACE; + terrno = TSDB_CODE_NO_DISKSPACE; ret = false; } if (!osTempSpaceAvailable()) { dError("temp disk space unavailable, i.e. %s", tsTempDir); - terrno = TSDB_CODE_VND_NO_DISKSPACE; + terrno = TSDB_CODE_NO_DISKSPACE; ret = false; } return ret; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 4334bcfb48..408244bfb5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -105,6 +105,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } if (IsReq(pRpc) && pRpc->pCont == NULL) { + dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; goto _OVER; } @@ -133,6 +134,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { break; } } else { + dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c4ab741a41..04f340b0ff 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -345,6 +345,19 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } } + int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pDnode, curMs); + bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); + bool reboot = (pDnode->rebootTime != statusReq.rebootTime); + bool needCheck = !online || dnodeChanged || reboot; + + pDnode->accessTimes++; + pDnode->lastAccessTime = curMs; + const STraceId *trace = &pReq->info.traceId; + mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, + pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); + for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) { SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v); @@ -396,13 +409,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { mndReleaseQnode(pMnode, pQnode); } - int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); - int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pDnode, curMs); - bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); - bool reboot = (pDnode->rebootTime != statusReq.rebootTime); - bool needCheck = !online || dnodeChanged || reboot; - if (needCheck) { if (statusReq.sver != tsVersion) { if (pDnode != NULL) { @@ -424,9 +430,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pMnode->clusterId); terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID; goto _OVER; - } else { - pDnode->accessTimes++; - mDebug("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } } @@ -453,6 +456,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode->memTotal = statusReq.memTotal; SStatusRsp statusRsp = {0}; + statusRsp.statusSeq++; statusRsp.dnodeVer = dnodeVer; statusRsp.dnodeCfg.dnodeId = pDnode->id; statusRsp.dnodeCfg.clusterId = pMnode->clusterId; @@ -473,8 +477,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pReq->info.rsp = pHead; } - pDnode->lastAccessTime = curMs; - pDnode->accessTimes++; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 76f440f536..f604a9289a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -487,14 +487,14 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { } if (mndAcquireRpc(pMsg->info.node) == 0) return 0; - SMnode *pMnode = pMsg->info.node; - const char *role = syncGetMyRoleStr(pMnode->syncMgmt.sync); - bool restored = syncIsRestoreFinish(pMnode->syncMgmt.sync); + SMnode *pMnode = pMsg->info.node; + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, - pMnode->stopped, restored, role); + pMnode->stopped, state.restored, syncStr(state.restored)); return -1; } @@ -505,8 +505,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { mDebug( "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "role:%s, redirect numOfEps:%d inUse:%d", - pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, restored, role, epSet.numOfEps, - epSet.inUse); + pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, + syncStr(state.restored), epSet.numOfEps, epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { @@ -729,8 +729,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr } int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { - pLoad->syncState = syncGetMyRole(pMnode->syncMgmt.sync); - pLoad->syncRestore = pMnode->restored; + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + pLoad->syncState = state.state; + pLoad->syncRestore = state.restored; mTrace("mnode current syncState is %s, syncRestore:%d", syncStr(pLoad->syncState), pLoad->syncRestore); return 0; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 3e4c8005b4..ca69d0c71c 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -55,7 +55,7 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg, 0); break; default: - terrno = TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; mError("unknown msg type:%d in query queue", pMsg->msgType); } @@ -92,7 +92,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { for (int32_t i = 0; i < msgNum; ++i) { if (offset >= pMsg->contLen) { mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -101,7 +101,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { offset += sizeof(req.msgIdx); if (offset >= pMsg->contLen) { mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -110,7 +110,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { offset += sizeof(req.msgType); if (offset >= pMsg->contLen) { mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -119,7 +119,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { offset += sizeof(req.msgLen); if (offset >= pMsg->contLen) { mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3fbe92b264..0f50391ac5 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -349,11 +349,15 @@ void mndSyncStop(SMnode *pMnode) { } bool mndIsLeader(SMnode *pMnode) { - SSyncMgmt *pMgmt = &pMnode->syncMgmt; + SSyncState state = syncGetState(pMnode->syncMgmt.sync); - if (!syncIsReady(pMgmt->sync)) { - // get terrno from syncIsReady - // terrno = TSDB_CODE_SYN_NOT_LEADER; + if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + } else { + terrno = TSDB_CODE_APP_NOT_READY; + } + mDebug("vgId:1, mnode not ready, state:%s, restore:%d", syncStr(state.state), state.restored); return false; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index b33c01a962..6558a98aaa 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -265,34 +265,47 @@ static int32_t sdbReadFileImp(SSdb *pSdb) { if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read sdb file:%s since %s", file, tstrerror(code)); - break; + goto _OVER; } if (ret != readLen) { code = TSDB_CODE_FILE_CORRUPTED; - mError("failed to read sdb file:%s since %s", file, tstrerror(code)); - break; + mError("failed to read sdb file:%s since %s, ret:%" PRId64 " != readLen:%d", file, tstrerror(code), ret, readLen); + goto _OVER; } readLen = pRaw->dataLen + sizeof(int32_t); + if (readLen >= pRaw->dataLen) { + SSdbRaw *pNewRaw = taosMemoryMalloc(pRaw->dataLen + TSDB_MAX_MSG_SIZE); + if (pNewRaw == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed read sdb file since malloc new sdbRaw size:%d failed", pRaw->dataLen + TSDB_MAX_MSG_SIZE); + goto _OVER; + } + mInfo("malloc new sdbRaw size:%d, type:%d", pRaw->dataLen + TSDB_MAX_MSG_SIZE, pRaw->type); + memcpy(pNewRaw, pRaw, sizeof(SSdbRaw)); + sdbFreeRaw(pRaw); + pRaw = pNewRaw; + } + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); - mError("failed to read sdb file:%s since %s", file, tstrerror(code)); - break; + mError("failed to read sdb file:%s since %s, ret:%" PRId64 " readLen:%d", file, tstrerror(code), ret, readLen); + goto _OVER; } if (ret != readLen) { code = TSDB_CODE_FILE_CORRUPTED; - mError("failed to read sdb file:%s since %s", file, tstrerror(code)); - break; + mError("failed to read sdb file:%s since %s, ret:%" PRId64 " != readLen:%d", file, tstrerror(code), ret, readLen); + goto _OVER; } int32_t totalLen = sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t); if ((!taosCheckChecksumWhole((const uint8_t *)pRaw, totalLen)) != 0) { code = TSDB_CODE_CHECKSUM_ERROR; - mError("failed to read sdb file:%s since %s", file, tstrerror(code)); - break; + mError("failed to read sdb file:%s since %s, readLen:%d", file, tstrerror(code), readLen); + goto _OVER; } code = sdbWriteWithoutFree(pSdb, pRaw); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7e6bd75b2d..289251f560 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -103,7 +103,7 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { break; default: qError("unknown msg type:%d in qnode queue", pMsg->msgType); - terrno = TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; } if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c4e7ff92b7..afd53b8dda 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -643,6 +643,7 @@ typedef struct SSttBlockLoadInfo { STSchema *pSchema; int16_t *colIds; int32_t numOfCols; + bool sttBlockLoaded; } SSttBlockLoadInfo; typedef struct SMergeTree { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 1196b512d3..6dadce80ca 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -53,7 +53,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); if (NULL == pMetaRsp->pSchemas) { - terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -498,7 +498,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); if (rc < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; } uid = *(tb_uid_t *)pData; @@ -743,7 +743,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; } @@ -824,7 +824,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl break; case TSDB_ALTER_TABLE_DROP_COLUMN: if (pColumn == NULL) { - terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; goto _err; } if (pColumn->colId == 0) { @@ -846,7 +846,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: if (pColumn == NULL) { - terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; goto _err; } if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes > pAlterTbReq->colModBytes) { @@ -866,7 +866,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl goto _err; } if (pColumn == NULL) { - terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; goto _err; } if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { @@ -934,7 +934,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; } @@ -990,7 +990,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } if (pColumn == NULL) { - terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS; + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; goto _err; } @@ -1093,7 +1093,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { - terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index b0ae972635..f5ba7b5014 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -197,7 +197,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } if (pTsmaStat->pTSma->indexUid != indexUid) { - terrno = TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid, pTsmaStat->pTSma->indexUid, tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index dbc02363ea..1396b822bf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -333,11 +333,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 code = TSDB_CODE_INVALID_PARA; } - _end: +_end: tsdbDataFReaderClose(&pr->pDataFReaderLast); tsdbDataFReaderClose(&pr->pDataFReader); tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l"); + pr->pDataFReaderLast = NULL; + pr->pDataFReader = NULL; for (int32_t j = 0; j < pr->numOfCols; ++j) { taosMemoryFree(pRes[j]); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 0189ced3c6..48b3e9ff77 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -116,6 +116,13 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI if (info.suid) { metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info); } + if (pMsgIter->sversion != info.skmVer) { + tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, + TD_VID(pTsdb->pVnode), pMsgIter->sversion, info.skmVer, suid, uid); + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto _err; + } + pRsp->sver = info.skmVer; // create/get STbData to op @@ -133,6 +140,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI return code; _err: + terrno = code; return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 745b877f09..0b614fd2db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -72,6 +72,7 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pLoadInfo[i].elapsedTime = 0; pLoadInfo[i].loadBlocks = 0; + pLoadInfo[i].sttBlockLoaded = false; } } @@ -278,9 +279,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t (*pIter)->pBlockLoadInfo = pBlockLoadInfo; - size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - if (size == 0) { +// size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); + pBlockLoadInfo->sttBlockLoaded = true; code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { @@ -288,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t } // only apply to the child tables, ordinary tables will not incur this filter procedure. - size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); if (size >= 1) { SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); @@ -296,10 +298,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t // all identical if (pStart->suid == pEnd->suid) { - if (pStart->suid == suid) { - // do nothing - } else if (pStart->suid != suid) { + if (pStart->suid != suid) { // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + (*pIter)->iSttBlk = -1; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); @@ -330,7 +332,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); } - size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a963b8eb8b..5dbeaa3fc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -138,7 +138,7 @@ typedef struct SReaderStatus { bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not SHashObj* pTableMap; // SHash - STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks. + STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. SUidOrderCheckInfo uidCheckInfo; // check all table in uid order SFileBlockDumpInfo fBlockDumpInfo; SDFileSet* pCurrentFileset; // current opened file set @@ -147,6 +147,12 @@ typedef struct SReaderStatus { SDataBlockIter blockIter; } SReaderStatus; +typedef struct SBlockInfoBuf { + int32_t currentIndex; + SArray* pData; + int32_t numPerBucket; +} SBlockInfoBuf; + struct STsdbReader { STsdb* pTsdb; uint64_t suid; @@ -164,9 +170,9 @@ struct STsdbReader { STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times SDataFReader* pFileReader; SVersionRange verRange; - - int32_t step; - STsdbReader* innerReader[2]; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -232,6 +238,50 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { return TSDB_CODE_SUCCESS; } +static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { + int32_t num = numOfTables / pBuf->numPerBucket; + int32_t remainder = numOfTables % pBuf->numPerBucket; + if (pBuf->pData == NULL) { + pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); + } + + for(int32_t i = 0; i < num; ++i) { + char* p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBuf->pData, &p); + } + + if (remainder > 0) { + char* p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo)); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArrayPush(pBuf->pData, &p); + } + + return TSDB_CODE_SUCCESS; +} + +static void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) { + size_t num = taosArrayGetSize(pBuf->pData); + for(int32_t i = 0; i < num; ++i) { + char** p = taosArrayGet(pBuf->pData, i); + taosMemoryFree(*p); + } + + taosArrayDestroy(pBuf->pData); +} + +static void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { + int32_t bucketIndex = index / pBuf->numPerBucket; + char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); + return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); +} + +// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableKeyInfo* idList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption @@ -242,9 +292,23 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } int64_t st = taosGetTimestampUs(); + initBlockScanInfoBuf(&pTsdbReader->blockInfoBuf, numOfTables); for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; + STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(&pTsdbReader->blockInfoBuf, j); + pScanInfo->uid = idList[j].uid; + if (ASCENDING_TRAVERSE(pTsdbReader->order)) { + int64_t skey = pTsdbReader->window.skey; + pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; + } else { + int64_t ekey = pTsdbReader->window.ekey; + pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + } + + taosHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + +#if 0 +// STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { int64_t skey = pTsdbReader->window.skey; info.lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; @@ -254,7 +318,9 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); - tsdbDebug("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, info.uid, info.lastKey, +#endif + + tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, pScanInfo->lastKey, pTsdbReader->idStr); } @@ -266,18 +332,19 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return pTableMap; } -static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { - STableBlockScanInfo* p = NULL; - +static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { + STableBlockScanInfo** p = NULL; while ((p = taosHashIterate(pTableMap, p)) != NULL) { - p->iterInit = false; - p->iiter.hasVal = false; - if (p->iter.iter != NULL) { - p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**) p; + + pInfo->iterInit = false; + pInfo->iiter.hasVal = false; + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); } - p->delSkyline = taosArrayDestroy(p->delSkyline); - p->lastKey = ts; + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + pInfo->lastKey = ts; } } @@ -298,10 +365,10 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyBlockScanInfo(SHashObj* pTableMap) { - STableBlockScanInfo* p = NULL; +static void destroyAllBlockScanInfo(SHashObj* pTableMap) { + void* p = NULL; while ((p = taosHashIterate(pTableMap, p)) != NULL) { - clearBlockScanInfo(p); + clearBlockScanInfo(*(STableBlockScanInfo**)p); } taosHashCleanup(pTableMap); @@ -506,7 +573,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); - + pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); limitOutputBufferSize(pCond, &pReader->capacity); @@ -577,7 +644,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, continue; } - STableBlockScanInfo* pScanInfo = p; + STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)p; if (pScanInfo->pBlockList == NULL) { pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBlockIndex)); } @@ -597,7 +664,7 @@ _end: } static void cleanupTableScanInfo(SHashObj* pTableMap) { - STableBlockScanInfo* px = NULL; + STableBlockScanInfo** px = NULL; while (1) { px = taosHashIterate(pTableMap, px); if (px == NULL) { @@ -605,8 +672,8 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { } // reset the index in last block when handing a new file - tMapDataClear(&px->mapData); - taosArrayClear(px->pBlockList); + tMapDataClear(&(*px)->mapData); + taosArrayClear((*px)->pBlockList); } } @@ -621,7 +688,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN for (int32_t i = 0; i < numOfTables; ++i) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); - STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); + STableBlockScanInfo* pScanInfo = + *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); tMapDataReset(&pScanInfo->mapData); tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); @@ -1069,14 +1137,14 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo != NULL) { - STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo** pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pScanInfo == NULL) { tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, %s", pBlockInfo->uid, idStr); return TSDB_CODE_INVALID_PARA; } - SBlockIndex* pIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); + SBlockIndex* pIndex = taosArrayGet((*pScanInfo)->pBlockList, pBlockInfo->tbBlockIdx); + tMapDataGetItemByIdx(&(*pScanInfo)->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -1111,7 +1179,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte break; } - STableBlockScanInfo* pTableScanInfo = (STableBlockScanInfo*)ptr; + STableBlockScanInfo* pTableScanInfo = *(STableBlockScanInfo**)ptr; if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { continue; } @@ -1299,10 +1367,10 @@ static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pV (pBlock->minVer <= pVerRange->maxVer); } -static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) { +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t startIndex) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); - for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { + for (int32_t i = startIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { if (p->version >= pBlock->minVer) { @@ -1341,7 +1409,7 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDa // version is not overlap if (ASCENDING_TRAVERSE(order)) { - return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock, pBlockScanInfo->fileDelIndex); } else { int32_t index = pBlockScanInfo->fileDelIndex; while (1) { @@ -1353,7 +1421,7 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDa } } - return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock, index); } } @@ -2118,7 +2186,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { // the last block reader has been initialized for this table. if (pLBlockReader->uid == pScanInfo->uid) { - return true; + return hasDataInLastBlock(pLBlockReader); } if (pLBlockReader->uid != 0) { @@ -2235,7 +2303,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { - pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pBlockScanInfo == NULL) { code = TSDB_CODE_INVALID_PARA; tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, @@ -2251,11 +2319,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (pReader->order == TSDB_ORDER_ASC || (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) { copyBlockDataToSDataBlock(pReader, pBlockScanInfo); + + // record the last key value + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pBlock->maxKey.ts:pBlock->minKey.ts; goto _end; } } } else { // file blocks not exist - pBlockScanInfo = pReader->status.pTableIter; + pBlockScanInfo = *pReader->status.pTableIter; } SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2480,7 +2551,7 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SRea void* p = taosHashIterate(pStatus->pTableMap, NULL); while (p != NULL) { - STableBlockScanInfo* pScanInfo = p; + STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**) p; pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid; p = taosHashIterate(pStatus->pTableMap, p); } @@ -2554,7 +2625,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table - STableBlockScanInfo* pScanInfo = pStatus->pTableIter; + STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**) pStatus->pTableIter; bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); @@ -2592,9 +2663,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; if (pBlockInfo != NULL) { - pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); } else { - pScanInfo = pReader->status.pTableIter; + pScanInfo = *pReader->status.pTableIter; } if (pScanInfo == NULL) { @@ -2642,6 +2713,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts}; setComposedBlockFlag(pReader, false); setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order); + + // update the last key for the corresponding table + pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pInfo->window.ekey:pInfo->window.skey; } } @@ -2659,11 +2733,11 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } } - STableBlockScanInfo* pBlockScanInfo = pStatus->pTableIter; - initMemDataIterator(pBlockScanInfo, pReader); + STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + initMemDataIterator(*pBlockScanInfo, pReader); int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? INT64_MAX : INT64_MIN; - int32_t code = buildDataBlockFromBuf(pReader, pBlockScanInfo, endKey); + int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3465,18 +3539,23 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num) { ASSERT(pReader != NULL); + int32_t size = taosHashGetSize(pReader->status.pTableMap); - STableBlockScanInfo* p = NULL; + STableBlockScanInfo** p = NULL; while ((p = taosHashIterate(pReader->status.pTableMap, p)) != NULL) { - clearBlockScanInfo(p); + clearBlockScanInfo(*p); } + // todo handle the case where size is less than the value of num + ASSERT(size >= num); + taosHashClear(pReader->status.pTableMap); STableKeyInfo* pList = (STableKeyInfo*) pTableList; for(int32_t i = 0; i < num; ++i) { - STableBlockScanInfo info = {.lastKey = 0, .uid = pList[i].uid}; - taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); + STableBlockScanInfo* pInfo = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i); + pInfo->uid = pList[i].uid; + taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); } return TDB_CODE_SUCCESS; @@ -3680,8 +3759,9 @@ void tsdbReaderClose(STsdbReader* pReader) { cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); - destroyBlockScanInfo(pReader->status.pTableMap); + destroyAllBlockScanInfo(pReader->status.pTableMap); blockDataDestroy(pReader->pResBlock); + clearBlockScanInfoBuf(&pReader->blockInfoBuf); if (pReader->pFileReader != NULL) { tsdbDataFReaderClose(&pReader->pFileReader); @@ -3765,7 +3845,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { if (pReader->step == EXTERNAL_ROWS_PREV) { // prepare for the main scan int32_t code = doOpenReaderImpl(pReader); - resetDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey); + resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3782,7 +3862,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { if (pReader->innerReader[1] != NULL && pReader->step == EXTERNAL_ROWS_MAIN) { // prepare for the next row scan int32_t code = doOpenReaderImpl(pReader->innerReader[1]); - resetDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey); + resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3798,7 +3878,7 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { } bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { - STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); if (pBlockScanInfo == NULL) { // no data block for the table of given uid return false; } @@ -3911,7 +3991,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { } SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); - STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pBlockScanInfo == NULL) { terrno = TSDB_CODE_INVALID_PARA; tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, @@ -3947,6 +4027,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { return TSDB_CODE_SUCCESS; } + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + pReader->order = pCond->order; pReader->type = TIMEWINDOW_RANGE_CONTAINED; pReader->status.loadFromFile = true; @@ -3963,13 +4045,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); - resetDataBlockIterator(&pReader->status.blockIter, pReader->order); + resetDataBlockIterator(pBlockIter, pReader->order); int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1; - resetDataBlockScanInfo(pReader->status.pTableMap, ts); + resetAllDataBlockScanInfo(pReader->status.pTableMap, ts); - int32_t code = 0; - SDataBlockIter* pBlockIter = &pReader->status.blockIter; + int32_t code = 0; // no data in files, let's try buffer in memory if (pReader->status.fileIter.numOfFiles == 0) { @@ -4071,7 +4152,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); while (pStatus->pTableIter != NULL) { - STableBlockScanInfo* pBlockScanInfo = pStatus->pTableIter; + STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; STbData* d = NULL; if (pReader->pTsdb->mem != NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f1bbb2d4b3..efedc12d80 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -290,7 +290,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { for (int32_t i = 0; i < msgNum; ++i) { if (offset >= pMsg->contLen) { qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -300,7 +300,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (offset >= pMsg->contLen) { qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -310,7 +310,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (offset >= pMsg->contLen) { qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -320,7 +320,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { if (offset >= pMsg->contLen) { qError("vnode offset %d is bigger than contLen %d", offset, pMsg->contLen); - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_INVALID_MSG_LEN; taosArrayDestroy(batchRsp); return -1; } @@ -362,6 +362,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { offset = 0; if (rspSize > MAX_META_BATCH_RSP_SIZE) { + qError("rspSize:%d overload", rspSize); code = TSDB_CODE_INVALID_MSG_LEN; goto _exit; } @@ -414,9 +415,11 @@ _exit: } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { + SSyncState state = syncGetState(pVnode->sync); + pLoad->vgId = TD_VID(pVnode); - pLoad->syncState = syncGetMyRole(pVnode->sync); - pLoad->syncRestore = pVnode->restored; + pLoad->syncState = state.state; + pLoad->syncRestore = state.restored; pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f1c4ab76d1..536764aa36 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -173,7 +173,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t ret; if (!pVnode->inUse) { - terrno = TSDB_CODE_VND_NOT_SYNCED; + terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL; vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); return -1; } @@ -348,7 +348,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0); default: vError("unknown msg type:%d in query queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + return TSDB_CODE_APP_ERROR; } } @@ -408,7 +408,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + return TSDB_CODE_APP_ERROR; } } @@ -774,7 +774,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq /* code */ ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid); if (ret < 0) { - if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { + if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; } else { dropTbRsp.code = terrno; @@ -863,6 +863,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq SEncoder encoder = {0}; SArray *newTbUids = NULL; SVStatis statis = {0}; + bool tbCreated = false; terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; @@ -896,11 +897,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (pBlock == NULL) break; SSubmitBlkRsp submitBlkRsp = {0}; + tbCreated = false; // create table for auto create table mode if (msgIter.schemaLen > 0) { - submitBlkRsp.hashMeta = 1; - tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { pRsp->code = TSDB_CODE_INVALID_MSG; @@ -937,12 +937,13 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq } taosArrayPush(newTbUids, &createTbReq.uid); + + submitBlkRsp.uid = createTbReq.uid; + submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); + sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name); + tbCreated = true; } - - submitBlkRsp.uid = createTbReq.uid; - submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); - sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name); - + msgIter.uid = createTbReq.uid; if (createTbReq.type == TSDB_CHILD_TABLE) { msgIter.suid = createTbReq.ctb.suid; @@ -955,10 +956,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq #endif tDecoderClear(&decoder); taosArrayDestroy(createTbReq.ctb.tagName); - } else { - submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN); - sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname); - } + } if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { submitBlkRsp.code = terrno; @@ -966,7 +964,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq submitRsp.numOfRows += submitBlkRsp.numOfRows; submitRsp.affectedRows += submitBlkRsp.affectedRows; - taosArrayPush(submitRsp.pArray, &submitBlkRsp); + if (tbCreated || submitBlkRsp.code) { + taosArrayPush(submitRsp.pArray, &submitBlkRsp); + } } if (taosArrayGetSize(newTbUids) > 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f9755bcd12..e27ae07460 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -309,13 +309,13 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const S const STraceId *trace = &pMsg->info.traceId; vGTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", - syncGetVgId(pVnode->sync), pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, - pMeta->code, pMeta->state, syncUtilState2String(pMeta->state), TMSG_INFO(pMsg->msgType)); + pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, rpcMsg.info.conn.applyIndex, pMeta->isWeak, + pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } else { SRpcMsg rsp = {.code = pMeta->code, .info = pMsg->info}; - vError("vgId:%d, commit-cb execute error, type:%s, index:%" PRId64 ", error:0x%x %s", syncGetVgId(pVnode->sync), + vError("vgId:%d, commit-cb execute error, type:%s, index:%" PRId64 ", error:0x%x %s", pVnode->config.vgId, TMSG_INFO(pMsg->msgType), pMeta->index, pMeta->code, tstrerror(pMeta->code)); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); @@ -338,8 +338,8 @@ static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, con static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", - syncGetVgId(pVnode->sync), pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, - syncUtilState2String(pMeta->state), TMSG_INFO(pMsg->msgType)); + pVnode->config.vgId, pFsm, pMeta->index, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), + TMSG_INFO(pMsg->msgType)); } #define USE_TSDB_SNAPSHOT @@ -552,12 +552,21 @@ void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } -bool vnodeIsRoleLeader(SVnode *pVnode) { return syncGetMyRole(pVnode->sync) == TAOS_SYNC_STATE_LEADER; } +bool vnodeIsRoleLeader(SVnode *pVnode) { + SSyncState state = syncGetState(pVnode->sync); + return state.state == TAOS_SYNC_STATE_LEADER; +} bool vnodeIsLeader(SVnode *pVnode) { - if (!syncIsReady(pVnode->sync)) { - vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncGetMyRoleStr(pVnode->sync), - syncRestoreFinish(pVnode->sync)); + SSyncState state = syncGetState(pVnode->sync); + + if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + } else { + terrno = TSDB_CODE_APP_NOT_READY; + } + vDebug("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored); return false; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0786321686..8c699bb59b 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -33,6 +33,7 @@ extern "C" { #define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_DEFAULT_BATCH_NUM 64 #define CTG_DEFAULT_FETCH_NUM 8 +#define CTG_MAX_COMMAND_LEN 512 #define CTG_RENT_SLOT_SECOND 1.5 @@ -223,6 +224,7 @@ typedef struct SCtgUserAuth { typedef struct SCatalog { uint64_t clusterId; + bool stopUpdate; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; @@ -671,7 +673,7 @@ void ctgdShowClusterCache(SCatalog* pCtg); int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); -int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); @@ -786,6 +788,7 @@ void ctgFreeTbCacheImpl(SCtgTbCache* pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); +int32_t ctgdGetOneHandle(SCatalog **pHandle); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e66cdb14ce..1f87066c82 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -202,7 +202,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx int32_t code = 0; STableMetaOutput* output = NULL; - CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, ctx, pTableMeta)); + CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, ctx, pTableMeta)); if (*pTableMeta || (ctx->flag & CTG_FLAG_ONLY_CACHE)) { goto _return; } @@ -959,14 +959,14 @@ int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta)); } -int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)pTableName; ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_ONLY_CACHE; - CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta)); + CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta)); } @@ -981,15 +981,14 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SNam CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta)); } -int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - STableMeta** pTableMeta) { +int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)pTableName; ctx.flag = CTG_FLAG_STB | CTG_FLAG_ONLY_CACHE; - CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta)); + CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta)); } @@ -1114,11 +1113,10 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL)); } -int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - SVgroupInfo* pVgroup, bool* exists) { +int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, exists)); + CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, NULL, pTableName, pVgroup, exists)); } #if 0 @@ -1387,16 +1385,16 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, +int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, bool* exists) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { + if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists)); + CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists)); _return: diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1f1a210699..b601865306 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1204,11 +1204,15 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu stbCtx.flag = flag; stbCtx.pName = &stbName; - taosMemoryFreeClear(pOut->tbMeta); - CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); - if (pOut->tbMeta) { + STableMeta *stbMeta = NULL; + ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta); + if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) { ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; + } else { + ctgDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName)); + taosMemoryFreeClear(pOut->tbMeta); + taosMemoryFreeClear(stbMeta); } } @@ -1641,7 +1645,7 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) { pMsgCtx->pBatchs = pJob->pBatchs; } - CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res)); + CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res)); if (pTask->res) { CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 61da0724c5..51807a145a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -248,6 +248,8 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, goto _return; } + taosHashRelease(dbCache->stbCache, stName); + CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); @@ -1550,7 +1552,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { char *dbFName = msg->dbFName; SCatalog *pCtg = msg->pCtg; - if (NULL == dbInfo->vgHash) { + if (pCtg->stopUpdate || NULL == dbInfo->vgHash) { goto _return; } @@ -1620,6 +1622,10 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { SCtgDropDBMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; + if (pCtg->stopUpdate) { + goto _return; + } + SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { @@ -1646,6 +1652,10 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { SCtgDropDbVgroupMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; + if (pCtg->stopUpdate) { + goto _return; + } + SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { @@ -1675,6 +1685,10 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { STableMetaOutput *pMeta = msg->pMeta; SCtgDBCache *dbCache = NULL; + if (pCtg->stopUpdate) { + goto _return; + } + if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -1723,6 +1737,10 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { SCtgDropStbMetaMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; + if (pCtg->stopUpdate) { + goto _return; + } + SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { @@ -1776,6 +1794,10 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { SCtgDropTblMetaMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; + if (pCtg->stopUpdate) { + goto _return; + } + SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { @@ -1819,6 +1841,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { SCtgUpdateUserMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; + if (pCtg->stopUpdate) { + goto _return; + } + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); if (NULL == pUser) { SCtgUserAuth userAuth = {0}; @@ -1872,8 +1898,12 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateEpsetMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; - SCtgDBCache *dbCache = NULL; + + if (pCtg->stopUpdate) { + goto _return; + } + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not exist, ignore epset update", msg->dbFName); @@ -1920,6 +1950,10 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { STableIndex *pIndex = msg->pIndex; SCtgDBCache *dbCache = NULL; + if (pCtg->stopUpdate) { + goto _return; + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); @@ -1942,6 +1976,10 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; + if (pCtg->stopUpdate) { + goto _return; + } + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { return TSDB_CODE_SUCCESS; @@ -2154,7 +2192,7 @@ int32_t ctgStartUpdateThread() { return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { +int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { if (IS_SYS_DBNAME(ctx->pName->dbname)) { CTG_FLAG_SET_SYS_DB(ctx->flag); } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index dbca2ad977..26b5903bb0 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -226,28 +226,45 @@ _return: CTG_RET(code); } -int32_t ctgdEnableDebug(char *option) { +int32_t ctgdEnableDebug(char *option, bool enable) { if (0 == strcasecmp(option, "lock")) { - gCTGDebug.lockEnable = true; - qDebug("lock debug enabled"); + gCTGDebug.lockEnable = enable; + qDebug("catalog lock debug set to %d", enable); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "cache")) { - gCTGDebug.cacheEnable = true; - qDebug("cache debug enabled"); + gCTGDebug.cacheEnable = enable; + qDebug("catalog cache debug set to %d", enable); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "api")) { - gCTGDebug.apiEnable = true; - qDebug("api debug enabled"); + gCTGDebug.apiEnable = enable; + qDebug("catalog api debug set to %d", enable); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "meta")) { - gCTGDebug.metaEnable = true; - qDebug("api debug enabled"); + gCTGDebug.metaEnable = enable; + qDebug("catalog meta debug set to %d", enable); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "stopUpdate")) { + SCatalog *pCtg = NULL; + + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + pCtg->stopUpdate = enable; + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + qDebug("catalog stopUpdate set to %d", enable); + return TSDB_CODE_SUCCESS; } @@ -256,6 +273,77 @@ int32_t ctgdEnableDebug(char *option) { return TSDB_CODE_CTG_INTERNAL_ERROR; } +int32_t ctgdHandleDbgCommand(char *command) { + if (NULL == command) { + CTG_RET(TSDB_CODE_INVALID_PARA); + } + + if (strlen(command) > CTG_MAX_COMMAND_LEN) { + CTG_RET(TSDB_CODE_INVALID_PARA); + } + + char *dup = strdup(command); + char *option = NULL; + char *param = NULL; + + int32_t i = 0; + bool newItem = true; + while (*(dup + i)) { + if (isspace(*(dup + i))) { + *(dup + i) = 0; + ++i; + newItem = true; + continue; + } + + if (!newItem) { + ++i; + continue; + } + + newItem = false; + if (NULL == option) { + option = dup + i; + ++i; + continue; + } + + if (NULL == param) { + param = dup + i; + ++i; + continue; + } + + taosMemoryFree(dup); + CTG_RET(TSDB_CODE_INVALID_PARA); + } + + bool enable = atoi(param); + + int32_t code = ctgdEnableDebug(option, enable); + + taosMemoryFree(dup); + + CTG_RET(code); +} + +int32_t ctgdGetOneHandle(SCatalog **pHandle) { + SCatalog *pCtg = NULL; + + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog **)pIter; + + taosHashCancelIterate(gCtgMgmt.pCluster, pIter); + break; + } + + *pHandle = pCtg; + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgdGetStatNum(char *option, void *res) { if (0 == strcasecmp(option, "runtime.numOfOpDequeue")) { *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.numOfOpDequeue); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 489d174e17..ebf7c7baeb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -41,7 +41,6 @@ namespace { extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type); -extern "C" int32_t ctgdEnableDebug(char *option); extern "C" int32_t ctgdGetStatNum(char *option, void *res); void ctgTestSetRspTableMeta(); @@ -49,6 +48,8 @@ void ctgTestSetRspCTableMeta(); void ctgTestSetRspSTableMeta(); void ctgTestSetRspMultiSTableMeta(); +extern int32_t clientConnRefPool; + enum { CTGT_RSP_VGINFO = 1, CTGT_RSP_TBMETA, @@ -151,10 +152,10 @@ void ctgTestInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); - ctgdEnableDebug("api"); - ctgdEnableDebug("meta"); - ctgdEnableDebug("cache"); - ctgdEnableDebug("lock"); + ctgdEnableDebug("api", true); + ctgdEnableDebug("meta", true); + ctgdEnableDebug("cache", true); + ctgdEnableDebug("lock", true); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -1204,6 +1205,34 @@ void *ctgTestSetCtableMetaThread(void *param) { } +void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { + TAOS_ROW row; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + (*rows)++; + memset(temp, 0, sizeof(temp)); + taos_print_row(temp, row, fields, num_fields); + printf("\t[%s]\n", temp); + } +} + +void ctgTestExecQuery(TAOS * taos, char* sql, bool fetch, int32_t *rows) { + TAOS_RES *result = taos_query(taos, sql); + int code = taos_errno(result); + ASSERT_EQ(code, 0); + + if (fetch) { + ctgTestFetchRows(result, rows); + } + + taos_free_result(result); +} + + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; SVgroupInfo vgInfo = {0}; @@ -1245,7 +1274,7 @@ TEST(tableMeta, normalTable) { memset(&vgInfo, 0, sizeof(vgInfo)); bool exists = false; - code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists); + code = catalogGetCachedTableHashVgroup(pCtg, &n, &vgInfo, &exists); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.epSet.numOfEps, 3); @@ -1292,7 +1321,7 @@ TEST(tableMeta, normalTable) { taosMemoryFree(tableMeta); tableMeta = NULL; - catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta); + catalogGetCachedTableMeta(pCtg, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); @@ -1500,7 +1529,7 @@ TEST(tableMeta, superTableCase) { } tableMeta = NULL; - code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta); + code = catalogGetCachedSTableMeta(pCtg, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 0); ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE); @@ -2772,7 +2801,7 @@ TEST(apiTest, catalogChkAuth_test) { bool pass = false; bool exists = false; - code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); ASSERT_EQ(code, 0); ASSERT_EQ(exists, false); @@ -2790,7 +2819,7 @@ TEST(apiTest, catalogChkAuth_test) { } } - code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); ASSERT_EQ(code, 0); ASSERT_EQ(pass, true); ASSERT_EQ(exists, true); @@ -3063,6 +3092,58 @@ TEST(apiTest, catalogGetDnodeList_test) { catalogDestroy(); } +#ifdef INTEGRATION_TEST +TEST(intTest, autoCreateTableTest) { + struct SCatalog *pCtg = NULL; + + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_TRUE(NULL != taos); + + ctgdEnableDebug("api", true); + ctgdEnableDebug("meta", true); + ctgdEnableDebug("cache", true); + ctgdEnableDebug("lock", true); + + ctgTestExecQuery(taos, "drop database if exists db1", false, NULL); + ctgTestExecQuery(taos, "create database db1", false, NULL); + ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL); + ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL); + + ctgdGetOneHandle(&pCtg); + + while (true) { + uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); + if (2 != n) { + taosMsleep(50); + } else { + break; + } + } + + uint64_t n = 0, m = 0; + ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + + ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 2)", false, NULL); + + ctgTestExecQuery(taos, "insert into db1.tb1 values(now, 3)", false, NULL); + + taosMsleep(1000); + ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&m); + + ASSERT_EQ(n, m); + + ctgdEnableDebug("stopUpdate", true); + ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL); + + ctgdEnableDebug("stopUpdate", false); + + ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL); + + taos_close(taos); +} + +#endif + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index ce4c4cf09b..4d0c5389e1 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -96,6 +96,7 @@ extern "C" { #define COMMAND_RESET_LOG "resetLog" #define COMMAND_SCHEDULE_POLICY "schedulePolicy" #define COMMAND_ENABLE_RESCHEDULE "enableReSchedule" +#define COMMAND_CATALOG_DEBUG "catalogDebug" typedef struct SExplainGroup { int32_t nodeNum; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 47a904bba2..76c84b3be9 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -571,6 +571,8 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { code = schedulerUpdatePolicy(atoi(value)); } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) { code = schedulerEnableReSchedule(atoi(value)); + } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) { + code = ctgdHandleDbgCommand(value); } else { goto _return; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 32386a72fd..567ced5b90 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -30,10 +30,10 @@ // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups struct STableListInfo { bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid + SHashObj* map; // speedup acquire the tableQueryInfo by table uid uint64_t suid; }; @@ -925,6 +925,15 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* SArray* pTbList = getTableNameList(pList); int32_t numOfTables = taosArrayGetSize(pTbList); + SHashObj *uHash = NULL; + size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids + if (listlen > 0) { + uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + for (int i = 0; i < listlen; i++) { + int64_t *uid = taosArrayGet(list, i); + taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); + } + } for (int i = 0; i < numOfTables; i++) { char* name = taosArrayGetP(pTbList, i); @@ -933,9 +942,12 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { ETableType tbType = TSDB_TABLE_MAX; if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { - taosArrayPush(list, &uid); + if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { + taosArrayPush(list, &uid); + } } else { taosArrayDestroy(pTbList); + taosHashCleanup(uHash); return -1; } } else { @@ -944,6 +956,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* } } + taosHashCleanup(uHash); taosArrayDestroy(pTbList); return 0; } @@ -1678,9 +1691,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { - return pTableList->suid; -} +uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; } STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -1718,7 +1729,7 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t } int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, - int32_t* size) { + int32_t* size) { int32_t total = tableListGetOutputGroups(pTableList); if (ordinalGroupIndex < 0 || ordinalGroupIndex >= total) { return TSDB_CODE_INVALID_PARA; @@ -1728,7 +1739,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalG // 1. only one group exists, and 2. one table exists for each group. if (total == 1) { *size = tableListGetSize(pTableList); - *pKeyInfo = (*size == 0)? NULL:taosArrayGet(pTableList->pTableList, 0); + *pKeyInfo = (*size == 0) ? NULL : taosArrayGet(pTableList->pTableList, 0); return TSDB_CODE_SUCCESS; } else if (total == tableListGetSize(pTableList)) { *size = 1; @@ -1806,13 +1817,13 @@ void tableListClear(STableListInfo* pTableListInfo) { } static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) { - STableKeyInfo* pInfo1 = (STableKeyInfo*) p1; - STableKeyInfo* pInfo2 = (STableKeyInfo*) p2; + STableKeyInfo* pInfo1 = (STableKeyInfo*)p1; + STableKeyInfo* pInfo2 = (STableKeyInfo*)p2; if (pInfo1->groupId == pInfo2->groupId) { return 0; } else { - return pInfo1->groupId < pInfo2->groupId? -1:1; + return pInfo1->groupId < pInfo2->groupId ? -1 : 1; } } @@ -1825,12 +1836,12 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { SArray* pList = taosArrayInit(4, sizeof(int32_t)); STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0); - uint64_t gid = pInfo->groupId; + uint64_t gid = pInfo->groupId; int32_t start = 0; taosArrayPush(pList, &start); - for(int32_t i = 1; i < size; ++i) { + for (int32_t i = 1; i < size; ++i) { pInfo = taosArrayGet(pTableListInfo->pTableList, i); if (pInfo->groupId != gid) { taosArrayPush(pList, &i); @@ -1845,16 +1856,17 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { return TDB_CODE_SUCCESS; } -int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) { +int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, + bool groupSort) { int32_t code = TSDB_CODE_SUCCESS; ASSERT(pTableListInfo->map != NULL); - bool groupByTbname = groupbyTbname(group); + bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); if (group == NULL || groupByTbname) { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - info->groupId = groupByTbname? info->uid:0; + info->groupId = groupByTbname ? info->uid : 0; } pTableListInfo->oneTableForEachGroup = groupByTbname; @@ -1878,7 +1890,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* // add all table entry in the hash map size_t size = taosArrayGetSize(pTableListInfo->pTableList); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { STableKeyInfo* p = taosArrayGet(pTableListInfo->pTableList, i); taosHashPut(pTableListInfo->map, &p->uid, sizeof(uint64_t), &i, sizeof(int32_t)); } @@ -1889,7 +1901,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, struct SExecTaskInfo* pTaskInfo) { - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); const char* idStr = GET_TASKID(pTaskInfo); if (pHandle == NULL) { @@ -1919,7 +1931,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } - pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1)/1000.0; + pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1) / 1000.0; qDebug("generate group id map completed, elapsed time:%.2f ms %s", pTaskInfo->cost.groupIdMapTime, idStr); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2964878a2c..1cf01c8661 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -321,7 +321,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S } int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (isAdd) { qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); @@ -473,7 +473,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); - _error: +_error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -1027,10 +1027,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pTableScanInfo->dataReader == NULL) { STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num, - &pTableScanInfo->dataReader, NULL) < 0 || pTableScanInfo->dataReader == NULL) { + &pTableScanInfo->dataReader, NULL) < 0 || + pTableScanInfo->dataReader == NULL) { ASSERT(0); } } @@ -1071,14 +1072,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; - if (pTaskInfo->pTableInfoList == NULL) { + if (pTaskInfo->pTableInfoList == NULL) { pTaskInfo->pTableInfoList = tableListCreate(); } tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0); STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d8340b72f2..98f8d57fc6 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -621,9 +621,10 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { } static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { - void* ite = NULL; - while ((ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL) { - taosArrayDestroy(((SDataGroupInfo*)ite)->pPageList); + int32_t size = taosArrayGetSize(pInfo->sortedGroupArray); + for (int32_t i = 0; i < size; i++) { + SDataGroupInfo* pGp = taosArrayGet(pInfo->sortedGroupArray, i); + taosArrayDestroy(pGp->pPageList); } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d596eab2f8..3352b2685a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1062,15 +1062,18 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); if (code != TSDB_CODE_SUCCESS) { - return NULL; + goto _error; } STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; + } } pInfo->readHandle = *readHandle; @@ -1164,6 +1167,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; + T_LONG_JMP(pTaskInfo->env, code); return NULL; } @@ -2426,8 +2430,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->dataReader = NULL; - if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL); + if (code != 0) { + terrno = code; destroyTableScanOperatorInfo(pTableScanOp); goto _error; } @@ -4292,131 +4297,6 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* return TSDB_CODE_SUCCESS; } -int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, - STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx, - STsdbReader** ppReader, const char* idstr) { - STsdbReader* pReader = NULL; - void* pStart = tableListGetInfo(pTableListInfo, tableStartIdx); - int32_t num = tableEndIdx - tableStartIdx + 1; - - int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr); - if (code != 0) { - return code; - } - - *ppReader = pReader; - return TSDB_CODE_SUCCESS; -} - -static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, - SSDataBlock* pBlock, uint32_t* status) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableMergeScanInfo* pInfo = pOperator->info; - - uint64_t uid = pBlock->info.uid; - - SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder; - - pCost->totalBlocks += 1; - pCost->totalRows += pBlock->info.rows; - - *status = pInfo->dataBlockLoadFlag; - if (pTableScanInfo->pFilterNode != NULL || - overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) { - (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; - taosMemoryFreeClear(pBlock->pBlockAgg); - - if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->filterOutBlocks += 1; - return TSDB_CODE_SUCCESS; - } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - - // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { - SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); - pcol->pData = NULL; - } - - return TSDB_CODE_SUCCESS; - } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { - pCost->loadBlockStatis += 1; - - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - STsdbReader* reader = pTableScanInfo->pReader; - tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg); - - if (allColumnsHaveAgg == true) { - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - - // todo create this buffer during creating operator - if (pBlock->pBlockAgg == NULL) { - pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); - if (!pColMatchInfo->needOutput) { - continue; - } - pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; - } - - return TSDB_CODE_SUCCESS; - } else { // failed to load the block sma data, data block statistics does not exist, load data block instead - *status = FUNC_DATA_REQUIRED_DATA_LOAD; - } - } - - ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); - - pCost->totalCheckedRows += pBlock->info.rows; - pCost->loadBlocks += 1; - - STsdbReader* reader = pTableScanInfo->pReader; - SArray* pCols = tsdbRetrieveDataBlock(reader, NULL); - if (pCols == NULL) { - return terrno; - } - - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - int32_t code = - addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, - pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } - - if (pTableScanInfo->pFilterNode != NULL) { - int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL); - - double el = (taosGetTimestampUs() - st) / 1000.0; - pTableScanInfo->readRecorder.filterTime += el; - - if (pBlock->info.rows == 0) { - pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms", - GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el); - } else { - qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el); - } - } - return TSDB_CODE_SUCCESS; -} - // todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { @@ -4543,7 +4423,7 @@ typedef struct STableMergeScanSortSourceParam { SSDataBlock* inputBlock; } STableMergeScanSortSourceParam; -static SSDataBlock* getTableDataBlockTemp(void* param) { +static SSDataBlock* getTableDataBlockImpl(void* param) { STableMergeScanSortSourceParam* source = param; SOperatorInfo* pOperator = source->pOperator; STableMergeScanInfo* pInfo = pOperator->info; @@ -4560,7 +4440,11 @@ static SSDataBlock* getTableDataBlockTemp(void* param) { void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->readHandle; - tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo)); + + int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo)); + if (code != 0) { + T_LONG_JMP(pOperator->pTaskInfo->env, code); + } STsdbReader* reader = pInfo->pReader; while (tsdbNextDataBlock(reader)) { @@ -4611,55 +4495,6 @@ static SSDataBlock* getTableDataBlockTemp(void* param) { pInfo->pReader = NULL; return NULL; } -static SSDataBlock* getTableDataBlock2(void* param) { - STableMergeScanSortSourceParam* source = param; - SOperatorInfo* pOperator = source->pOperator; - int64_t uid = source->uid; - SSDataBlock* pBlock = source->inputBlock; - STableMergeScanInfo* pTableScanInfo = pOperator->info; - - int64_t st = taosGetTimestampUs(); - - blockDataCleanup(pBlock); - - STsdbReader* reader = pTableScanInfo->pReader; - while (tsdbTableNextDataBlock(reader, uid)) { - if (isTaskKilled(pOperator->pTaskInfo)) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - // process this data block based on the probabilities - bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); - if (!processThisBlock) { - continue; - } - - blockDataCleanup(pBlock); - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - - uint32_t status = 0; - int32_t code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pOperator->pTaskInfo->env, code); - } - - // current block is filter out according to filter condition, continue load the next block - if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { - continue; - } - - pBlock->info.groupId = getTableGroupId(pOperator->pTaskInfo->pTableInfoList, pBlock->info.uid); - pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; - pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; - - return pBlock; - } - return NULL; -} static SSDataBlock* getTableDataBlock(void* param) { STableMergeScanSortSourceParam* source = param; @@ -4769,7 +4604,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); - tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL); + tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL); // one table has one data block int32_t numOfTable = tableEndIdx - tableStartIdx + 1; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 35ad10ffbf..c8f0b3d826 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2307,6 +2307,12 @@ int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { double param00 = param[0][0] - param[1][0] * (param[0][1] / param[1][1]); double param02 = param[0][2] - param[1][2] * (param[0][1] / param[1][1]); + + if (0 == param00) { + colDataAppendNULL(pCol, currentRow); + return 0; + } + // param[0][1] = 0; double param12 = param[1][2] - param02 * (param[1][0] / param00); // param[1][0] = 0; @@ -2707,7 +2713,9 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { taosMemoryFree(res); } else { // no need to free // setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); - return TSDB_CODE_SUCCESS; + // return TSDB_CODE_SUCCESS; + qDebug("%s get the final res, elements:%" PRId64 ", numOfEntry:%d. result is null", __FUNCTION__, + pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries); } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 459ee583a4..1dedfe8364 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -122,15 +122,18 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { taosGetCpuCores(&numCpuCores); snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2); - char pathTaosdLdLib[512] = {0}; + char pathTaosdLdLib[512] = {0}; size_t taosdLdLibPathLen = sizeof(pathTaosdLdLib); - uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen); + int ret = uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen); + if (ret != UV_ENOBUFS) { + taosdLdLibPathLen = strlen(pathTaosdLdLib); + } - char udfdPathLdLib[1024] = {0}; + char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); udfdPathLdLib[udfdLdLibPathLen] = ':'; - strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen); + strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { fnInfo("udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); } else { @@ -359,7 +362,7 @@ typedef struct SUdfcProxy { SArray *udfStubs; // SUdfcFuncStub uv_mutex_t udfcUvMutex; - int8_t initialized; + int8_t initialized; } SUdfcProxy; SUdfcProxy gUdfcProxy = {0}; diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 33960ad8c2..5538584754 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -152,7 +152,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int if (ctx->type == TFILE) { // ugly code, refactor later ctx->file.readOnly = readOnly; - memcpy(ctx->file.buf, path, sizeof(ctx->file.buf)); + memcpy(ctx->file.buf, path, strlen(path)); if (readOnly == false) { ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); taosFtruncateFile(ctx->file.pFile, 0); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d1d0680e65..47de2528fa 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -280,7 +280,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa } atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); - SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); + SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); SCH_LOCK(SCH_WRITE, &pJob->resLock); if (pJob->execRes.res) { diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 55ce1470ce..cf4e3309f1 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -52,6 +52,7 @@ typedef struct SSyncEnv { } SSyncEnv; SSyncEnv* syncEnv(); +bool syncIsInit(); int64_t syncNodeAdd(SSyncNode* pNode); void syncNodeRemove(int64_t rid); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 63aa6d81c9..6ec29d69f5 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -22,9 +22,9 @@ extern "C" { #include "sync.h" #include "syncTools.h" -#include "tlog.h" -#include "ttimer.h" #include "taosdef.h" +#include "tlog.h" +#include "trpc.h" #include "ttimer.h" // clang-format off @@ -314,6 +314,7 @@ int32_t syncNodePeerStateInit(SSyncNode* pSyncNode); // trace log void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s); +void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s); void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); @@ -336,7 +337,17 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s); void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s); -void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s); +void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s); +void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s); + +void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s); +void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s); + +void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s); +void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s); + +void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s); +void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index b8b7af2dda..760fc8ac73 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -28,10 +28,11 @@ extern "C" { #include "syncMessage.h" #include "taosdef.h" -#define SYNC_SNAPSHOT_SEQ_INVALID -1 -#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -2 -#define SYNC_SNAPSHOT_SEQ_BEGIN 0 -#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF +#define SYNC_SNAPSHOT_SEQ_INVALID -2 +#define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 +#define SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT -1 +#define SYNC_SNAPSHOT_SEQ_BEGIN 0 +#define SYNC_SNAPSHOT_SEQ_END 0x7FFFFFFF #define SYNC_SNAPSHOT_RETRY_MS 5000 @@ -47,19 +48,19 @@ typedef struct SSyncSnapshotSender { SSnapshot snapshot; SSyncCfg lastConfig; int64_t sendingMS; - SSyncNode *pSyncNode; - int32_t replicaIndex; SyncTerm term; - SyncTerm privateTerm; int64_t startTime; bool finish; + + // init when create + SSyncNode *pSyncNode; + int32_t replicaIndex; } SSyncSnapshotSender; SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); -int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot, - void *pReader); +int32_t snapshotSenderStart(SSyncSnapshotSender *pSender); int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); @@ -76,11 +77,13 @@ typedef struct SSyncSnapshotReceiver { int32_t ack; void *pWriter; SyncTerm term; - SyncTerm privateTerm; SSnapshotParam snapshotParam; SSnapshot snapshot; SRaftId fromId; - SSyncNode *pSyncNode; + int64_t startTime; + + // init when create + SSyncNode *pSyncNode; } SSyncSnapshotReceiver; diff --git a/include/libs/sync/syncTools.h b/source/libs/sync/inc/syncTools.h similarity index 91% rename from include/libs/sync/syncTools.h rename to source/libs/sync/inc/syncTools.h index 9586d1febb..b48519a5b0 100644 --- a/include/libs/sync/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -20,15 +20,13 @@ extern "C" { #endif -#include "trpc.h" - // ------------------ ds ------------------- typedef struct SRaftId { SyncNodeId addr; SyncGroupId vgId; } SRaftId; -char* sync2SimpleStr(int64_t rid); +char* sync2SimpleStr(int64_t rid); // for compatibility, the same as syncPropose int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak); @@ -498,6 +496,69 @@ void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg); void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg); void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg); +// --------------------------------------------- +typedef struct SyncPreSnapshot { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + +} SyncPreSnapshot; + +SyncPreSnapshot* syncPreSnapshotBuild(int32_t vgId); +void syncPreSnapshotDestroy(SyncPreSnapshot* pMsg); +void syncPreSnapshotSerialize(const SyncPreSnapshot* pMsg, char* buf, uint32_t bufLen); +void syncPreSnapshotDeserialize(const char* buf, uint32_t len, SyncPreSnapshot* pMsg); +char* syncPreSnapshotSerialize2(const SyncPreSnapshot* pMsg, uint32_t* len); +SyncPreSnapshot* syncPreSnapshotDeserialize2(const char* buf, uint32_t len); +void syncPreSnapshot2RpcMsg(const SyncPreSnapshot* pMsg, SRpcMsg* pRpcMsg); +void syncPreSnapshotFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshot* pMsg); +SyncPreSnapshot* syncPreSnapshotFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncPreSnapshot2Json(const SyncPreSnapshot* pMsg); +char* syncPreSnapshot2Str(const SyncPreSnapshot* pMsg); + +// for debug ---------------------- +void syncPreSnapshotPrint(const SyncPreSnapshot* pMsg); +void syncPreSnapshotPrint2(char* s, const SyncPreSnapshot* pMsg); +void syncPreSnapshotLog(const SyncPreSnapshot* pMsg); +void syncPreSnapshotLog2(char* s, const SyncPreSnapshot* pMsg); + +// --------------------------------------------- +typedef struct SyncPreSnapshotReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncIndex snapStart; + +} SyncPreSnapshotReply; + +SyncPreSnapshotReply* syncPreSnapshotReplyBuild(int32_t vgId); +void syncPreSnapshotReplyDestroy(SyncPreSnapshotReply* pMsg); +void syncPreSnapshotReplySerialize(const SyncPreSnapshotReply* pMsg, char* buf, uint32_t bufLen); +void syncPreSnapshotReplyDeserialize(const char* buf, uint32_t len, SyncPreSnapshotReply* pMsg); +char* syncPreSnapshotReplySerialize2(const SyncPreSnapshotReply* pMsg, uint32_t* len); +SyncPreSnapshotReply* syncPreSnapshotReplyDeserialize2(const char* buf, uint32_t len); +void syncPreSnapshotReply2RpcMsg(const SyncPreSnapshotReply* pMsg, SRpcMsg* pRpcMsg); +void syncPreSnapshotReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshotReply* pMsg); +SyncPreSnapshotReply* syncPreSnapshotReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncPreSnapshotReply2Json(const SyncPreSnapshotReply* pMsg); +char* syncPreSnapshotReply2Str(const SyncPreSnapshotReply* pMsg); + +// for debug ---------------------- +void syncPreSnapshotReplyPrint(const SyncPreSnapshotReply* pMsg); +void syncPreSnapshotReplyPrint2(char* s, const SyncPreSnapshotReply* pMsg); +void syncPreSnapshotReplyLog(const SyncPreSnapshotReply* pMsg); +void syncPreSnapshotReplyLog2(char* s, const SyncPreSnapshotReply* pMsg); + // --------------------------------------------- typedef struct SyncApplyMsg { uint32_t bytes; @@ -543,7 +604,7 @@ typedef struct SyncSnapshotSend { SyncTerm lastTerm; // snapshot.lastTerm SyncIndex lastConfigIndex; // snapshot.lastConfigIndex SSyncCfg lastConfig; - SyncTerm privateTerm; + int64_t startTime; int32_t seq; uint32_t dataLen; char data[]; @@ -578,9 +639,10 @@ typedef struct SyncSnapshotRsp { SyncTerm term; SyncIndex lastIndex; SyncTerm lastTerm; - SyncTerm privateTerm; + int64_t startTime; int32_t ack; int32_t code; + SyncIndex snapBeginIndex; // when ack = SYNC_SNAPSHOT_SEQ_BEGIN, it's valid } SyncSnapshotRsp; SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId); @@ -711,6 +773,9 @@ int32_t syncNodeOnRequestVoteReply(SSyncNode* ths, SyncRequestVoteReply* pMsg); int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg); int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg); +int32_t syncNodeOnPreSnapshot(SSyncNode* ths, SyncPreSnapshot* pMsg); +int32_t syncNodeOnPreSnapshotReply(SSyncNode* ths, SyncPreSnapshotReply* pMsg); + int32_t syncNodeOnSnapshot(SSyncNode* ths, SyncSnapshotSend* pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 96e22720e8..ac8dd0928c 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -49,7 +49,7 @@ int32_t syncUtilQuorum(int32_t replicaNum); cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); char* syncUtilRaftId2Str(const SRaftId* p); -const char* syncUtilState2String(ESyncState state); +const char* syncStr(ESyncState state); bool syncUtilCanPrint(char c); char* syncUtilprintBin(char* ptr, uint32_t len); char* syncUtilprintBin2(char* ptr, uint32_t len); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7dab112a51..4638475e71 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -142,7 +142,6 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { // pReply->matchIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); pReply->matchIndex = SYNC_INDEX_INVALID; pReply->lastSendIndex = pMsg->prevLogIndex + 1; - pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->startTime = ths->startTime; if (pMsg->term < ths->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 5e6c9f1534..5d54db5b07 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -73,6 +73,7 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync int32_t code = ths->pFsm->FpSnapshotStartRead(ths->pFsm, &readerParam, &pReader); ASSERT(code == 0); +#if 0 if (pMsg->privateTerm < pSender->privateTerm) { ASSERT(pReader != NULL); snapshotSenderStart(pSender, readerParam, snapshot, pReader); @@ -82,6 +83,7 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync ths->pFsm->FpSnapshotStopRead(ths->pFsm, pReader); } } +#endif } int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3089e9b39b..eb78ea2894 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -505,50 +505,20 @@ int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak) { return ret; } -ESyncState syncGetMyRole(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return TAOS_SYNC_STATE_ERROR; - } - ASSERT(rid == pSyncNode->rid); - ESyncState state = pSyncNode->state; +SSyncState syncGetState(int64_t rid) { + SSyncState state = {.state = TAOS_SYNC_STATE_ERROR}; + + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { + state.state = pSyncNode->state; + state.restored = pSyncNode->restoreFinish; + syncNodeRelease(pSyncNode); + } - syncNodeRelease(pSyncNode); return state; } -bool syncIsReady(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return false; - } - ASSERT(rid == pSyncNode->rid); - bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish; - syncNodeRelease(pSyncNode); - - // if false, set error code - if (false == b) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } - } - return b; -} - -bool syncIsRestoreFinish(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return false; - } - ASSERT(rid == pSyncNode->rid); - bool b = pSyncNode->restoreFinish; - - syncNodeRelease(pSyncNode); - return b; -} - +#if 0 int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot) { if (index < SYNC_INDEX_BEGIN) { return -1; @@ -618,6 +588,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct syncNodeRelease(pSyncNode); return 0; } +#endif SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex) { ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1); @@ -635,23 +606,7 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho return lastIndex; } -const char* syncGetMyRoleStr(int64_t rid) { - const char* s = syncUtilState2String(syncGetMyRole(rid)); - return s; -} - -bool syncRestoreFinish(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return false; - } - ASSERT(rid == pSyncNode->rid); - bool restoreFinish = pSyncNode->restoreFinish; - - syncNodeRelease(pSyncNode); - return restoreFinish; -} - +#if 0 SyncTerm syncGetMyTerm(int64_t rid) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { @@ -719,6 +674,7 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { syncNodeRelease(pSyncNode); } +#endif void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { SSyncNode* pSyncNode = syncNodeAcquire(rid); @@ -742,7 +698,6 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { syncNodeRelease(pSyncNode); } - static void syncGetAndDelRespRpc(SSyncNode* pSyncNode, uint64_t index, SRpcHandleInfo* pInfo) { SRespStub stub; int32_t ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub); @@ -877,7 +832,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } else { ret = -1; terrno = TSDB_CODE_SYN_NOT_LEADER; - sError("vgId:%d, sync propose not leader, %s, type:%s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + sError("vgId:%d, sync propose not leader, %s, type:%s", pSyncNode->vgId, syncStr(pSyncNode->state), TMSG_INFO(pMsg->msgType)); goto _END; } @@ -1603,7 +1558,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ server vars cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); - cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state)); + cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); // tla+ candidate vars @@ -1743,7 +1698,7 @@ inline void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { "stgy:%d, bch:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->vgId, syncStr(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, pSyncNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1767,7 +1722,7 @@ inline void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { "stgy:%d, bch:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->vgId, syncStr(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, pSyncNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1821,7 +1776,7 @@ inline void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { "stgy:%d, bch:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->vgId, syncStr(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, pSyncNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1843,7 +1798,7 @@ inline void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { "stgy:%d, bch:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->vgId, syncStr(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, pSyncNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1875,9 +1830,9 @@ inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { ", sby:%d, " "r-num:%d, " "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, - pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); return s; } @@ -2157,7 +2112,13 @@ void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { } void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm) { - ASSERT(pSyncNode->pRaftStore->currentTerm <= newTerm); + if (pSyncNode->pRaftStore->currentTerm > newTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "step down, ignore, new-term:%" PRIu64 ", current-term:%" PRIu64, newTerm, + pSyncNode->pRaftStore->currentTerm); + syncNodeEventLog(pSyncNode, logBuf); + return; + } do { char logBuf[128]; @@ -2275,6 +2236,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // init peer mgr syncNodePeerStateInit(pSyncNode); +#if 0 // update sender private term SSyncSnapshotSender* pMySender = syncNodeGetSnapshotSender(pSyncNode, &(pSyncNode->myRaftId)); if (pMySender != NULL) { @@ -2285,6 +2247,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { } (pMySender->privateTerm) += 100; } +#endif // close receiver if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { @@ -3600,3 +3563,52 @@ void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const c syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, s); syncNodeEventLog(pSyncNode, logBuf); } + +void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "send sync-pre-snapshot to %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "recv sync-pre-snapshot from %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, + s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-pre-snapshot-reply to %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port, + pMsg->term, pMsg->snapStart, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-pre-snapshot-reply from %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port, + pMsg->term, pMsg->snapStart, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {} + +void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {} + +void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {} + +void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {} diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index f9609d9c39..91e8ec91b7 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -2315,6 +2315,303 @@ void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) { } } +// ---- message process SyncPreSnapshot---- +SyncPreSnapshot* syncPreSnapshotBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncPreSnapshot); + SyncPreSnapshot* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT; + return pMsg; +} + +void syncPreSnapshotDestroy(SyncPreSnapshot* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncPreSnapshotSerialize(const SyncPreSnapshot* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncPreSnapshotDeserialize(const char* buf, uint32_t len, SyncPreSnapshot* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncPreSnapshotSerialize2(const SyncPreSnapshot* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncPreSnapshotSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncPreSnapshot* syncPreSnapshotDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncPreSnapshot* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncPreSnapshotDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncPreSnapshot2RpcMsg(const SyncPreSnapshot* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncPreSnapshotSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncPreSnapshotFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshot* pMsg) { + syncPreSnapshotDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncPreSnapshot* syncPreSnapshotFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncPreSnapshot* pMsg = syncPreSnapshotDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncPreSnapshot2Json(const SyncPreSnapshot* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncPreSnapshot", pRoot); + return pJson; +} + +char* syncPreSnapshot2Str(const SyncPreSnapshot* pMsg) { + cJSON* pJson = syncPreSnapshot2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncPreSnapshotPrint(const SyncPreSnapshot* pMsg) { + char* serialized = syncPreSnapshot2Str(pMsg); + printf("syncPreSnapshotPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncPreSnapshotPrint2(char* s, const SyncPreSnapshot* pMsg) { + char* serialized = syncPreSnapshot2Str(pMsg); + printf("syncPreSnapshotPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncPreSnapshotLog(const SyncPreSnapshot* pMsg) { + char* serialized = syncPreSnapshot2Str(pMsg); + sTrace("syncPreSnapshotLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncPreSnapshotLog2(char* s, const SyncPreSnapshot* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncPreSnapshot2Str(pMsg); + sTrace("syncPreSnapshotLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// ---- message process SyncPreSnapshotReply---- +SyncPreSnapshotReply* syncPreSnapshotReplyBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncPreSnapshotReply); + SyncPreSnapshotReply* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_PRE_SNAPSHOT_REPLY; + return pMsg; +} + +void syncPreSnapshotReplyDestroy(SyncPreSnapshotReply* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncPreSnapshotReplySerialize(const SyncPreSnapshotReply* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncPreSnapshotReplyDeserialize(const char* buf, uint32_t len, SyncPreSnapshotReply* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncPreSnapshotReplySerialize2(const SyncPreSnapshotReply* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncPreSnapshotReplySerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncPreSnapshotReply* syncPreSnapshotReplyDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncPreSnapshotReply* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncPreSnapshotReplyDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncPreSnapshotReply2RpcMsg(const SyncPreSnapshotReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncPreSnapshotReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncPreSnapshotReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPreSnapshotReply* pMsg) { + syncPreSnapshotReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncPreSnapshotReply* syncPreSnapshotReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncPreSnapshotReply* pMsg = syncPreSnapshotReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncPreSnapshotReply2Json(const SyncPreSnapshotReply* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->snapStart); + cJSON_AddStringToObject(pRoot, "snap-start", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncPreSnapshotReply", pRoot); + return pJson; +} + +char* syncPreSnapshotReply2Str(const SyncPreSnapshotReply* pMsg) { + cJSON* pJson = syncPreSnapshotReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncPreSnapshotReplyPrint(const SyncPreSnapshotReply* pMsg) { + char* serialized = syncPreSnapshotReply2Str(pMsg); + printf("syncPreSnapshotReplyPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncPreSnapshotReplyPrint2(char* s, const SyncPreSnapshotReply* pMsg) { + char* serialized = syncPreSnapshotReply2Str(pMsg); + printf("syncPreSnapshotReplyPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncPreSnapshotReplyLog(const SyncPreSnapshotReply* pMsg) { + char* serialized = syncPreSnapshotReply2Str(pMsg); + sTrace("syncPreSnapshotReplyLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncPreSnapshotReplyLog2(char* s, const SyncPreSnapshotReply* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncPreSnapshotReply2Str(pMsg); + sTrace("syncPreSnapshotReplyLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + // ---- message process SyncApplyMsg---- SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncApplyMsg) + dataLen; @@ -2413,7 +2710,7 @@ cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) { cJSON_AddNumberToObject(pRoot, "fsmMeta.isWeak", pMsg->fsmMeta.isWeak); cJSON_AddNumberToObject(pRoot, "fsmMeta.code", pMsg->fsmMeta.code); cJSON_AddNumberToObject(pRoot, "fsmMeta.state", pMsg->fsmMeta.state); - cJSON_AddStringToObject(pRoot, "fsmMeta.state.str", syncUtilState2String(pMsg->fsmMeta.state)); + cJSON_AddStringToObject(pRoot, "fsmMeta.state.str", syncStr(pMsg->fsmMeta.state)); snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->fsmMeta.seqNum); cJSON_AddStringToObject(pRoot, "fsmMeta.seqNum", u64buf); @@ -2576,8 +2873,8 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); @@ -2751,8 +3048,8 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); @@ -2762,6 +3059,9 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); cJSON_AddNumberToObject(pRoot, "code", pMsg->code); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->snapBeginIndex); + cJSON_AddStringToObject(pRoot, "snap-begin", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index af53123421..4aa8b0bc34 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -62,8 +62,8 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { syncNodeEventLog(pSyncNode, logBuf); // start snapshot - int32_t code = syncNodeStartSnapshot(pSyncNode, pDestId); - ASSERT(code == 0); + // int32_t code = syncNodeStartSnapshot(pSyncNode, pDestId); + return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a7bafa9f28..b728315746 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -41,8 +41,6 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI } memset(pSender, 0, sizeof(*pSender)); - int64_t timeNow = taosGetTimestampMs(); - pSender->start = false; pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; @@ -53,8 +51,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode = pSyncNode; pSender->replicaIndex = replicaIndex; pSender->term = pSyncNode->pRaftStore->currentTerm; - pSender->privateTerm = timeNow + 100; - pSender->startTime = timeNow; + pSender->startTime = 0; pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; } else { @@ -88,88 +85,30 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } -// begin send snapshot by param, snapshot, pReader -// -// action: -// 1. assert reader not start -// 2. update state -// 3. send first snapshot block -int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapshotParam, SSnapshot snapshot, - void *pReader) { +int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { ASSERT(!snapshotSenderIsStart(pSender)); - // init snapshot, parm, reader - ASSERT(pSender->pReader == NULL); - pSender->pReader = pReader; - pSender->snapshot = snapshot; - pSender->snapshotParam = snapshotParam; - - // init current block - if (pSender->pCurrentBlock != NULL) { - taosMemoryFree(pSender->pCurrentBlock); - } - pSender->blockLen = 0; - - // update term - pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; - ++(pSender->privateTerm); // increase private term - - // update state - pSender->finish = false; pSender->start = true; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; + pSender->pReader = NULL; + pSender->pCurrentBlock = NULL; + pSender->blockLen = 0; - // init last config - if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { - int32_t code = 0; - SSyncRaftEntry *pEntry = NULL; - bool getLastConfig = false; + pSender->snapshotParam.start = SYNC_INDEX_INVALID; + pSender->snapshotParam.end = SYNC_INDEX_INVALID; - code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, - pSender->snapshot.lastConfigIndex, &pEntry); - if (code == 0 && pEntry != NULL) { - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pEntry, &rpcMsg); + pSender->snapshot.data = NULL; + pSender->snapshotParam.end = SYNC_INDEX_INVALID; + pSender->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; + pSender->snapshot.lastApplyTerm = SYNC_TERM_INVALID; + pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - SSyncCfg lastConfig; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &lastConfig); - ASSERT(ret == 0); - pSender->lastConfig = lastConfig; - getLastConfig = true; - - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); - } else { - if (pSender->snapshot.lastConfigIndex == pSender->pSyncNode->pRaftCfg->lastConfigIndex) { - sTrace("vgId:%d, sync sender get cfg from local", pSender->pSyncNode->vgId); - pSender->lastConfig = pSender->pSyncNode->pRaftCfg->cfg; - getLastConfig = true; - } - } - - // last config not found in wal, update to -1 - if (!getLastConfig) { - SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex; - SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID; - pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; - memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); - - // event log - do { - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %" PRId64 " to %" PRId64, - oldLastConfigIndex, newLastConfigIndex); - char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); - syncNodeEventLog(pSender->pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } - - } else { - // no last config - memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); - } + memset(&(pSender->lastConfig), 0, sizeof(pSender->lastConfig)); + pSender->sendingMS = 0; + pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSender->startTime = taosGetTimestampMs(); + pSender->finish = false; // build begin msg SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); @@ -181,8 +120,8 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho pMsg->lastTerm = pSender->snapshot.lastApplyTerm; pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; - pMsg->seq = pSender->seq; // SYNC_SNAPSHOT_SEQ_BEGIN - pMsg->privateTerm = pSender->privateTerm; + pMsg->startTime = pSender->startTime; + pMsg->seq = SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT; // send msg SRpcMsg rpcMsg; @@ -201,6 +140,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho } int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { + // update flag + pSender->start = false; + pSender->finish = finish; + // close reader if (pSender->pReader != NULL) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); @@ -215,12 +158,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { pSender->blockLen = 0; } - // update flag - pSender->start = false; - pSender->finish = finish; - - // do not update term, maybe print - // event log do { char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop"); @@ -263,7 +200,9 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; - pMsg->privateTerm = pSender->privateTerm; + + // pMsg->privateTerm = pSender->privateTerm; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); // send msg @@ -302,7 +241,9 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; pMsg->lastConfig = pSender->lastConfig; pMsg->seq = pSender->seq; - pMsg->privateTerm = pSender->privateTerm; + + // pMsg->privateTerm = pSender->privateTerm; + memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); // send msg @@ -367,8 +308,10 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); } @@ -395,30 +338,38 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { snprintf(s, len, "%s {%p s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64 - " seq:%d ack:%d finish:%d pterm:%" PRIu64 - " " - "replica-index:%d %s:%d}", + " seq:%d ack:%d finish:%d replica-index:%d %s:%d}", event, pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, - pSender->finish, pSender->privateTerm, pSender->replicaIndex, host, port); + pSender->finish, pSender->replicaIndex, host, port); return s; } int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { - // calculate index - - syncNodeEventLog(pSyncNode, "start snapshot ..."); + syncNodeEventLog(pSyncNode, "starting snapshot ..."); SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId); if (pSender == NULL) { - // create sender - } else { - // if is same - // return 0; + syncNodeErrorLog(pSyncNode, "start snapshot error, sender is null"); + return -1; } - // send begin msg + int32_t code = 0; + + if (snapshotSenderIsStart(pSender)) { + code = snapshotSenderStop(pSender, false); + if (code != 0) { + syncNodeErrorLog(pSyncNode, "snapshot sender stop error"); + return -1; + } + } + + code = snapshotSenderStart(pSender); + if (code != 0) { + syncNodeErrorLog(pSyncNode, "snapshot sender start error"); + return -1; + } return 0; } @@ -440,7 +391,6 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->pSyncNode = pSyncNode; pReceiver->fromId = fromId; pReceiver->term = pSyncNode->pRaftStore->currentTerm; - pReceiver->privateTerm = 0; pReceiver->snapshot.data = NULL; pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID; pReceiver->snapshot.lastApplyTerm = 0; @@ -474,19 +424,8 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive // receive first snapshot data // write first block data static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { - // update state - pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; - pReceiver->privateTerm = pBeginMsg->privateTerm; - pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; - pReceiver->fromId = pBeginMsg->srcId; pReceiver->start = true; - - // update snapshot - pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; - pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm; - pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex; - pReceiver->snapshotParam.start = pBeginMsg->beginIndex; - pReceiver->snapshotParam.end = pBeginMsg->lastIndex; + pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; // start writer ASSERT(pReceiver->pWriter == NULL); @@ -494,6 +433,19 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapsh &(pReceiver->snapshotParam), &(pReceiver->pWriter)); ASSERT(ret == 0); + pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm; + pReceiver->snapshotParam.start = pBeginMsg->beginIndex; + pReceiver->snapshotParam.end = pBeginMsg->lastIndex; + + pReceiver->fromId = pBeginMsg->srcId; + + // update snapshot + pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; + pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm; + pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex; + + pReceiver->startTime = pBeginMsg->startTime; + // event log do { char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver start"); @@ -523,22 +475,9 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { } // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver -// if already start, force close, start again int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { - if (!snapshotReceiverIsStart(pReceiver)) { - // first start - snapshotReceiverDoStart(pReceiver, pBeginMsg); - - } else { - // already start - sInfo("vgId:%d, snapshot recv, receiver already start", pReceiver->pSyncNode->vgId); - - // force close, abandon incomplete data - snapshotReceiverForceStop(pReceiver); - - // start again - snapshotReceiverDoStart(pReceiver, pBeginMsg); - } + ASSERT(!snapshotReceiverIsStart(pReceiver)); + snapshotReceiverDoStart(pReceiver, pBeginMsg); return 0; } @@ -698,8 +637,8 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); } cJSON *pJson = cJSON_CreateObject(); @@ -724,38 +663,204 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) syncUtilU642Addr(fromId.addr, host, sizeof(host), &port); snprintf(s, len, - "%s {%p start:%d ack:%d term:%" PRIu64 " pterm:%" PRIu64 " from:%s:%d s-param:%" PRId64 " e-param:%" PRId64 - " laindex:%" PRId64 " laterm:%" PRIu64 + "%s {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from:%s:%d s-param:%" PRId64 + " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " " "lcindex:%" PRId64 "}", - event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port, + event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->startTime, host, port, pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex); return s; } +SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { + SyncIndex snapStart = SYNC_INDEX_INVALID; + + if (syncNodeIsMnode(ths)) { + snapStart = SYNC_INDEX_BEGIN; + + } else { + SSyncLogStoreData *pData = ths->pLogStore->data; + SWal *pWal = pData->pWal; + + bool isEmpty = ths->pLogStore->syncLogIsEmpty(ths->pLogStore); + int64_t walCommitVer = walGetCommittedVer(pWal); + + if (!isEmpty && ths->commitIndex != walCommitVer) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "commit not same, wal-commit:%" PRId64 ", commit:%" PRId64 ", ignore", + walCommitVer, ths->commitIndex); + syncNodeErrorLog(ths, logBuf); + + snapStart = walCommitVer + 1; + } else { + snapStart = ths->commitIndex + 1; + } + } + + return snapStart; +} + +static int32_t syncNodeOnSnapshotPre(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + + if (snapshotReceiverIsStart(pReceiver)) { + // already start + + if (pMsg->startTime > pReceiver->startTime) { + goto _START_RECEIVER; + + } else if (pMsg->startTime == pReceiver->startTime) { + goto _SEND_REPLY; + + } else { + // ignore + return 0; + } + + } else { + // start new + goto _START_RECEIVER; + } + +_START_RECEIVER: + if (taosGetTimestampMs() - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) { + syncNodeErrorLog(pSyncNode, "snapshot receiver time skew too much"); + return -1; + } else { + // waiting for clock match + while (taosGetTimestampMs() > pMsg->startTime) { + taosMsleep(10); + } + + snapshotReceiverStart(pReceiver, pMsg); // set start-time same with sender + } + +_SEND_REPLY: + // build msg + ; // make complier happy + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->startTime = pReceiver->startTime; + pRspMsg->ack = pMsg->seq; // receiver maybe already closed + pRspMsg->code = 0; + pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode); + + // send msg + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + syncSnapshotRspDestroy(pRspMsg); + + return 0; +} + +static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { + // condition 1 + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; + + if (snapshotReceiverIsStart(pReceiver)) { + if (pMsg->startTime > pReceiver->startTime) { + snapshotReceiverStop(pReceiver); + + } else if (pMsg->startTime == pReceiver->startTime) { + return 0; + } else { + // ignore + syncNodeEventLog(pSyncNode, "msg ignore"); + return 0; + } + } + +_START_RECEIVER: + if (taosGetTimestampMs() - pMsg->startTime > SNAPSHOT_MAX_CLOCK_SKEW_MS) { + syncNodeErrorLog(pSyncNode, "snapshot receiver time skew too much"); + return -1; + } else { + // waiting for clock match + while (taosGetTimestampMs() > pMsg->startTime) { + taosMsleep(10); + } + + snapshotReceiverStart(pReceiver, pMsg); + + // build msg + SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); + pRspMsg->srcId = pSyncNode->myRaftId; + pRspMsg->destId = pMsg->srcId; + pRspMsg->term = pSyncNode->pRaftStore->currentTerm; + pRspMsg->lastIndex = pMsg->lastIndex; + pRspMsg->lastTerm = pMsg->lastTerm; + pRspMsg->ack = pReceiver->ack; // receiver maybe already closed + pRspMsg->code = 0; + + // send msg + SRpcMsg rpcMsg; + syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); + syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); + syncSnapshotRspDestroy(pRspMsg); + } + + return 0; +} + +static int32_t syncNodeOnSnapshotTransfer(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; } + +static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; } + // receiver on message // -// condition 1, recv SYNC_SNAPSHOT_SEQ_BEGIN, start receiver, update privateTerm -// condition 2, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig) -// condition 3, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close -// condition 4, got data, update ack +// condition 1, recv SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT +// if receiver already start +// if sender.start-time > receiver.start-time, restart receiver(reply snapshot start) +// if sender.start-time = receiver.start-time, maybe duplicate msg +// if sender.start-time < receiver.start-time, ignore +// else +// waiting for clock match +// start receiver(reply snapshot start) +// +// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN +// a. create writer with +// +// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig) +// +// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close +// +// condition 5, got data, update ack // int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { - // get receiver - SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; - bool needRsp = false; + // if already drop replica, do not process + if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) { + syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "not in my config"); + return 0; + } + + if (pMsg->term < pSyncNode->pRaftStore->currentTerm) { + syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject, small term"); + return 0; + } + + if (pMsg->term > pSyncNode->pRaftStore->currentTerm) { + syncNodeStepDown(pSyncNode, pMsg->term); + } + syncNodeResetElectTimer(pSyncNode); + int32_t code = 0; + SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { - if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - // condition 1 - // begin, no data - snapshotReceiverStart(pReceiver, pMsg); - needRsp = true; + if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { + syncNodeOnSnapshotPre(pSyncNode, pMsg); + + } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { + syncNodeOnSnapshotBegin(pSyncNode, pMsg); } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // condition 2 @@ -764,7 +869,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (code == 0) { snapshotReceiverStop(pReceiver); } - needRsp = true; + bool needRsp = true; // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { @@ -782,7 +887,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // condition 3 // force close snapshotReceiverForceStop(pReceiver); - needRsp = false; + bool needRsp = false; } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // condition 4 @@ -790,7 +895,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (pMsg->seq == pReceiver->ack + 1) { snapshotReceiverGotData(pReceiver, pMsg); } - needRsp = true; + bool needRsp = true; } else { // error log @@ -805,26 +910,6 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return -1; } - // send ack - if (needRsp) { - // build msg - SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); - pRspMsg->srcId = pSyncNode->myRaftId; - pRspMsg->destId = pMsg->srcId; - pRspMsg->term = pSyncNode->pRaftStore->currentTerm; - pRspMsg->lastIndex = pMsg->lastIndex; - pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pReceiver->ack; // receiver maybe already closed - pRspMsg->code = 0; - pRspMsg->privateTerm = pReceiver->privateTerm; // receiver maybe already closed - - // send msg - SRpcMsg rpcMsg; - syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); - syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); - syncSnapshotRspDestroy(pRspMsg); - } - } else { // error log do { @@ -849,6 +934,52 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { return 0; } +int32_t syncNodeOnSnapshotReplyPre(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + // get sender + SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); + ASSERT(pSender != NULL); + + SSnapshot snapshot; + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + + // prepare + pSender->snapshotParam.start = pMsg->snapBeginIndex; + pSender->snapshotParam.end = snapshot.lastApplyIndex; + + if (pMsg->snapBeginIndex > snapshot.lastApplyIndex) { + syncNodeErrorLog(pSyncNode, "snapshot last index too small"); + return -1; + } + + // start reader + int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &(pSender->snapshotParam), &(pSender->pReader)); + if (code != 0) { + syncNodeErrorLog(pSyncNode, "create snapshot reader error"); + return -1; + } + + // build begin msg + SyncSnapshotSend *pSendMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId); + pSendMsg->srcId = pSender->pSyncNode->myRaftId; + pSendMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pSendMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pSendMsg->beginIndex = pSender->snapshotParam.start; + pSendMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pSendMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pSendMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pSendMsg->lastConfig = pSender->lastConfig; + pSendMsg->startTime = pSender->startTime; + pSendMsg->seq = SYNC_SNAPSHOT_SEQ_BEGIN; + + // send msg + SRpcMsg rpcMsg; + syncSnapshotSend2RpcMsg(pSendMsg, &rpcMsg); + syncNodeSendMsgById(&(pSendMsg->destId), pSender->pSyncNode, &rpcMsg); + syncSnapshotSendDestroy(pSendMsg); + + return 0; +} + // sender on message // // condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender @@ -857,8 +988,8 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { // if already drop replica, do not process - if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - sError("vgId:%d, recv sync-snapshot-rsp, maybe replica already dropped", pSyncNode->vgId); + if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId))) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "maybe replica already dropped"); return -1; } @@ -866,17 +997,26 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); ASSERT(pSender != NULL); + if (pMsg->startTime != pSender->startTime) { + syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "sender/receiver start time not match"); + return -1; + } + // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { - // condition 1 + // prepare , send begin msg + if (pMsg->ack == SYNC_SNAPSHOT_SEQ_PRE_SNAPSHOT) { + syncNodeOnSnapshotReplyPre(pSyncNode, pMsg); + return 0; + } + // receive ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { snapshotSenderStop(pSender, true); return 0; } - // condition 2 // send next msg if (pMsg->ack == pSender->seq) { // update sender ack @@ -922,3 +1062,63 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { return 0; } + +int32_t syncNodeOnPreSnapshot(SSyncNode *ths, SyncPreSnapshot *pMsg) { + syncLogRecvSyncPreSnapshot(ths, pMsg, ""); + + SyncPreSnapshotReply *pMsgReply = syncPreSnapshotReplyBuild(ths->vgId); + pMsgReply->srcId = ths->myRaftId; + pMsgReply->destId = pMsg->srcId; + pMsgReply->term = ths->pRaftStore->currentTerm; + + SSyncLogStoreData *pData = ths->pLogStore->data; + SWal *pWal = pData->pWal; + + if (syncNodeIsMnode(ths)) { + pMsgReply->snapStart = SYNC_INDEX_BEGIN; + + } else { + bool isEmpty = ths->pLogStore->syncLogIsEmpty(ths->pLogStore); + int64_t walCommitVer = walGetCommittedVer(pWal); + + if (!isEmpty && ths->commitIndex != walCommitVer) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "commit not same, wal-commit:%" PRId64 ", commit:%" PRId64 ", ignore", + walCommitVer, ths->commitIndex); + syncNodeErrorLog(ths, logBuf); + + goto _IGNORE; + } + + pMsgReply->snapStart = ths->commitIndex + 1; + + // make local log clean + int32_t code = ths->pLogStore->syncLogTruncate(ths->pLogStore, pMsgReply->snapStart); + if (code != 0) { + syncNodeErrorLog(ths, "truncate wal error"); + goto _IGNORE; + } + } + + // can not write behind _RESPONSE + SRpcMsg rpcMsg; + +_RESPONSE: + syncPreSnapshotReply2RpcMsg(pMsgReply, &rpcMsg); + syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + + syncPreSnapshotReplyDestroy(pMsgReply); + return 0; + +_IGNORE: + syncPreSnapshotReplyDestroy(pMsgReply); + return 0; +} + +int32_t syncNodeOnPreSnapshotReply(SSyncNode *ths, SyncPreSnapshotReply *pMsg) { + syncLogRecvSyncPreSnapshotReply(ths, pMsg, ""); + + // start snapshot + + return 0; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1750dce5ec..164e050930 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -176,18 +176,6 @@ char* syncUtilRaftId2Str(const SRaftId* p) { return serialized; } -const char* syncUtilState2String(ESyncState state) { - if (state == TAOS_SYNC_STATE_FOLLOWER) { - return "follower"; - } else if (state == TAOS_SYNC_STATE_CANDIDATE) { - return "candidate"; - } else if (state == TAOS_SYNC_STATE_LEADER) { - return "leader"; - } else { - return "state_error"; - } -} - bool syncUtilCanPrint(char c) { if (c >= 32 && c <= 126) { return true; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 26dd32942b..1b7ec7f0df 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -60,6 +60,8 @@ add_executable(syncRaftCfgIndexTest "") add_executable(syncHeartbeatTest "") add_executable(syncHeartbeatReplyTest "") add_executable(syncLocalCmdTest "") +add_executable(syncPreSnapshotTest "") +add_executable(syncPreSnapshotReplyTest "") target_sources(syncTest @@ -310,6 +312,14 @@ target_sources(syncLocalCmdTest PRIVATE "syncLocalCmdTest.cpp" ) +target_sources(syncPreSnapshotTest + PRIVATE + "syncPreSnapshotTest.cpp" +) +target_sources(syncPreSnapshotReplyTest + PRIVATE + "syncPreSnapshotReplyTest.cpp" +) target_include_directories(syncTest @@ -622,6 +632,16 @@ target_include_directories(syncLocalCmdTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncPreSnapshotTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncPreSnapshotReplyTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -872,6 +892,14 @@ target_link_libraries(syncLocalCmdTest sync gtest_main ) +target_link_libraries(syncPreSnapshotTest + sync + gtest_main +) +target_link_libraries(syncPreSnapshotReplyTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 96e0b6c483..dcf7249d30 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -47,8 +47,8 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag, cbMeta.term); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag, + cbMeta.term); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); @@ -57,10 +57,10 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; - snprintf( - logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 + "\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -68,8 +68,7 @@ void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -158,6 +157,7 @@ SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -172,6 +172,7 @@ SSyncFSM* createFsm() { pFsm->FpSnapshotDoWrite = SnapshotDoWrite; pFsm->FpReConfigCb = ReConfigCb; +#endif return pFsm; } diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index bf01f76607..c5548d25d4 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -33,7 +33,7 @@ void init() { void cleanup() { walCleanUp(); } -void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { +void CommitCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta cbMeta) { SyncIndex beginIndex = SYNC_INDEX_INVALID; if (pFsm->FpGetSnapshotInfo != NULL) { SSnapshot snapshot; @@ -45,33 +45,31 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } -void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { +void PreCommitCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; - snprintf( - logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); + snprintf(logBuf, sizeof(logBuf), + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 + "\n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } -void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { +void RollBackCb(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } -int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { +int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { pSnapshot->data = NULL; pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; pSnapshot->lastApplyTerm = 100; @@ -89,6 +87,7 @@ SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -97,6 +96,7 @@ SSyncFSM* createFsm() { pFsm->FpRestoreFinishCb = RestoreFinishCb; pFsm->FpReConfigCb = ReConfigCb; +#endif return pFsm; } diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 404ab32d58..a220c2aed4 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -26,12 +26,12 @@ int main() { assert(ret == 0); for (int i = 0; i < 5; ++i) { - ret = syncEnvStartTimer(); + //ret = syncEnvStartTimer(); assert(ret == 0); taosMsleep(5000); - ret = syncEnvStopTimer(); + //ret = syncEnvStopTimer(); assert(ret == 0); taosMsleep(5000); diff --git a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp new file mode 100644 index 0000000000..a30dcc2c54 --- /dev/null +++ b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp @@ -0,0 +1,99 @@ + +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncPreSnapshotReply *createMsg() { + SyncPreSnapshotReply *pMsg = syncPreSnapshotReplyBuild(789); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 9527; + pMsg->snapStart = 12306; + return pMsg; +} + +void test1() { + SyncPreSnapshotReply *pMsg = createMsg(); + syncPreSnapshotReplyLog2((char *)"test1:", pMsg); + syncPreSnapshotReplyDestroy(pMsg); +} + +void test2() { + SyncPreSnapshotReply *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncPreSnapshotReplySerialize(pMsg, serialized, len); + SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyBuild(789); + syncPreSnapshotReplyDeserialize(serialized, len, pMsg2); + syncPreSnapshotReplyLog2((char *)"test2: syncPreSnapshotReplySerialize -> syncPreSnapshotReplyDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncPreSnapshotReplyDestroy(pMsg); + syncPreSnapshotReplyDestroy(pMsg2); +} + +void test3() { + SyncPreSnapshotReply *pMsg = createMsg(); + uint32_t len; + char * serialized = syncPreSnapshotReplySerialize2(pMsg, &len); + SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyDeserialize2(serialized, len); + syncPreSnapshotReplyLog2((char *)"test3: syncPreSnapshotReplySerialize2 -> syncPreSnapshotReplyDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncPreSnapshotReplyDestroy(pMsg); + syncPreSnapshotReplyDestroy(pMsg2); +} + +void test4() { + SyncPreSnapshotReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncPreSnapshotReply2RpcMsg(pMsg, &rpcMsg); + SyncPreSnapshotReply *pMsg2 = (SyncPreSnapshotReply *)taosMemoryMalloc(rpcMsg.contLen); + syncPreSnapshotReplyFromRpcMsg(&rpcMsg, pMsg2); + syncPreSnapshotReplyLog2((char *)"test4: syncPreSnapshotReply2RpcMsg -> syncPreSnapshotReplyFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncPreSnapshotReplyDestroy(pMsg); + syncPreSnapshotReplyDestroy(pMsg2); +} + +void test5() { + SyncPreSnapshotReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncPreSnapshotReply2RpcMsg(pMsg, &rpcMsg); + SyncPreSnapshotReply *pMsg2 = syncPreSnapshotReplyFromRpcMsg2(&rpcMsg); + syncPreSnapshotReplyLog2((char *)"test5: syncPreSnapshotReply2RpcMsg -> syncPreSnapshotReplyFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncPreSnapshotReplyDestroy(pMsg); + syncPreSnapshotReplyDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncPreSnapshotTest.cpp b/source/libs/sync/test/syncPreSnapshotTest.cpp new file mode 100644 index 0000000000..03894dfa84 --- /dev/null +++ b/source/libs/sync/test/syncPreSnapshotTest.cpp @@ -0,0 +1,97 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncPreSnapshot *createMsg() { + SyncPreSnapshot *pMsg = syncPreSnapshotBuild(789); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 9527; + return pMsg; +} + +void test1() { + SyncPreSnapshot *pMsg = createMsg(); + syncPreSnapshotLog2((char *)"test1:", pMsg); + syncPreSnapshotDestroy(pMsg); +} + +void test2() { + SyncPreSnapshot *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncPreSnapshotSerialize(pMsg, serialized, len); + SyncPreSnapshot *pMsg2 = syncPreSnapshotBuild(789); + syncPreSnapshotDeserialize(serialized, len, pMsg2); + syncPreSnapshotLog2((char *)"test2: syncPreSnapshotSerialize -> syncPreSnapshotDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncPreSnapshotDestroy(pMsg); + syncPreSnapshotDestroy(pMsg2); +} + +void test3() { + SyncPreSnapshot *pMsg = createMsg(); + uint32_t len; + char * serialized = syncPreSnapshotSerialize2(pMsg, &len); + SyncPreSnapshot *pMsg2 = syncPreSnapshotDeserialize2(serialized, len); + syncPreSnapshotLog2((char *)"test3: syncPreSnapshotSerialize2 -> syncPreSnapshotDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncPreSnapshotDestroy(pMsg); + syncPreSnapshotDestroy(pMsg2); +} + +void test4() { + SyncPreSnapshot *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncPreSnapshot2RpcMsg(pMsg, &rpcMsg); + SyncPreSnapshot *pMsg2 = (SyncPreSnapshot *)taosMemoryMalloc(rpcMsg.contLen); + syncPreSnapshotFromRpcMsg(&rpcMsg, pMsg2); + syncPreSnapshotLog2((char *)"test4: syncPreSnapshot2RpcMsg -> syncPreSnapshotFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncPreSnapshotDestroy(pMsg); + syncPreSnapshotDestroy(pMsg2); +} + +void test5() { + SyncPreSnapshot *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncPreSnapshot2RpcMsg(pMsg, &rpcMsg); + SyncPreSnapshot *pMsg2 = syncPreSnapshotFromRpcMsg2(&rpcMsg); + syncPreSnapshotLog2((char *)"test5: syncPreSnapshot2RpcMsg -> syncPreSnapshotFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncPreSnapshotDestroy(pMsg); + syncPreSnapshotDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 0cb9b51fba..78c08a6d8d 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -25,7 +25,7 @@ const char* pWalPath = "./syncLogStoreTest_wal"; SyncIndex gSnapshotLastApplyIndex; SyncIndex gSnapshotLastApplyTerm; -int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { +int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { pSnapshot->data = NULL; pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index 1171ef8785..cf862feb2a 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -27,7 +27,7 @@ const char* pWalPath = "./syncLogStoreTest_wal"; SyncIndex gSnapshotLastApplyIndex; SyncIndex gSnapshotLastApplyTerm; -int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { +int32_t GetSnapshotCb(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { pSnapshot->data = NULL; pSnapshot->lastApplyIndex = gSnapshotLastApplyIndex; pSnapshot->lastApplyTerm = gSnapshotLastApplyTerm; diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 7552fc7ae3..7506ae19bd 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -42,7 +42,7 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); @@ -53,7 +53,7 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -61,7 +61,7 @@ void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -74,10 +74,14 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); + +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; pFsm->FpGetSnapshotInfo = GetSnapshotCb; +#endif + return pFsm; } diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 0f8e76f121..6eb8548740 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -37,9 +37,12 @@ SSyncSnapshotReceiver* createReceiver() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + +#if 0 pSyncNode->pFsm->FpSnapshotStartWrite = SnapshotStartWrite; pSyncNode->pFsm->FpSnapshotStopWrite = SnapshotStopWrite; pSyncNode->pFsm->FpSnapshotDoWrite = SnapshotDoWrite; +#endif SRaftId id; id.addr = syncUtilAddr2U64("1.2.3.4", 99); @@ -50,7 +53,6 @@ SSyncSnapshotReceiver* createReceiver() { pReceiver->ack = 20; pReceiver->pWriter = (void*)0x11; pReceiver->term = 66; - pReceiver->privateTerm = 99; return pReceiver; } diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 89348da617..63905c2182 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -21,7 +21,7 @@ SyncSnapshotRsp *createMsg() { pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; pMsg->term = 11; - pMsg->privateTerm = 99; + pMsg->startTime = 99; pMsg->lastIndex = 22; pMsg->lastTerm = 33; pMsg->ack = 44; diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 6fcfa6f6c4..83f1dfebb3 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -21,7 +21,6 @@ SyncSnapshotSend *createMsg() { pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); pMsg->destId.vgId = 100; pMsg->term = 11; - pMsg->privateTerm = 99; pMsg->lastIndex = 22; pMsg->lastTerm = 33; diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 8d1f83b3b1..7b2c13463c 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -37,10 +37,13 @@ SSyncSnapshotSender* createSender() { SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(*pSyncNode)); pSyncNode->pRaftStore = (SRaftStore*)taosMemoryMalloc(sizeof(*(pSyncNode->pRaftStore))); pSyncNode->pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(*(pSyncNode->pFsm))); + + #if 0 pSyncNode->pFsm->FpSnapshotStartRead = SnapshotStartRead; pSyncNode->pFsm->FpSnapshotStopRead = SnapshotStopRead; pSyncNode->pFsm->FpSnapshotDoRead = SnapshotDoRead; pSyncNode->pFsm->FpGetSnapshotInfo = GetSnapshot; +#endif SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, 2); pSender->start = true; @@ -55,7 +58,8 @@ SSyncSnapshotSender* createSender() { pSender->snapshot.lastApplyTerm = 88; pSender->sendingMS = 77; pSender->term = 66; - pSender->privateTerm = 99; + + //pSender->privateTerm = 99; return pSender; } diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index 2aff0aad93..ea90621bac 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -33,7 +33,7 @@ SyncIndex snapshotLastApplyIndex = SYNC_INDEX_INVALID; const char *pDir = "./syncSnapshotTest"; const char *pWalDir = "./syncSnapshotTest_wal"; -void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void CommitCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SyncIndex beginIndex = SYNC_INDEX_INVALID; if (pFsm->FpGetSnapshotInfo != NULL) { SSnapshot snapshot; @@ -45,30 +45,30 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } else { sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } -void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void PreCommitCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { +void RollBackCb(const struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -int32_t GetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { +int32_t GetSnapshotCb(const struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { pSnapshot->data = NULL; pSnapshot->lastApplyIndex = snapshotLastApplyIndex; pSnapshot->lastApplyTerm = 100; @@ -78,10 +78,14 @@ int32_t GetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) { void initFsm() { pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); + +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; pFsm->FpGetSnapshotInfo = GetSnapshotCb; +#endif + } SSyncNode *syncNodeInit() { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index a36f3af450..e6f6006410 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -44,8 +44,8 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { ", term:%" PRIu64 " " "currentTerm:%" PRIu64 " \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag, + cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -56,8 +56,8 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) ", term:%" PRIu64 " " "currentTerm:%" PRIu64 " \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag, + cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -68,8 +68,8 @@ void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { ", term:%" PRIu64 " " "currentTerm:%" PRIu64 " \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag, + cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -168,8 +168,8 @@ void LeaderTransferCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbM ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " " "currentTerm:%" PRIu64 " \n", - pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), - cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state), cbMeta.flag, + cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -177,6 +177,7 @@ SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; @@ -193,6 +194,7 @@ SSyncFSM* createFsm() { pFsm->FpSnapshotDoWrite = SnapshotDoWrite; pFsm->FpLeaderTransferCb = LeaderTransferCb; +#endif return pFsm; } diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 0547f39bee..cafecabc4d 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -35,7 +35,7 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } @@ -43,7 +43,7 @@ void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } @@ -51,15 +51,19 @@ void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, - cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); + cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncStr(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void initFsm() { pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM)); + +#if 0 pFsm->FpCommitCb = CommitCb; pFsm->FpPreCommitCb = PreCommitCb; pFsm->FpRollBackCb = RollBackCb; +#endif + } SSyncNode *syncNodeInit() { diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 19580d486a..e4d3cc3488 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1715,16 +1715,20 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { memcpy(pKey, cd.pKey, cd.kLen); if (ppVal) { - // TODO: vLen may be zero - pVal = tdbRealloc(*ppVal, cd.vLen); - if (pVal == NULL) { - tdbFree(pKey); - return -1; + if (cd.vLen > 0) { + pVal = tdbRealloc(*ppVal, cd.vLen); + if (pVal == NULL) { + tdbFree(pKey); + return -1; + } + + memcpy(pVal, cd.pVal, cd.vLen); + } else { + pVal = NULL; } *ppVal = pVal; *vLen = cd.vLen; - memcpy(pVal, cd.pVal, cd.vLen); } ret = tdbBtcMoveToNext(pBtc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 97915e1ded..21444018cd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -718,8 +718,10 @@ static void cliSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); } else { - tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); - cliHandleExcept(pConn); + if (!uv_is_closing((uv_handle_t*)&pConn->stream)) { + tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); + cliHandleExcept(pConn); + } return; } if (cliHandleNoResp(pConn) == true) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b829251ccd..944995c892 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -127,7 +127,7 @@ static void uvFreeCb(uv_handle_t* handle); static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg); -static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb); +static int uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); @@ -363,9 +363,11 @@ void uvOnSendCb(uv_write_t* req, int status) { } transUnrefSrvHandle(conn); } else { - tError("conn %p failed to write data, %s", conn, uv_err_name(status)); - conn->broken = true; - transUnrefSrvHandle(conn); + if (!uv_is_closing((uv_handle_t*)(conn->pTcp))) { + tError("conn %p failed to write data, %s", conn, uv_err_name(status)); + conn->broken = true; + transUnrefSrvHandle(conn); + } } } static void uvOnPipeWriteCb(uv_write_t* req, int status) { @@ -382,7 +384,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { taosMemoryFree(req); } -static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { +static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { SSvrConn* pConn = smsg->pConn; STransMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { @@ -395,6 +397,13 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { pHead->hasEpSet = pMsg->info.hasEpSet; pHead->magicNum = htonl(TRANS_MAGIC_NUM); + // handle invalid drop_task resp, TD-20098 + if (pMsg->msgType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + transQueuePop(&pConn->srvMsgs); + destroySmsg(smsg); + return -1; + } + if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); if (smsg->type == Release) pHead->msgType = 0; @@ -429,6 +438,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { wb->base = (char*)pHead; wb->len = len; + return 0; } static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { @@ -438,7 +448,9 @@ static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { } uv_buf_t wb; - uvPrepareSendData(smsg, &wb); + if (uvPrepareSendData(smsg, &wb) < 0) { + return; + } transRefSrvHandle(pConn); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); @@ -449,8 +461,9 @@ static void uvStartSendResp(SSvrMsg* smsg) { SSvrConn* pConn = smsg->pConn; if (pConn->broken == true) { // persist by - transFreeMsg(smsg->msg.pCont); - taosMemoryFree(smsg); + destroySmsg(smsg); + // transFreeMsg(smsg->msg.pCont); + // taosMemoryFree(smsg); transUnrefSrvHandle(pConn); return; } @@ -746,10 +759,11 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { return; } transSockInfo2Str(&sockname, pConn->src); - struct sockaddr_in addr = *(struct sockaddr_in*)&sockname; + struct sockaddr_in addr = *(struct sockaddr_in*)&peername; pConn->clientIp = addr.sin_addr.s_addr; pConn->port = ntohs(addr.sin_port); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); } else { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3baa906b19..1c6e1a2e17 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -201,7 +201,13 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } if (end == fileSize) firstTrial = false; - if (firstTrial && terrno == TSDB_CODE_SUCCESS) continue; + if (firstTrial) { + if (terrno == TSDB_CODE_SUCCESS) { + continue; + } else { + firstTrial = false; + } + } if (retVer >= 0 || offset == 0) break; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 7afb3c7260..f7e56f372f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -94,6 +94,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp for TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISKSPACE, "Out of disk space") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -109,7 +110,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE, "System out of disk space") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query @@ -305,35 +305,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, "Action need to be reprocessed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, "Vnode initialization failed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE, "System out of disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, "No write permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, "Missing data file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, "Unexpected generic error in vnode") - -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, "Database memory is full") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, "Database is dropping") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_UPDATING, "Database is updating") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_CLOSING, "Database is closing") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TSDB_STATE, "Invalid tsdb state") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_TB_NOT_EXIST, "Table not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_SMA_NOT_EXIST, "SMA not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS, "Table column not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_CFG_FILE, "Invalid config file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TERM_FILE, "Invalid term file") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -342,7 +321,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") @@ -362,13 +340,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last ro TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED, "Table re-created") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, "No diskspace for query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") @@ -431,7 +408,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INVALID_CONFIG, "TQ invalid config") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INIT_FAILED, "TQ init falied") -TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISKSPACE, "TQ no disk space") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISK_PERMISSIONS, "TQ no disk permissions") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_CORRUPTED, "TQ file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_OUT_OF_MEMORY, "TQ out of memory") diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py index 23b6341e66..4e3c9d07ce 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py @@ -25,7 +25,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py index 4473ff2c2d..163cdd0055 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py index 4f8e94a887..0fb39c210c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 95afdc413e..7599c82483 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -23,7 +23,8 @@ class TDTestCase: [TD-11510] taosBenchmark test cases """ - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 90d0ffd8b1..6be5117b08 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -27,7 +27,8 @@ class TDTestCase: """ return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py index 7437b46353..e63908ce33 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index a706967bc1..73894d5e33 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py index 3b9123974f..afefabef66 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py index dbddb840ca..d95e6741f8 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py index 7f0b082e33..375113c3ef 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py @@ -29,7 +29,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py index 5b59d84b79..fd79f1e01b 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py index ee7bb8a925..200a3c0e69 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py index d7ccdc2716..2c6d09b0f5 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py index 7b0f5d1672..241c6d6359 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py @@ -24,7 +24,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py index 1588b4077c..4d0adde192 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py @@ -25,7 +25,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py index b8d4c35e58..78cb49846e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py @@ -25,7 +25,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py index 09976953e1..a6c2062d6c 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -52,7 +53,7 @@ class TDTestCase: return paths[0] def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py index 7eaddd9e2d..d10734abd7 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py index c70748e9c2..98a128b489 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py index eb4b8a364a..1342295d7d 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py index 70501965fe..f3de1c7628 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py @@ -28,7 +28,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -51,7 +52,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py index 5e551a373c..1133c1d1a5 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py @@ -28,7 +28,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -51,7 +52,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py index f2d4c4f814..933bc41bae 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py index 5971902cb6..77dd3d3938 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py index f2dca23488..5568c3679f 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -49,7 +50,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py index 6f2781d645..852aa17dc5 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -49,7 +50,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py index b226204654..ab61b48c8f 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py index 766110d74e..ae895712c0 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py index 242681b0fc..86bcfd92f2 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py index d512cba97c..d67dfea408 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py @@ -27,7 +27,8 @@ class TDTestCase: ''' return - def init(self, conn, logSql, replicaVarl=1): + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -50,7 +51,7 @@ class TDTestCase: return buildPath def run(self): - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdSql.execute("drop database if exists db") tdSql.execute("create database db days 11 keep 3649 blocks 8 ") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 8584555333..3b00c6bf24 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -451,6 +451,7 @@ ./test.sh -f tsim/tag/smallint.sim ./test.sh -f tsim/tag/tinyint.sim ./test.sh -f tsim/tag/drop_tag.sim +./test.sh -f tsim/tag/tbNameIn.sim ./test.sh -f tmp/monitor.sim #======================b1-end=============== diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index c59a43263b..cdabc97c09 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -45,74 +45,9 @@ if $data(4)[4] != ready then goto step1 endi -return - +return print =============== step2: create database -sql create database db vgroups 1 replica 3 -sql select * from information_schema.ins_databases -if $rows != 3 then - return -1 -endi -if $data(db)[4] != 3 then - return -1 -endi - -sql select * from information_schema.ins_dnodes -if $data(2)[2] != 1 then - return -1 -endi -if $data(3)[2] != 1 then - return -1 -endi -if $data(4)[2] != 1 then - return -1 -endi - -# vnodes -sql select * from information_schema.ins_dnodes -if $data(2)[2] != 1 then - return -1 -endi -if $data(3)[2] != 1 then - return -1 -endi -if $data(4)[2] != 1 then - return -1 -endi - -# v1_dnode -$hasleader = 0 -$x = 0 -step2: - $x = $x + 1 - sleep 1000 - if $x == 20 then - print ====> dnode not ready! - return -1 - endi -sql show db.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 -if $data(2)[3] != 2 then - return -1 -endi -if $data(2)[5] != 3 then - return -1 -endi -if $data(2)[7] != 4 then - return -1 -endi -if $data(2)[4] == leader then - $hasleader = 1 -endi -if $data(2)[6] == leader then - $hasleader = 1 -endi -if $data(2)[8] == leader then - $hasleader = 1 -endi -if $hasleader != 1 then - goto step2 -endi +sql create database db vgroups 33 replica 3 sql use db; sql create table stb (ts timestamp, c int) tags (t int); @@ -122,6 +57,8 @@ sql select * from information_schema.ins_stables where db_name = 'db'; sql select * from information_schema.ins_tables where db_name = 'db'; sql show db.vgroups; +system sh/exec.sh -n dnode2 -s stop + return print ======== start back run_back tmp/back.sim diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index e3ef94e1cc..ec9cdf2666 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -1013,4 +1013,68 @@ sql_error select sum(cast(_wend as bigint)), a from ft1 state_window(a); sql_error create stream streams1 trigger at_once into streamt as select _wstart, sum(_wduration) from ft1 interval(10s); -sql_error create stream streams1 trigger at_once into streamt as select _wstart, sum(cast(_wend as bigint)) from ft1 interval(10s); \ No newline at end of file +sql_error create stream streams1 trigger at_once into streamt as select _wstart, sum(cast(_wend as bigint)) from ft1 interval(10s); + +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql insert into t1 values(1648791213000,1,1,3,1.0); +sql insert into t1 values(1648791223000,1,2,NULL,NULL); + +sleep 200 + +sql select apercentile(c, 50), apercentile(d, 50, "t-digest") from t1; + +if $data00 != 3.000000000 then + print ======data00=$data00 + return -1 +endi + +if $data01 != 1.000000000 then + print ======data01=$data01 + return -1 +endi + +sql select apercentile(c, 50) a, apercentile(d, 50, "t-digest") from t1 partition by b session(ts, 5s) order by a desc; + +if $data00 != 3.000000000 then + print ======data00=$data00 + return -1 +endi + +if $data01 != 1.000000000 then + print ======data01=$data01 + return -1 +endi + +if $data10 != NULL then + print ======data10=$data10 + return -1 +endi + +if $data11 != NULL then + print ======data11=$data11 + return -1 +endi + +sql select apercentile(c, 50) a, apercentile(d, 50, "t-digest") from t1 state_window(b) order by a desc; + +if $data00 != 3.000000000 then + print ======data00=$data00 + return -1 +endi + +if $data01 != 1.000000000 then + print ======data01=$data01 + return -1 +endi + +if $data10 != NULL then + print ======data10=$data10 + return -1 +endi + +if $data11 != NULL then + print ======data11=$data11 + return -1 +endi \ No newline at end of file diff --git a/tests/script/tsim/tag/tbNameIn.sim b/tests/script/tsim/tag/tbNameIn.sim new file mode 100644 index 0000000000..1af4bd6a9e --- /dev/null +++ b/tests/script/tsim/tag/tbNameIn.sim @@ -0,0 +1,102 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql use db1; +sql create stable st1 (ts timestamp, f1 int) tags(tg1 int); +sql create table tb1 using st1 tags(1); +sql create table tb2 using st1 tags(2); +sql create table tb3 using st1 tags(3); +sql create table tb4 using st1 tags(4); +sql create table tb5 using st1 tags(5); +sql create table tb6 using st1 tags(6); +sql create table tb7 using st1 tags(7); +sql create table tb8 using st1 tags(8); + +sql insert into tb1 values ('2022-07-10 16:31:01', 1); +sql insert into tb2 values ('2022-07-10 16:31:02', 2); +sql insert into tb3 values ('2022-07-10 16:31:03', 3); +sql insert into tb4 values ('2022-07-10 16:31:04', 4); +sql insert into tb5 values ('2022-07-10 16:31:05', 5); +sql insert into tb6 values ('2022-07-10 16:31:06', 6); +sql insert into tb7 values ('2022-07-10 16:31:07', 7); +sql insert into tb8 values ('2022-07-10 16:31:08', 8); + +sql select * from tb1 where tbname in ('tb1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from tb1 where tbname in ('tb1','tb1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from tb1 where tbname in ('tb1','tb2','tb1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from tb1 where tbname in ('tb1','tb2','st1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from tb1 where tbname = 'tb1'; +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from tb1 where tbname > 'tb1'; +if $rows != 0 then + return -1 +endi +sql select * from st1 where tbname in ('tb1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from st1 where tbname in ('tb1','tb1'); +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from st1 where tbname in ('tb1','tb2','tb1'); +if $rows != 2 then + return -1 +endi +sql select * from st1 where tbname in ('tb1','tb2','st1'); +if $rows != 2 then + return -1 +endi +sql select * from st1 where tbname = 'tb1'; +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +sql select * from st1 where tbname > 'tb1'; +if $rows != 7 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/cachemodel.py b/tests/system-test/0-others/cachemodel.py index 53834f792d..a8efd0d719 100644 --- a/tests/system-test/0-others/cachemodel.py +++ b/tests/system-test/0-others/cachemodel.py @@ -15,6 +15,7 @@ class TDTestCase: "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 88fbcd179d..619031fc29 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -23,6 +23,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index fe470c442e..04a9f5c941 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 6ce8872cda..b5d6a0d1a3 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -21,6 +21,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index b9ea39fc84..4ddae42ac5 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -21,6 +21,7 @@ from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db' diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index accb49dbfb..e3095e8b93 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -109,6 +109,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index e8271387a8..5e6a590806 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -111,6 +111,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 72c06f71be..781fcae638 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -111,6 +111,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a9907419e4..944ff52d5b 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -285,6 +285,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 83b4956c9b..f38efdb03c 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -45,6 +45,7 @@ class TDTestCase: # print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index ba265abc2d..d4698960cd 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -11,6 +11,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index 021b9aed2c..bc5d276faa 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -174,6 +174,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index a58e22ba1c..1f6096dd5a 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -14,6 +14,7 @@ import subprocess class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_cfg1.py b/tests/system-test/0-others/udf_cfg1.py index 9b1874d576..35d43ea9b7 100644 --- a/tests/system-test/0-others/udf_cfg1.py +++ b/tests/system-test/0-others/udf_cfg1.py @@ -16,6 +16,7 @@ class TDTestCase: "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":0} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index 67848a9430..b49c99de34 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -16,6 +16,7 @@ class TDTestCase: "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":1} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_cluster.py b/tests/system-test/0-others/udf_cluster.py index a20bc128a2..90b6df16ff 100644 --- a/tests/system-test/0-others/udf_cluster.py +++ b/tests/system-test/0-others/udf_cluster.py @@ -20,6 +20,7 @@ class MyDnodes(TDDnodes): class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None self.depoly_cluster(3) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 736943cc56..9426e7eacb 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -16,6 +16,7 @@ import threading class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 02c59c21e1..f6134909b4 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -13,6 +13,7 @@ import subprocess class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c6a1a7eac2..3918828646 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -155,6 +155,7 @@ class User: class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 6d9e04ec42..1918c0ef76 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -12,6 +12,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.buffer_boundary = [3,4097,8193,12289,16384] diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index 77ff6ae0f8..3e82b573b1 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -21,6 +21,7 @@ from util import constant from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index aed4089cfa..6a437d4601 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -22,6 +22,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/block_wise.py b/tests/system-test/1-insert/block_wise.py index bb35154746..0e17a01d05 100644 --- a/tests/system-test/1-insert/block_wise.py +++ b/tests/system-test/1-insert/block_wise.py @@ -143,6 +143,7 @@ class BSMAschema: class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.precision = "ms" diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index d16e506b53..e673815c73 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -45,6 +45,7 @@ NTBNAME = "nt1" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index c921619e56..862edbdde9 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -29,6 +29,7 @@ class TDTestCase: "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index fd0bd43b01..23bb539620 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -32,6 +32,7 @@ from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.special_name = ['!','@','#','$','%','^','&','*','(',')','[',']','{','}',\ diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index d935a08133..fe30f69229 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -24,8 +24,9 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 82db318c2b..6372502484 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -30,6 +30,7 @@ if platform.system().lower() == 'windows': class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index 9d033ff6e5..2356835ce8 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -63,6 +63,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/insert_drop.py b/tests/system-test/1-insert/insert_drop.py index d063501484..029d013d5b 100644 --- a/tests/system-test/1-insert/insert_drop.py +++ b/tests/system-test/1-insert/insert_drop.py @@ -9,6 +9,7 @@ import threading class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/1-insert/keep_expired.py b/tests/system-test/1-insert/keep_expired.py index 00cbc771c5..0cb6d841ed 100644 --- a/tests/system-test/1-insert/keep_expired.py +++ b/tests/system-test/1-insert/keep_expired.py @@ -8,6 +8,7 @@ import time class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.dbname = "test" diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 08f0ba8313..3e2bec130e 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -58,6 +58,7 @@ DATA_PRE2 = f"data2" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath diff --git a/tests/system-test/1-insert/mutipythonnodebugtaosd.py b/tests/system-test/1-insert/mutipythonnodebugtaosd.py index 1f568fdea1..042d881658 100644 --- a/tests/system-test/1-insert/mutipythonnodebugtaosd.py +++ b/tests/system-test/1-insert/mutipythonnodebugtaosd.py @@ -61,6 +61,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) # tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 9be51eb445..44243fe029 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -25,6 +25,7 @@ import json class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self._conn = conn diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index ab11691cf8..f588827206 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -29,6 +29,7 @@ if platform.system().lower() == 'windows': class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn diff --git a/tests/system-test/1-insert/table_comment.py b/tests/system-test/1-insert/table_comment.py index 2a8aa9b26a..270e0ca424 100644 --- a/tests/system-test/1-insert/table_comment.py +++ b/tests/system-test/1-insert/table_comment.py @@ -21,6 +21,7 @@ from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # prepare data diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index a184c51178..51d7d596cd 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -19,6 +19,7 @@ from util.common import * class TDTestCase: updatecfgDict = {'ttlUnit':5,'ttlPushInterval':3} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ntbname = 'ntb' diff --git a/tests/system-test/1-insert/tb_100w_data_order.py b/tests/system-test/1-insert/tb_100w_data_order.py index 85fe559cc2..ec166f5f6c 100644 --- a/tests/system-test/1-insert/tb_100w_data_order.py +++ b/tests/system-test/1-insert/tb_100w_data_order.py @@ -6,6 +6,7 @@ from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/1-insert/test_stmt_muti_insert_query.py b/tests/system-test/1-insert/test_stmt_muti_insert_query.py index 89beb30dc9..d75e04d2db 100644 --- a/tests/system-test/1-insert/test_stmt_muti_insert_query.py +++ b/tests/system-test/1-insert/test_stmt_muti_insert_query.py @@ -58,6 +58,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py index 535a65ffed..afd9d45b56 100644 --- a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py +++ b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py @@ -58,6 +58,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 1499c178d4..3d5c9197d1 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -135,6 +135,7 @@ class TDTestCase: updatecfgDict = {"querySmaOptimize": 1} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.precision = "ms" diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index d6b37db8b1..6fcd987ea4 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -21,6 +21,7 @@ from util.common import * from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/update_data_muti_rows.py b/tests/system-test/1-insert/update_data_muti_rows.py index af78ba5d19..67406f327e 100644 --- a/tests/system-test/1-insert/update_data_muti_rows.py +++ b/tests/system-test/1-insert/update_data_muti_rows.py @@ -24,6 +24,7 @@ from util.common import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 0e514cf43b..db97cdad64 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -8,6 +8,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py index c126ce926a..4e72c07b30 100644 --- a/tests/system-test/2-query/Timediff.py +++ b/tests/system-test/2-query/Timediff.py @@ -5,6 +5,7 @@ from util.gettime import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.get_time = GetTime() diff --git a/tests/system-test/2-query/To_iso8601.py b/tests/system-test/2-query/To_iso8601.py index bf4fe404aa..92aacbb350 100644 --- a/tests/system-test/2-query/To_iso8601.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -11,6 +11,7 @@ import os class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.rowNum = 10 diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index df99b08862..8ee2007450 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -11,6 +11,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 0f89a378be..08f6ba6baf 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -11,6 +11,7 @@ import pandas as pd class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 6bc9457264..d7478a55a8 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -15,12 +15,14 @@ class TDTestCase: # "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 self.row_nums = 20 self.ts = 1434938400000 self.time_step = 1000 + self.replicaVar = int(replicaVar) def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): tdLog.info(" prepare datas for auto check abs function ") @@ -28,7 +30,7 @@ class TDTestCase: stbname = f"{dbname}.stb" ctbname_pre = f"{dbname}.sub_tb_" - tdSql.execute(f" create database {dbname} ") + tdSql.execute(f" create database {dbname} replica {self.replicaVar} ") tdSql.execute(f" use {dbname} ") tdSql.execute(f" create stable {stbname} (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") @@ -125,7 +127,7 @@ class TDTestCase: def prepare_tag_datas(self, dbname="testdb"): # prepare datas tdSql.execute( - f"create database if not exists {dbname} keep 3650 duration 1000") + f"create database if not exists {dbname} keep 3650 duration 1000 replica {self.replicaVar} ") tdSql.execute(" use testdb ") tdSql.execute( f'''create table {dbname}.stb1 @@ -456,7 +458,7 @@ class TDTestCase: dbname = "bound_test" tdSql.execute(f"drop database if exists {dbname}") - tdSql.execute(f"create database if not exists {dbname}") + tdSql.execute(f"create database if not exists {dbname} replica {self.replicaVar} ") time.sleep(3) tdSql.execute(f"use {dbname}") tdSql.execute( @@ -589,7 +591,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.prepare() + tdSql.prepare(replica=f"{self.replicaVar}") tdLog.printNoPrefix("==========step1:create table ==============") diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 682d729f0e..479918f2f9 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -15,6 +15,7 @@ class TDTestCase: # "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index 8997238d7b..e190bf9374 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -19,6 +19,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),False) self.rowNum = 10 diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index 69e42a3911..ed717741c5 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -13,6 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 57d08d0587..71de088979 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -13,6 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index 64a29f7ccb..9780f9855b 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -13,6 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index e7086744f8..ec7ec34ed3 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -11,6 +11,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/between.py b/tests/system-test/2-query/between.py index dd7dda668b..719852ff27 100644 --- a/tests/system-test/2-query/between.py +++ b/tests/system-test/2-query/between.py @@ -10,6 +10,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py index 4c352b619f..76052301d9 100644 --- a/tests/system-test/2-query/bottom.py +++ b/tests/system-test/2-query/bottom.py @@ -22,6 +22,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index b5881221f4..b3969a9c45 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -13,6 +13,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.dbname = "db" diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index 221b571e8e..fffd484720 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -14,6 +14,7 @@ class TDTestCase: # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/char_length.py b/tests/system-test/2-query/char_length.py index e8546ca72e..e42e5e4f2c 100644 --- a/tests/system-test/2-query/char_length.py +++ b/tests/system-test/2-query/char_length.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/check_tsdb.py b/tests/system-test/2-query/check_tsdb.py index 0cbbc60171..b3ff259cc5 100644 --- a/tests/system-test/2-query/check_tsdb.py +++ b/tests/system-test/2-query/check_tsdb.py @@ -13,6 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index dc05a17185..326f6940f6 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat2.py b/tests/system-test/2-query/concat2.py index 1316a3a228..a7e61444fe 100644 --- a/tests/system-test/2-query/concat2.py +++ b/tests/system-test/2-query/concat2.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index 8443d41818..26731715c1 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat_ws2.py b/tests/system-test/2-query/concat_ws2.py index 38b93660e1..bf340826b6 100644 --- a/tests/system-test/2-query/concat_ws2.py +++ b/tests/system-test/2-query/concat_ws2.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index 2675c34266..d6bddc4e84 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -13,6 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 6362974d0b..254a8792af 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -4,6 +4,7 @@ from util.cases import * from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),False) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index 4ab51c4efe..f59376a979 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -5,6 +5,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index fb828b85df..e933eb54cb 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -27,6 +27,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index aea16dd162..7fe8b96157 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -12,6 +12,7 @@ import random class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 74dae9d0fd..d48a01db6a 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -7,6 +7,7 @@ import numpy as np class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index c7c6e1c9b0..5c07544d5d 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -10,6 +10,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index d2364df65a..897580fbcc 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -9,6 +9,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py index 4b5d3d8c5a..1cd24103f8 100644 --- a/tests/system-test/2-query/distribute_agg_avg.py +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index e6cf718c05..7d131cd77d 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -9,6 +9,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index 92188bd765..fb91216c3e 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index afc42755b3..2667798640 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index ab073c76a2..0247a91861 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -9,6 +9,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to execute {__file__}") tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_stddev.py b/tests/system-test/2-query/distribute_agg_stddev.py index ae9479dcdd..80bab3082d 100644 --- a/tests/system-test/2-query/distribute_agg_stddev.py +++ b/tests/system-test/2-query/distribute_agg_stddev.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_sum.py b/tests/system-test/2-query/distribute_agg_sum.py index 59804b7dc5..da26fd58f9 100644 --- a/tests/system-test/2-query/distribute_agg_sum.py +++ b/tests/system-test/2-query/distribute_agg_sum.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index d31df49a8e..dc89cd513d 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -21,6 +21,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/explain.py b/tests/system-test/2-query/explain.py index 1126e23f06..f164d3aedf 100644 --- a/tests/system-test/2-query/explain.py +++ b/tests/system-test/2-query/explain.py @@ -31,6 +31,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/first.py b/tests/system-test/2-query/first.py index 9523527076..5b5b596a57 100644 --- a/tests/system-test/2-query/first.py +++ b/tests/system-test/2-query/first.py @@ -24,6 +24,7 @@ import numpy as np class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 65974ff2e9..6a75872bcf 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -13,6 +13,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 7f11b12e89..4abef03037 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -27,6 +27,7 @@ from util.dnodes import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py index 3a74ca268f..44b86ee543 100644 --- a/tests/system-test/2-query/function_null.py +++ b/tests/system-test/2-query/function_null.py @@ -12,6 +12,7 @@ import random class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index 8f25595713..ad9b8b0d79 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -6,6 +6,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index ecbb7fe927..cc6e8fe7e9 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -145,6 +145,7 @@ class Hsgschema: class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/hyperloglog.py b/tests/system-test/2-query/hyperloglog.py index eff687cba7..d2cbe07b65 100644 --- a/tests/system-test/2-query/hyperloglog.py +++ b/tests/system-test/2-query/hyperloglog.py @@ -32,6 +32,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 780565c778..ce6b85c1cf 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -10,6 +10,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index 7b3ec10a8d..cee595d186 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -12,6 +12,7 @@ import random ,math class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 04b4fa89a2..120cc2eb30 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -64,6 +64,7 @@ class DataSet: class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 79cc62e4bc..7ddc5e57ba 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index c6f1e5076d..1c2a6055bc 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -14,6 +14,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index 06170e0adf..ea3963640a 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -31,6 +31,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index 0561dc9695..f4a1ab1790 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -9,6 +9,7 @@ import numpy as np class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 333c1516b3..f8d6ce4c6c 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -12,6 +12,7 @@ import random class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) self.tb_nums = 10 diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index 8870e416f8..8ece4c46f0 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -30,6 +30,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/length.py b/tests/system-test/2-query/length.py index 44c951dd4a..161ee20a9a 100644 --- a/tests/system-test/2-query/length.py +++ b/tests/system-test/2-query/length.py @@ -24,6 +24,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 11a0a9c317..7305a44f56 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -12,6 +12,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/lower.py b/tests/system-test/2-query/lower.py index f8ac1ab217..4499047a04 100644 --- a/tests/system-test/2-query/lower.py +++ b/tests/system-test/2-query/lower.py @@ -23,6 +23,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/ltrim.py b/tests/system-test/2-query/ltrim.py index ba769ba350..193aa45198 100644 --- a/tests/system-test/2-query/ltrim.py +++ b/tests/system-test/2-query/ltrim.py @@ -28,6 +28,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index 7f545f2048..ec4a087a8e 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -28,6 +28,7 @@ from util.dnodes import * dbname = 'db' class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index ee75afb501..b8da02b9a6 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -7,6 +7,7 @@ import numpy as np class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index f50b9fb2bb..a9b7a14eb0 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -5,6 +5,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py index 8849b14a33..bf9993da64 100644 --- a/tests/system-test/2-query/min.py +++ b/tests/system-test/2-query/min.py @@ -7,6 +7,7 @@ import numpy as np class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 4fbd840efe..b1008efa86 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -29,6 +29,7 @@ class TDTestCase: "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/percentile.py b/tests/system-test/2-query/percentile.py index 58660eec6a..935f55a8c2 100644 --- a/tests/system-test/2-query/percentile.py +++ b/tests/system-test/2-query/percentile.py @@ -22,6 +22,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index f803da6176..a067d66547 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -11,6 +11,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index 6f3856146d..d15ef89eb8 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -27,6 +27,7 @@ class TDTestCase: updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index e3f4732c08..fec553436c 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -65,6 +65,7 @@ class TDTestCase: # init def init(self, conn, logSql=True, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index 9fc016cb40..b8d4e3c314 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -17,6 +17,7 @@ from util.common import tdCom import random class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) ## add for TD-6672 tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 8f969a71d7..e3d98d6986 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -10,6 +10,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/rtrim.py b/tests/system-test/2-query/rtrim.py index e4835fbd8e..effb596e1d 100644 --- a/tests/system-test/2-query/rtrim.py +++ b/tests/system-test/2-query/rtrim.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index d1890d8623..a09d6a2b85 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -25,6 +25,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 46b1ac50ae..4fdec8fd73 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -11,6 +11,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 20ed103f8e..0390bae114 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -32,6 +32,7 @@ class TDTestCase: # init def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) tdSql.prepare() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 64fe92438c..676bc0c127 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/2-query/spread.py b/tests/system-test/2-query/spread.py index 79881d7293..61fe6793f9 100644 --- a/tests/system-test/2-query/spread.py +++ b/tests/system-test/2-query/spread.py @@ -31,6 +31,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 4784b71fc6..9229444f74 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -11,6 +11,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/stablity.py b/tests/system-test/2-query/stablity.py index 3c8d18e9e6..569bee62ec 100755 --- a/tests/system-test/2-query/stablity.py +++ b/tests/system-test/2-query/stablity.py @@ -29,6 +29,7 @@ class TDTestCase: "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 1d9a04cc73..2aa9194d37 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -13,6 +13,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.ts = 1420041600000 # 2015-01-01 00:00:00 this is begin time for first record diff --git a/tests/system-test/2-query/stateduration.py b/tests/system-test/2-query/stateduration.py index 009ec37764..9c37dc68e7 100644 --- a/tests/system-test/2-query/stateduration.py +++ b/tests/system-test/2-query/stateduration.py @@ -18,6 +18,7 @@ from util.sql import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/2-query/stddev.py b/tests/system-test/2-query/stddev.py index d044fb0dd7..1a2118a966 100644 --- a/tests/system-test/2-query/stddev.py +++ b/tests/system-test/2-query/stddev.py @@ -21,6 +21,7 @@ from util.common import * from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index 9b317aeab6..302711b13d 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -30,6 +30,7 @@ LENS = 6 class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(),False) diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py index ec1f60a720..27096fe5ad 100644 --- a/tests/system-test/2-query/sum.py +++ b/tests/system-test/2-query/sum.py @@ -25,6 +25,7 @@ DBNAME = "db" class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 3d5ecfaa9a..f925380c09 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -12,6 +12,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index 203f149712..27e6efb475 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -11,6 +11,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 917b98daa9..32d6ef98e9 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -8,6 +8,7 @@ from datetime import datetime from util.gettime import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) print(conn) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 0a2da114de..316e776cb1 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -12,6 +12,7 @@ if platform.system().lower() == 'windows': class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/top.py b/tests/system-test/2-query/top.py index 1cba18c2e2..d85e4dc768 100644 --- a/tests/system-test/2-query/top.py +++ b/tests/system-test/2-query/top.py @@ -20,6 +20,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index c4604799a1..106d43ea38 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -21,6 +21,7 @@ class TDTestCase: updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index fabd641da7..3ae602fa23 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -25,6 +25,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py index 1fd3b8fdda..16b9779fa8 100644 --- a/tests/system-test/2-query/twa.py +++ b/tests/system-test/2-query/twa.py @@ -10,6 +10,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 737817f262..82dcfe12e6 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union1.py b/tests/system-test/2-query/union1.py index 1ca25e7844..8db5ce01f3 100644 --- a/tests/system-test/2-query/union1.py +++ b/tests/system-test/2-query/union1.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union2.py b/tests/system-test/2-query/union2.py index c063b5c383..b30d699482 100644 --- a/tests/system-test/2-query/union2.py +++ b/tests/system-test/2-query/union2.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union3.py b/tests/system-test/2-query/union3.py index 3322f30359..7e2bdf6143 100644 --- a/tests/system-test/2-query/union3.py +++ b/tests/system-test/2-query/union3.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union4.py b/tests/system-test/2-query/union4.py index 2da83d8190..efb5b71c4d 100644 --- a/tests/system-test/2-query/union4.py +++ b/tests/system-test/2-query/union4.py @@ -27,6 +27,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index b954fd004b..6c51854b43 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -13,6 +13,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/upper.py b/tests/system-test/2-query/upper.py index cc5347774c..e28483dc56 100644 --- a/tests/system-test/2-query/upper.py +++ b/tests/system-test/2-query/upper.py @@ -23,6 +23,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index 3d5f443b3c..f0849010c6 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -10,6 +10,7 @@ from util.cases import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/5-taos-tools/TD-12478.py b/tests/system-test/5-taos-tools/TD-12478.py index 576e59f339..66f960338e 100644 --- a/tests/system-test/5-taos-tools/TD-12478.py +++ b/tests/system-test/5-taos-tools/TD-12478.py @@ -37,6 +37,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py b/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py index 8cb58b8d70..cbc9e83ba8 100644 --- a/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py +++ b/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py @@ -29,6 +29,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py index e591114ef4..f174975a8e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py +++ b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py @@ -31,6 +31,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() + self.replicaVar = int(replicaVar) def getBuildPath(self): @@ -118,6 +119,7 @@ class TDTestCase: rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] rowsall=rowsPerStb*paraDict['stbNumbers'] dbNumbers = 1 + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -167,8 +169,8 @@ class TDTestCase: threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) for tr in threads: tr.start() - dnode6Port=int(6030+5*100) - tdSql.execute("create dnode '%s:%d'"%(hostname,dnode6Port)) + dnode7Port=int(6030+6*100) + tdSql.execute("create dnode '%s:%d'"%(hostname,dnode7Port)) clusterComCheck.checkDnodes(dnodeNumbers) for tr in threads: tr.join() @@ -219,7 +221,7 @@ class TDTestCase: tdSql.checkRows(rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index 52f734b534..73c19b8cea 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -25,6 +25,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None + self.replicaVar = int(replicaVar) def buildcluster(self,dnodenumber): self.depoly_cluster(dnodenumber) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 52c9773544..5af9e55472 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -31,6 +31,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() + self.replicaVar = int(replicaVar) def getBuildPath(self): @@ -118,6 +119,7 @@ class TDTestCase: rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] rowsall=rowsPerStb*paraDict['stbNumbers'] dbNumbers = 1 + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -157,7 +159,7 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) - #insert date + #insert data for i in range(paraDict['stbNumbers']): stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() @@ -203,17 +205,19 @@ class TDTestCase: clusterComCheck.checkDbRows(dbNumbers) # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) - tdSql.execute("use %s" %(paraDict["dbName"])) - tdSql.query("show stables") - tdSql.checkRows(paraDict["stbNumbers"]) - # for i in range(paraDict['stbNumbers']): - # stableName= '%s_%d'%(paraDict['stbName'],i) - # tdSql.query("select * from %s"%stableName) - # tdSql.checkRows(rowsPerStb) + newTdSql=tdCom.newTdSql() + newTdSql.execute("reset query cache") + newTdSql.execute("use %s" %(paraDict["dbName"])) + newTdSql.query("show %s.stables"%(paraDict["dbName"])) + newTdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql.query("select * from %s"%stableName) + newTdSql.checkRows(rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index 4d641b0e27..1c488cab5f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -31,6 +31,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() + self.replicaVar = int(replicaVar) def getBuildPath(self): @@ -118,6 +119,7 @@ class TDTestCase: rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] rowsall=rowsPerStb*paraDict['stbNumbers'] dbNumbers = 1 + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -214,7 +216,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index eba456d1c2..93d60e6561 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -30,7 +30,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - + self.replicaVar=int(replicaVar) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -113,6 +113,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allDbNumbers=(paraDict['dbNumbers']*restartNumbers) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -198,7 +199,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=10,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=10,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 1949157191..62f4b248f9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -30,7 +30,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - + self.replicaVar = int(replicaVar) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -87,6 +87,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) dbNumbers = 1 + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -171,7 +172,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 9acb313203..397c6f5ccc 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -30,7 +30,8 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - self.replicaVar = replicaVar + self.replicaVar = int(replicaVar) + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -87,7 +88,8 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allDbNumbers=(paraDict['dbNumbers']*restartNumbers) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) - paraDict['replica']=int(self.replicaVar) + paraDict['replica'] = self.replicaVar + tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) @@ -171,7 +173,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='mnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index 14ae1a8a48..e3f1a91de8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -30,7 +30,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - + self.replicaVar = int(replicaVar) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -112,6 +112,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) dbNumbers = 1 + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -197,7 +198,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='mnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='mnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 056e3225e8..4ff6bffc07 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -30,6 +30,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() + self.replicaVar = int(replicaVar) def getBuildPath(self): @@ -88,6 +89,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allDbNumbers=(paraDict['dbNumbers']*restartNumbers) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") @@ -171,7 +173,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=10,stopRole='vnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=10,stopRole='vnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 5fd79e4296..4bd8628a66 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -30,7 +30,7 @@ class TDTestCase: self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - print(tdSql) + self.replicaVar = int(replicaVar) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -112,8 +112,8 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) dbNumbers = 1 + paraDict['replica'] = self.replicaVar - print(tdSql) tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) @@ -175,7 +175,7 @@ class TDTestCase: # dnodeNumbers don't include database of schema if clusterComCheck.checkDnodes(dnodeNumbers): - tdLog.info("123") + tdLog.info("check numbers of dnodes right ") else: print("456") @@ -192,14 +192,15 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") - tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) # # tdLog.info("check Stable Rows:") - tdSql.checkRows(allStbNumbers) - + if self.replicaVar==1: + tdSql.checkRows(allStbNumbers) + else: + tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='vnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='vnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 28c4810833..7cbe9d9680 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -47,13 +47,14 @@ class ClusterComCheck: for i in range(dnodeNumbers): if tdSql.queryResult[i][4] == "ready": status+=1 - tdLog.info(status) + # tdLog.info(status) if status == dnodeNumbers: - tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count)) + tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count+1)) return True - count+=1 time.sleep(1) + count+=1 + else: tdSql.query("select * from information_schema.ins_dnodes") tdLog.debug(tdSql.queryResult) @@ -74,10 +75,10 @@ class ClusterComCheck: tdLog.debug(tdSql.queryResult) tdLog.exit("we find %d databases but expect %d in clusters! " %(tdSql.queryRows,dbNumbers)) - def checkDb(self,dbNumbers,restartNumber,dbNameIndex): + def checkDb(self,dbNumbers,restartNumber,dbNameIndex, timeout=100): count=0 alldbNumbers=(dbNumbers*restartNumber)+2 - while count < 5: + while count < timeout: query_status=0 for j in range(dbNumbers): for i in range(alldbNumbers): @@ -87,22 +88,24 @@ class ClusterComCheck: query_status+=1 tdLog.debug("check %s_%d that status is ready "%(dbNameIndex,j)) else: + sleep(1) continue # print(query_status) - count+=1 if query_status == dbNumbers: - tdLog.success(" check %d database and all databases are ready within 5s! " %dbNumbers) + tdLog.success(" check %d database and all databases are ready within %ds! " %(dbNumbers,count+1)) return True + count+=1 + else: tdLog.debug(tdSql.queryResult) tdLog.debug("query status is %d"%query_status) - tdLog.exit("database is not ready within 5s") + tdLog.exit("database is not ready within %ds"%(timeout+1)) def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,): tdSql.execute("use %s"%dbname) - tdSql.query("show stables") + tdSql.query("show %s.stables"%dbname) tdSql.checkRows(stableCount) - tdSql.query("show tables") + tdSql.query("show %s.tables"%dbname) tdSql.checkRows(CtableCount) for i in range(stableCount): tdSql.query("select count(*) from %s%d"%(stbname,i)) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py index 435701bd8f..9bdc0a2cf4 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py index f0d5e8328a..d33a1b0d27 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py index 8d0801500e..75e01977fd 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py index 379615d358..77dcab90bf 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py index 240169dbb6..1a2c31a311 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py index 25b5de5afa..a9fb9555e8 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py index ab5359601a..2eb631d433 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py index 4eec3c348d..6102a82b04 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py index 0fe18ab705..d87ec3d35e 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py index 75a7ca51b6..fc6d3c0683 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py index e9c63151f3..00b808b8b4 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index 5dd57542b3..e64649189d 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py index fd00e89216..b633887009 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index f98e2c07b5..6415da94b4 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -20,6 +20,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py index 5be2b67c31..1b99c1e92b 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -20,6 +20,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py index e8fa3099e9..ea5c4679a9 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py @@ -19,6 +19,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py index 21d0c2d2b8..1dcaae452e 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py index ff6ce98c81..945fcf2990 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py index e36f488c2b..6b87bee5a3 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py index 050b98024d..8ef151a385 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py @@ -22,6 +22,7 @@ sys.path.append(os.path.dirname(__file__)) class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py index d086743306..49e5cafe96 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -18,6 +18,7 @@ import subprocess class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py index b9d3afa765..20cf7c583a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -20,6 +20,7 @@ import subprocess class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index d1985c6567..7d42a3e81f 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -27,6 +27,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index dbdcedcbe8..39d519fec1 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -14,6 +14,7 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 484969010c..808a4935e3 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index ed22df2b07..8386c22cd0 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index 17668d8d03..04f5aac559 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index af6c51d947..4371a909c2 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -27,6 +27,7 @@ class TDTestCase: self.rowsPerTbl = 2 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 2a67cd1bc1..04224fbc7e 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 0f0e7c5287..6b48a6d570 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 0ef454f3e2..c4a7c8cae5 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 0ec114d3bd..c380d201b2 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 3790bd1690..fd06eedefd 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index 81d824156d..d4c5e2f87f 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 02e72d0475..ea78c90abd 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb2.py b/tests/system-test/7-tmq/subscribeDb2.py index edf1a8ec80..d045842e45 100644 --- a/tests/system-test/7-tmq/subscribeDb2.py +++ b/tests/system-test/7-tmq/subscribeDb2.py @@ -22,6 +22,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 34df61dc09..819588badc 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 27efbee016..7f5169361c 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -52,6 +52,7 @@ class TDTestCase: hostname = socket.gethostname() def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 213f9aa89c..3ff0b25ff6 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 0c188754b2..1463cad627 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 4d5407e927..edbe1bc3c6 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index bb38a981d3..6c3e122902 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 32272491c6..025f403282 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index e347d27ac1..6aa3da66a4 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 837317e5d7..a6bf01aa06 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -33,6 +33,7 @@ class TDTestCase: self.dnodeOfLeader = 0 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index 3bff0e4754..a70678219f 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -33,6 +33,7 @@ class TDTestCase: self.dnodeOfLeader = 0 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 568e49388b..41073d83ae 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 9995af15c2..04d0744ab5 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index 5b055cf725..b9dac62833 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index e71e4d257d..c153e94caa 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -38,6 +38,7 @@ from util.common import * class TMQCom: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdSql.init(conn.cursor()) # tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index 655097c924..f372a2b742 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 100000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index 709464fc1e..c7f95f6f41 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 100000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 09ad6e38c9..26dacf514d 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 150 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index 95e060d581..d6f100041b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 70 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index fc3437613b..9bfc01529f 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 73af4d196f..975f89cbd7 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 20db44ef19..6a03f0f751 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index bd42f74c03..009862137f 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 100000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index 2eb1a7c52a..95ce03e653 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 150 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index 351a10ed1f..8ff01444f8 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 70 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 6504274993..528b3a8088 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index 2cfb6581d8..c0c459d315 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index 5c7d21069b..02093a2d88 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 7509e43af7..4b45b1a834 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index ded6fe94d7..3b72b4aea5 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDnode.py b/tests/system-test/7-tmq/tmqDnode.py index 921e543e2d..e1d6d91e2d 100644 --- a/tests/system-test/7-tmq/tmqDnode.py +++ b/tests/system-test/7-tmq/tmqDnode.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 205f513351..a44ff916e5 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index 0d75df58f9..6c49fae299 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 79cb79a83c..3fc5a2fdc7 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 7679f0ca44..3789632984 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -52,6 +52,7 @@ class TDTestCase: hostname = socket.gethostname() def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index 95780538d1..c9e34136cc 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 5c858d0dce..164e7f1c8c 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -30,6 +30,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqModule.py b/tests/system-test/7-tmq/tmqModule.py index 0063edce56..d6b4aff938 100644 --- a/tests/system-test/7-tmq/tmqModule.py +++ b/tests/system-test/7-tmq/tmqModule.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 0a21680e78..406fd9f7f3 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 7e14f2568d..5602876893 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -28,6 +28,7 @@ class TDTestCase: self.rowsPerTbl = 100 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index 8158249fa6..297429b495 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -24,6 +24,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index 97e50cf654..9c139b50de 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -24,6 +24,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index d081423142..8593fd4f1e 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -24,6 +24,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index a95920b7c4..b974e4a41a 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 0ced8fc34b..d5df88cf43 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -23,6 +23,7 @@ class TDTestCase: self.autoCtbPrefix = 'aCtb' def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index bdcbb9578e..6b8c10de27 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -23,6 +23,7 @@ class TDTestCase: self.autoCtbPrefix = 'aCtb' def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index df150130c0..3975013e74 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -23,6 +23,7 @@ class TDTestCase: self.autoCtbPrefix = 'aCtb' def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index dbeac427ac..49a475ff16 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -22,6 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index fd7e1b7bb1..f2fbd84865 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index e45a54c272..db06700284 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index 02654b5703..51934ccd44 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -20,6 +20,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15557.py b/tests/system-test/99-TDcase/TD-15557.py index ca29e1282f..884c028a65 100644 --- a/tests/system-test/99-TDcase/TD-15557.py +++ b/tests/system-test/99-TDcase/TD-15557.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index 6b37f25bb6..5ea652b4fb 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -21,6 +21,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-16025.py b/tests/system-test/99-TDcase/TD-16025.py index 0adc313429..8c9fa9319f 100644 --- a/tests/system-test/99-TDcase/TD-16025.py +++ b/tests/system-test/99-TDcase/TD-16025.py @@ -28,6 +28,7 @@ class TDTestCase: #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index b2b5e3f425..f57fae752f 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -16,6 +16,7 @@ from tmqCommon import * class TDTestCase: def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index bcf02b654f..0f83468754 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -21,6 +21,7 @@ class TDTestCase: self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index d4c4a4cc32..2862f4a78d 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -53,6 +53,7 @@ class TDTestCase: hostname = socket.gethostname() def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/99-TDcase/TD-19201.py b/tests/system-test/99-TDcase/TD-19201.py index 0fd86bf6b4..2a25d1ad7e 100644 --- a/tests/system-test/99-TDcase/TD-19201.py +++ b/tests/system-test/99-TDcase/TD-19201.py @@ -14,6 +14,7 @@ class TDTestCase: hostname = socket.gethostname() def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8529f8d99b..febcc4b728 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -236,22 +236,29 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 6 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 -M 3 -# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartMnodeInsertData.py -N 5 -M 3 -# python3 ./test.py -f 6-cluster/5dnode3mnodeRestartVnodeInsertData.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 + + +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # TD-19646 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3