diff --git a/cmake/cmake.version b/cmake/cmake.version index 92f7f8b895..03598519ed 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -65,13 +65,14 @@ ELSE () ENDIF () MESSAGE(STATUS "============= compile version parameter information start ============= ") -MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) -MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) -MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) -MESSAGE(STATUS "build date:" ${TD_VER_DATE}) -MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) -MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) -MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "version: " ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible: " ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "commit id: " ${TD_VER_GIT}) +MESSAGE(STATUS "build date: " ${TD_VER_DATE}) +MESSAGE(STATUS "build type: " ${CMAKE_BUILD_TYPE}) +MESSAGE(STATUS "type: " ${TD_VER_VERTYPE}) +MESSAGE(STATUS "cpu: " ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os: " ${TD_VER_OSTYPE}) MESSAGE(STATUS "============= compile version parameter information end ============= ") STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c507ae2536..79d54f522e 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 8c3d57d + GIT_TAG 0d5663d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e0d5250d84..d18d85171d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 0fb640b + GIT_TAG a921bd4 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 67fecdbdc7..0c55578efa 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -334,6 +334,20 @@ If no comparison or condition is true, the result after ELSE is returned, or NUL The return type of the CASE expression is the result type of the first WHEN WHEN part, and the result type of the other WHEN WHEN parts and ELSE parts can be converted to them, otherwise TDengine will report an error. +### Examples + +A device has three status codes to display its status. The statements are as follows: + +```sql +SELECT CASE dev_status WHEN 1 THEN 'Running' WHEN 2 THEN 'Warning' WHEN 3 THEN 'Downtime' ELSE 'Unknown' END FROM dev_table; +``` + +The average voltage value of the smart meter is counted. When the voltage is less than 200 or more than 250, it is considered that the statistics is wrong, and the value is corrected to 220. The statement is as follows: + +```sql +SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) FROM meters; +``` + ## JOIN TDengine supports natural joins between supertables, between standard tables, and between subqueries. The difference between natural joins and inner joins is that natural joins require that the fields being joined in the supertables or standard tables must have the same name. Data or tag columns must be joined with the equivalent column in another table. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index aadf563a63..bb5516ae70 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy | Applicable | Server only | | Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. | Value Range | 0: monitoring disabled, 1: monitoring enabled | -| Default | 0 | +| Default | 1 | ### monitorFqdn diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 83af412fe1..9d4faae23a 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -325,14 +325,28 @@ CASE WHEN condition THEN result [WHEN condition THEN result ...] [ELSE result] E ``` ### 说明 + TDengine 通过 CASE 表达式让用户可以在 SQL 语句中使用 IF ... THEN ... ELSE 逻辑。 + 第一种 CASE 语法返回第一个 value 等于 compare_value 的 result，如果没有 compare_value 符合，则返回 ELSE 之后的 result，如果没有 ELSE 部分，则返回 NULL。 第二种语法返回第一个 condition 为真的 result。 如果没有 condition 符合，则返回 ELSE 之后的 result，如果没有 ELSE 部分，则返回 NULL。 CASE 表达式的返回类型为第一个 WHEN THEN 部分的 result 类型，其余 WHEN THEN 部分和 ELSE 部分，result 类型都需要可以向其转换，否则 TDengine 会报错。 +### 示例 +某设备有三个状态码，显示其状态，语句如下： + +```sql +SELECT CASE dev_status WHEN 1 THEN 'Running' WHEN 2 THEN 'Warning' WHEN 3 THEN 'Downtime' ELSE 'Unknown' END FROM dev_table; +``` + +统计智能电表的电压平均值，当电压小于 200 或大于 250 时认为是统计有误，修正其值为 220，语句如下： + +```sql +SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) FROM meters; +``` ## JOIN 子句 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 54c362de95..145c5eed93 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -106,7 +106,7 @@ taos --dump-config | 适用范围 | 仅服务端适用 | | 含义 | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况，包括 CPU、内存、硬盘、网络带宽的监控记录，监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 | | 取值范围 | 0：关闭监控服务， 1：激活监控服务。 | -| 缺省值 | 0 | +| 缺省值 | 1 | ### monitorFqdn diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fdbb2c5123..502ba10d33 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -244,7 +244,7 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols); const char* blockDecode(SSDataBlock* pBlock, const char* pData); void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 60e8f63c22..2eda2f66cc 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -74,9 +74,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema * void tTSchemaDestroy(STSchema *pTSchema); // SValue ================================ -int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type); -int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); -int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) @@ -132,8 +130,9 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) void tColDataClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); -uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal); +uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // STRUCT ================================ struct STColumn { @@ -282,6 +281,15 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { + if (IS_VAR_DATA_TYPE(type)) { + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + } else { + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; + } +} + #endif #ifdef __cplusplus diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 643588de19..b16b5a2d4b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -190,6 +190,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SBuildUseDBInput; typedef struct SField { @@ -838,6 +839,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; // ms } SUseDbReq; int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); @@ -852,6 +854,8 @@ typedef struct { int16_t hashSuffix; int8_t hashMethod; SArray* pVgroupInfos; // Array of SVgroupInfo + int32_t errCode; + int64_t stateTs; // ms } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, const SUseDbRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3902224f0b..6154882756 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -56,6 +56,7 @@ typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; int64_t dbId; + int64_t stateTs; } SDbInfo; typedef struct STablesReq { @@ -124,6 +125,7 @@ typedef struct SDbVgVersion { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; } SDbVgVersion; typedef struct STbSVersion { @@ -152,7 +154,7 @@ int32_t catalogInit(SCatalogCfg* cfg); */ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle); -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum); +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* stateTs); /** * Get a DB's all vgroup info. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 74c1a8c07c..96ccef7164 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -127,6 +127,7 @@ typedef struct SDBVgInfo { int16_t hashSuffix; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT + int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo } SDBVgInfo; @@ -203,6 +204,8 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +void destroyAhandle(void* ahandle); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 225466a015..9443df5e14 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -60,19 +60,19 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number); -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); int32_t streamStateSessionClear(SStreamState* pState); -int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); -int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey); +int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); -SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); @@ -99,7 +99,9 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); -// char* streamStateSessionDump(SStreamState* pState); +#if 0 +char* streamStateSessionDump(SStreamState* pState); +#endif #ifdef __cplusplus } diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 74a73f6b10..a27be95049 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -139,6 +139,7 @@ typedef struct SSyncFSM { void (*FpReConfigCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, const SRpcMsg* pMsg, const SFsmCbMeta* pMeta); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); + int32_t (*FpApplyQueueItems)(const struct SSyncFSM* pFsm); void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm); void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 53edcd4fee..8a0cccc71f 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -72,6 +72,7 @@ typedef struct SRpcMsg { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -97,6 +98,9 @@ typedef struct SRpcInit { // set up timeout for particular msg RpcTfp tfp; + // destroy client ahandle; + RpcDfp dfp; + void *parent; } SRpcInit; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 447a90447a..a03dc7d9f9 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -154,7 +154,7 @@ int32_t* taosGetErrno(); // mnode-sdb #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) +#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // internal #define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) #define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) #define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) @@ -218,14 +218,24 @@ int32_t* taosGetErrno(); // mnode-db #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) // +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) // #define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) +// #define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) // 2.x #define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) -#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) -#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) +#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) // +// #define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) // 2.x +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) // +#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) // internal +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) +// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x +// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x0393) // 2.x +// #define TSDB_CODE_MND_INVALID_TOPIC_PARTITONSTAOS_DEF_ERROR_CODE(0, 0x0394) // 2.x +// #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) // 2.x +#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) // mnode-node diff --git a/include/util/tdef.h b/include/util/tdef.h index 6e1fa87854..b6a7f5c824 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -290,7 +290,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB -#define TSDB_DEFAULT_BUFFER_PER_VNODE 96 +#define TSDB_DEFAULT_BUFFER_PER_VNODE 256 #define TSDB_MIN_PAGES_PER_VNODE 64 #define TSDB_MAX_PAGES_PER_VNODE (INT32_MAX - 1) #define TSDB_DEFAULT_PAGES_PER_VNODE 256 diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 5b79345916..5dc8024f29 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -13,7 +13,7 @@ def sync_source(branch_name) { git branch git pull || git pull git log | head -n 20 - git submodule update --init --recursive + git clean -fxd ''' return 1 } @@ -37,19 +37,24 @@ def build_run() { pipeline { agent none parameters { + choice( + name: 'sourcePath', + choices: ['nas','web'], + description: 'choice which way to download the installation pacakge;web is Office Web and nas means taos nas server ' + ) string ( name:'version', - defaultValue:'3.0.0.1', + defaultValue:'3.0.1.6', description: 'release version number,eg: 3.0.0.1 or 3.0.0.' ) string ( name:'baseVersion', - defaultValue:'3.0.0.1', + defaultValue:'3.0.1.6', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) string ( name:'toolsVersion', - defaultValue:'2.1.2', + defaultValue:'2.2.7', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) string ( @@ -62,7 +67,7 @@ pipeline { WORK_DIR = '/var/lib/jenkins/workspace' TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' - BRANCH_NAME = 'test/chr/TD-14699' + BRANCH_NAME = '3.0' TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" @@ -104,17 +109,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -127,17 +132,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py dpkg -r tdengine ''' @@ -152,17 +157,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -175,17 +180,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py sudo rpm -e tdengine ''' @@ -199,7 +204,7 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server + bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${sourcePath} python3 checkPackageRuning.py ''' } @@ -215,7 +220,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.21 ''' } @@ -227,7 +232,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.24 ''' } @@ -241,7 +246,7 @@ pipeline { timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client + bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client ${sourcePath} python3 checkPackageRuning.py 192.168.0.21 ''' } diff --git a/packaging/release.bat b/packaging/release.bat index 4c82c5ead5..4ab7297f03 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -39,7 +39,7 @@ if not exist %work_dir%\debug\ver-%2-x86 ( md %work_dir%\debug\ver-%2-x86 ) cd %work_dir%\debug\ver-%2-x64 -call vcvarsall.bat x64 +rem #call vcvarsall.bat x64 cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64 cmake --build . rd /s /Q C:\TDengine diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 5b9a12179d..b0e33835dd 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -6,6 +6,8 @@ version=$2 originPackageName=$3 originversion=$4 testFile=$5 +# sourcePath:web/nas +sourcePath=$6 subFile="taos.tar.gz" # Color setting @@ -71,16 +73,23 @@ fi function wgetFile { file=$1 - -if [ ! -f ${file} ];then - echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" - wget https://www.taosdata.com/assets-download/3.0/${file} -else - echoColor YD "${file} already exists and use new file " +versionPath=$2 +sourceP=$3 +nasServerIP="192.168.1.131" +packagePath="/nas/TDengine/v${versionPath}/community" +if [ -f ${file} ];then + echoColor YD "${file} already exists ,it will delete it and download it again " rm -rf ${file} - echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" - wget https://www.taosdata.com/assets-download/3.0/${file} fi + +if [ ${sourceP} = 'web' ];then + echoColor BD "====download====:wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} +elif [ ${sourceP} = 'nas' ];then + echoColor BD "====download====:scp root@${nasServerIP}:${packagePath}/${file} ." + scp root@${nasServerIP}:${packagePath}/${file} . +fi + } function newPath { @@ -142,8 +151,9 @@ if [ -d ${installPath}/${tdPath} ] ;then fi echoColor G "===== download installPackage =====" -cd ${installPath} && wgetFile ${packgeName} -cd ${oriInstallPath} && wgetFile ${originPackageName} +cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} +cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} + cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . @@ -193,7 +203,7 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath} if [ ! -f {originPackageName} ];then echoColor YD "download base installPackage" - wgetFile ${originPackageName} + wgetFile ${originPackageName} ${originversion} ${sourcePath} fi echoColor YD "unzip the base installation package" echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} @@ -238,14 +248,19 @@ cd ${installPath} if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-x64.tar.gz . - tar xf taosTools-2.1.3-Linux-x64.tar.gz + if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then + wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web + tar xf taosTools-2.1.3-Linux-x64.tar.gz + fi cd taosTools-2.1.3 && bash install-taostools.sh elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} - wgetFile taosTools-2.1.3-Linux-arm64.tar.gz . - tar xf taosTools-2.1.3-Linux-arm64.tar.gz + if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then + wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web + tar xf taosTools-2.1.3-Linux-arm64.tar.gz + fi + cd taosTools-2.1.3 && bash install-taostools.sh fi diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index f5d1e45690..c19519f9a1 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -1,4 +1,7 @@ @echo off + +for /F %%a in ('echo prompt $E ^| cmd') do set "ESC=%%a" + goto %1 :needAdmin @@ -11,60 +14,94 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 -set tagert_dir=C:\\TDengine +set target_dir=C:\\TDengine -if not exist %tagert_dir% ( - mkdir %tagert_dir% +if not exist %target_dir% ( + mkdir %target_dir% ) -if not exist %tagert_dir%\\cfg ( - mkdir %tagert_dir%\\cfg +if not exist %target_dir%\\cfg ( + mkdir %target_dir%\\cfg ) -if not exist %tagert_dir%\\include ( - mkdir %tagert_dir%\\include +if not exist %target_dir%\\include ( + mkdir %target_dir%\\include ) -if not exist %tagert_dir%\\driver ( - mkdir %tagert_dir%\\driver +if not exist %target_dir%\\driver ( + mkdir %target_dir%\\driver ) if not exist C:\\TDengine\\cfg\\taos.cfg ( - copy %source_dir%\\packaging\\cfg\\taos.cfg %tagert_dir%\\cfg\\taos.cfg > nul + copy %source_dir%\\packaging\\cfg\\taos.cfg %target_dir%\\cfg\\taos.cfg > nul ) if exist %binary_dir%\\test\\cfg\\taosadapter.toml ( - if not exist %tagert_dir%\\cfg\\taosadapter.toml ( - copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul + if not exist %target_dir%\\cfg\\taosadapter.toml ( + copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul ) ) -copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul -copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul -copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul -copy %binary_dir%\\build\\lib\\taos.lib %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\lib\\taos_static.lib %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\lib\\taos.dll %tagert_dir%\\driver > nul -copy %binary_dir%\\build\\bin\\taos.exe %tagert_dir% > nul -copy %binary_dir%\\build\\bin\\taosd.exe %tagert_dir% > nul -copy %binary_dir%\\build\\bin\\udfd.exe %tagert_dir% > nul +copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul +copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul +copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul +copy %binary_dir%\\build\\lib\\taos.lib %target_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos_static.lib %target_dir%\\driver > nul +copy %binary_dir%\\build\\lib\\taos.dll %target_dir%\\driver > nul +copy %binary_dir%\\build\\bin\\taos.exe %target_dir% > nul if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( - copy %binary_dir%\\build\\bin\\taosBenchmark.exe %tagert_dir% > nul + copy %binary_dir%\\build\\bin\\taosBenchmark.exe %target_dir% > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll.lib ( - copy %binary_dir%\\build\\lib\\taosws.dll.lib %tagert_dir%\\driver > nul + copy %binary_dir%\\build\\lib\\taosws.dll.lib %target_dir%\\driver > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll ( - copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver > nul - copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %tagert_dir%\\include > nul + copy %binary_dir%\\build\\lib\\taosws.dll %target_dir%\\driver > nul + copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %target_dir%\\include > nul ) if exist %binary_dir%\\build\\bin\\taosdump.exe ( - copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul -) -if exist %binary_dir%\\build\\bin\\taosadapter.exe ( - copy %binary_dir%\\build\\bin\\taosadapter.exe %tagert_dir% > nul + copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) -mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)&& echo To start/stop TDengine with administrator privileges: sc start/stop taosd &goto :eof +copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul +copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul + +if exist %binary_dir%\\build\\bin\\taosadapter.exe ( + copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul +) + +mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close) + +echo. +echo Please manually remove C:\TDengine from your system PATH environment after you remove TDengine software +echo. +echo To start/stop TDengine with administrator privileges: %ESC%[92msc start/stop taosd %ESC%[0m + +if exist %binary_dir%\\build\\bin\\taosadapter.exe ( + echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m +) + +goto :eof + :hasAdmin + +sc query "taosd" && sc stop taosd && sc delete taosd +sc query "taosadapter" && sc stop taosadapter && sc delete taosd + copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul if exist C:\\TDengine\\driver\\taosws.dll ( copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul ) + sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND + +set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment" +for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\"') do ( + + rem // make addition persistent through reboots + reg add "%env%" /f /v Path /t REG_EXPAND_SZ /d "%%J;C:\TDengine" + + rem // apply change to the current process + for %%a in ("%%J;C:\TDengine") do path %%~a +) + +rem // use setx to set a temporary throwaway value to trigger a WM_SETTINGCHANGE +rem // applies change to new console windows without requiring a reboot +(setx /m foo bar & reg delete "%env%" /f /v foo) >NUL 2>NUL + diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index ec9c432092..981bee91b8 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -60,6 +60,29 @@ Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: i Source: {#MyAppSourceDir}{#MyAppExeName}; DestDir: "{app}"; Excludes: {#MyAppExcludeSource} ; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs + +[Registry] +Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ + ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\TDengine"; \ + Check: NeedsAddPath('C:\TDengine') + +[Code] +function NeedsAddPath(Param: string): boolean; +var + OrigPath: string; +begin + if not RegQueryStringValue(HKEY_LOCAL_MACHINE, + 'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', + 'Path', OrigPath) + then begin + Result := True; + exit; + end; + { look for the path with leading and trailing semicolon } + { Pos() returns 0 if not found } + Result := Pos(';' + Param + ';', ';' + OrigPath + ';') = 0; +end; + [UninstallDelete] Name: {app}\driver; Type: filesandordirs Name: {app}\connector; Type: filesandordirs diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb25a595af..bb93e4d934 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -140,12 +140,13 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; - // rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index fa0e3212a3..bf34d3e2df 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -61,7 +61,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); @@ -72,6 +72,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } vgInfo->vgVersion = rsp->vgVersion; + vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; vgInfo->hashPrefix = rsp->hashPrefix; vgInfo->hashSuffix = rsp->hashSuffix; @@ -486,6 +487,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); db->numOfTable = htonl(db->numOfTable); + db->stateTs = htobe64(db->stateTs); } SKv kv = { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 71a87a4b54..efa7d095c5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -990,7 +990,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // all data has returned to App already, no need to try again if (pResultInfo->completed) { // it is a local executed query, no need to do async fetch - if (QUERY_EXEC_MODE_LOCAL == pRequest->body.execMode) { + if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { if (pResultInfo->localResultFetched) { pResultInfo->numOfRows = 0; pResultInfo->current = 0; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 938cc4e41d..8437c93bae 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -175,12 +175,13 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; - if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); struct SCatalog* pCatalog = NULL; - if (usedbRsp.vgVersion >= 0) { + if (usedbRsp.vgVersion >= 0) { // cached in local uint64_t clusterId = pRequest->pTscObj->pAppInfo->clusterId; int32_t code1 = catalogGetHandle(clusterId, &pCatalog); if (code1 != TSDB_CODE_SUCCESS) { @@ -212,7 +213,20 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); if (strlen(usedbRsp.db) == 0) { - return TSDB_CODE_MND_DB_NOT_EXIST; + if (usedbRsp.errCode != 0) { + return usedbRsp.errCode; + } else { + return TSDB_CODE_APP_ERROR; + } + } + + tscTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo* pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); + tscTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); + for (int32_t j = 0; j < pInfo->epSet.numOfEps; ++j) { + tscTrace("vgId:%d, index:%d epset:%s:%u", pInfo->vgId, j, pInfo->epSet.eps[j].fqdn, pInfo->epSet.eps[j].port); + } } SName name = {0}; @@ -428,8 +442,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5d0121aa15..536cbed33e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2197,7 +2197,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return rname.ctbShortName; } -void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { +int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { + int32_t dataLen = 0; + // todo extract method int32_t* version = (int32_t*)data; *version = 1; @@ -2238,7 +2240,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = blockDataGetSerialMetaSize(numOfCols); + dataLen = blockDataGetSerialMetaSize(numOfCols); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -2255,26 +2257,23 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ } data += metaSize; - (*dataLen) += metaSize; + dataLen += metaSize; - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } + colSizes[col] = colDataGetLength(pColRes, numOfRows); + dataLen += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); } - *actualLen = *dataLen; + *actualLen = dataLen; *groupId = pBlock->info.groupId; - ASSERT(*dataLen > 0); - uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); + ASSERT(dataLen > 0); + + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols); + + return dataLen; } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 513c54c7e9..73b887342c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -56,7 +56,7 @@ typedef struct { #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) // SValue -int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { +static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { if (IS_VAR_DATA_TYPE(type)) { return tPutBinary(p, pValue->pData, pValue->nData); } else { @@ -65,20 +65,6 @@ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { } } -int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - if (IS_VAR_DATA_TYPE(type)) { - return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); - } else { - memcpy(&pValue->val, p, tDataTypes[type].bytes); - return tDataTypes[type].bytes; - } -} - -int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { - // TODO - return 0; -} - // STSRow2 ======================================================================== static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { if (pb) { @@ -923,7 +909,7 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { - if(!pTag || !pTagVal){ + if (!pTag || !pTagVal) { return false; } @@ -1164,31 +1150,27 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type); + memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + pColData->nData += tDataTypes[pColData->type].bytes; } + pColData->nVal++; _exit: return code; } static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_VALUE; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_NONE; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->flag = HAS_NULL; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1216,16 +1198,11 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1268,11 +1245,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1290,9 +1263,8 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p return code; } static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; pColData->nVal++; - return code; + return 0; } static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1325,11 +1297,7 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1353,15 +1321,7 @@ static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *p return code; } -static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; - - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - pColData->nVal++; - - return code; -} +#define tColDataAppendValue40 tColDataPutValue static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1374,12 +1334,7 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1393,12 +1348,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1408,12 +1358,7 @@ static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *p SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1423,12 +1368,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *p SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1447,12 +1387,7 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1461,12 +1396,7 @@ static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *p if (code) return code; SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1485,12 +1415,7 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1499,12 +1424,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *p if (code) return code; SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1513,12 +1433,7 @@ static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1527,12 +1442,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { int32_t code = 0; @@ -1541,12 +1451,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - code = tColDataPutValue(pColData, pColVal); - if (code) return code; - - pColData->nVal++; - - return code; + return tColDataPutValue(pColData, pColVal); } static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 @@ -1652,7 +1557,7 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); } -uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) { +uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { uint8_t v; switch (pColData->flag) { case HAS_NONE: @@ -1723,3 +1628,385 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { _exit: return code; } + +#define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ + do { \ + (SUM) += (VAL); \ + if ((MAX) < (VAL)) (MAX) = (VAL); \ + if ((MIN) > (VAL)) (MIN) = (VAL); \ + } while (0) + +static FORCE_INLINE void tColDataCalcSMABool(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = 0; + *min = 1; + *numOfNull = 0; + + int8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((int8_t *)pColData->pData)[iVal] ? 1 : 0; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT8_MIN; + *min = INT8_MAX; + *numOfNull = 0; + + int8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((int8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinySmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT16_MIN; + *min = INT16_MAX; + *numOfNull = 0; + + int16_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((int16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT32_MIN; + *min = INT32_MAX; + *numOfNull = 0; + + int32_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((int32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMABigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *sum = 0; + *max = INT64_MIN; + *min = INT64_MAX; + *numOfNull = 0; + + int64_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((int64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((int64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*sum, *max, *min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAFloat(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(double *)sum = 0; + *(double *)max = -FLT_MAX; + *(double *)min = FLT_MAX; + *numOfNull = 0; + + float val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((float *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((float *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMADouble(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(double *)sum = 0; + *(double *)max = -DBL_MAX; + *(double *)min = DBL_MAX; + *numOfNull = 0; + + double val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((double *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((double *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(double *)sum, *(double *)max, *(double *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUTinyInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT8_MAX; + *numOfNull = 0; + + uint8_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((uint8_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMATinyUSmallInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT16_MAX; + *numOfNull = 0; + + uint16_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((uint16_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT32_MAX; + *numOfNull = 0; + + uint32_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((uint32_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = UINT64_MAX; + *numOfNull = 0; + + uint64_t val; + if (HAS_VALUE == pColData->flag) { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + val = ((uint64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + } + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + switch (tColDataGetBitValue(pColData, iVal)) { + case 0: + case 1: + (*numOfNull)++; + break; + case 2: + val = ((uint64_t *)pColData->pData)[iVal]; + CALC_SUM_MAX_MIN(*(uint64_t *)sum, *(uint64_t *)max, *(uint64_t *)min, val); + break; + default: + ASSERT(0); + break; + } + } + } +} + +void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { + NULL, + tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL + tColDataCalcSMATinyInt, // TSDB_DATA_TYPE_TINYINT + tColDataCalcSMATinySmallInt, // TSDB_DATA_TYPE_SMALLINT + tColDataCalcSMAInt, // TSDB_DATA_TYPE_INT + tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT + tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT + tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT + tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT + tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT + tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL // TSDB_DATA_TYPE_MEDIUMBLOB +}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 31f532b6ce..2eb94773e9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2238,6 +2238,7 @@ int32_t tSerializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tEncodeI64(&encoder, pReq->dbId) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTable) < 0) return -1; + if (tEncodeI64(&encoder, pReq->stateTs) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2254,6 +2255,7 @@ int32_t tDeserializeSUseDbReq(void *buf, int32_t bufLen, SUseDbReq *pReq) { if (tDecodeI64(&decoder, &pReq->dbId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTable) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->stateTs) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2489,6 +2491,8 @@ int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1; } + if (tEncodeI32(pEncoder, pRsp->errCode) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->stateTs) < 0) return -1; return 0; } @@ -2553,6 +2557,8 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) { taosArrayPush(pRsp->pVgroupInfos, &vgInfo); } + if (tDecodeI32(pDecoder, &pRsp->errCode) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->stateTs) < 0) return -1; return 0; } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index ac4b5f86de..d39d3c501a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -73,8 +73,8 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } else if (tdValTypeIsNone(pVal->valType)) { printf("NONE "); return; - } - if(!pVal->val) { + } + if (!pVal->val) { ASSERT(0); printf("BadVal "); return; @@ -1083,13 +1083,15 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV } else if (tdValTypeIsNull(cv.valType)) { *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); } else { - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - value.nData = varDataLen(cv.val); - value.pData = varDataVal(cv.val); - } else { - tGetValue(cv.val, &value, pTColumn->type); - } + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pColVal->value.nData = varDataLen(cv.val); + pColVal->value.pData = varDataVal(cv.val); + } else { + memcpy(&pColVal->value.val, cv.val, tDataTypes[pTColumn->type].bytes); + } } } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 85a09b79fd..8a8561161b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -307,8 +307,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, pStart, numOfCols); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 976f2cfd8b..ec4f42c847 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -91,7 +91,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -102,7 +102,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -116,7 +116,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -127,7 +127,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -151,7 +151,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 51213a6ab3..2265f58060 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -191,6 +191,16 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, replica:%d id:%d fqdn:%s port:%u", req.vgId, i, req.replicas[i].id, req.replicas[i].fqdn, req.replicas[i].port); } + + SReplica *pReplica = &req.replicas[req.selfIndex]; + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + return -1; + } + vmGenerateVnodeCfg(&req, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { @@ -285,6 +295,15 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + SReplica *pReplica = &alterReq.replicas[alterReq.selfIndex]; + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", alterReq.vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + return -1; + } + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); @@ -341,6 +360,12 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vgId = dropReq.vgId; dDebug("vgId:%d, start to drop vnode", vgId); + if (dropReq.dnodeId != pMgmt->pData->dnodeId) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d not matched with local dnode", dropReq.vgId, dropReq.dnodeId); + return -1; + } + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d401f9c17b..17bc1526b5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -155,9 +155,13 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + code = vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + if (code) { + dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, terrstr(code)); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 74a92c9fcd..04ac5aba49 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -321,6 +321,7 @@ typedef struct { int32_t vgVersion; SDbCfg cfg; SRWLatch lock; + int64_t stateTs; } SDbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index aad2832aa6..78d84c9c25 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -285,8 +285,17 @@ static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); - if (pDb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + if (pDb == NULL) { + if (terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { + terrno = TSDB_CODE_MND_DB_IN_CREATING; + } else if (terrno == TSDB_CODE_SDB_OBJ_DROPPING) { + terrno = TSDB_CODE_MND_DB_IN_DROPPING; + } else { + terrno = TSDB_CODE_APP_ERROR; + mFatal("db:%s, failed to acquire db since %s", db, terrstr()); + } } return pDb; } @@ -594,16 +603,22 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; goto _OVER; } - } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and response after trans finished", createReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; + } else { + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { + mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } else { + goto _OVER; + } + } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) { goto _OVER; - } else { + } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) { + // continue + } else { // TSDB_CODE_APP_ERROR goto _OVER; } - } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); @@ -786,7 +801,6 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } @@ -836,7 +850,6 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } @@ -1066,11 +1079,8 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { if (pDb == NULL) { if (dropReq.ignoreNotExists) { code = mndBuildDropDbRsp(pDb, &pReq->info.rspLen, &pReq->info.rsp, true); - goto _OVER; - } else { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto _OVER; } + goto _OVER; } if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { @@ -1165,13 +1175,14 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { + if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); } memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); pRsp->uid = pDb->uid; pRsp->vgVersion = pDb->vgVersion; + pRsp->stateTs = pDb->stateTs; pRsp->vgNum = taosArrayGetSize(pRsp->pVgroupInfos); pRsp->hashMethod = pDb->cfg.hashMethod; pRsp->hashPrefix = pDb->cfg.hashPrefix; @@ -1197,10 +1208,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); if (usedbReq.vgVersion < vgVersion) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } + if (usedbRsp.pVgroupInfos == NULL) goto _OVER; mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; @@ -1209,16 +1217,21 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { } usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); code = 0; - - // no jump, need to construct rsp } else { pDb = mndAcquireDb(pMnode, usedbReq.db); if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); usedbRsp.uid = usedbReq.dbId; usedbRsp.vgVersion = usedbReq.vgVersion; + usedbRsp.errCode = terrno; + + if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { + mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } + } mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { @@ -1230,6 +1243,8 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { goto _OVER; } + mDebug("db:%s, process usedb req vgVersion:%d stateTs:%" PRId64 ", rsp vgVersion:%d stateTs:%" PRId64, + usedbReq.db, usedbReq.vgVersion, usedbReq.stateTs, usedbRsp.vgVersion, usedbRsp.stateTs); code = 0; } } @@ -1248,7 +1263,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; _OVER: - if (code != 0) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1268,12 +1283,31 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, for (int32_t i = 0; i < numOfDbs; ++i) { SDbVgVersion *pDbVgVersion = &pDbs[i]; - pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->dbId = be64toh(pDbVgVersion->dbId); pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); pDbVgVersion->numOfTable = htonl(pDbVgVersion->numOfTable); + pDbVgVersion->stateTs = be64toh(pDbVgVersion->stateTs); SUseDbRsp usedbRsp = {0}; + if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); + if (pDbVgVersion->vgVersion < vgVersion) { + usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); + + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); + usedbRsp.vgVersion = vgVersion++; + } else { + usedbRsp.vgVersion = pDbVgVersion->vgVersion; + } + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + + continue; + } + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { mTrace("db:%s, no exist", pDbVgVersion->dbFName); @@ -1286,13 +1320,19 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { - mInfo("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && + pDbVgVersion->stateTs == pDb->stateTs) { + mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, + pDb->vgVersion, pDb->stateTs, numOfTable); mndReleaseDb(pMnode, pDb); continue; } else { - mInfo("db:%s, vgroup version changed from %d to %d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, - pDb->vgVersion); + mInfo("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 + " numOfTables:%d, changed to vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", + pDbVgVersion->dbFName, pDbVgVersion->vgVersion, pDbVgVersion->stateTs, pDbVgVersion->numOfTable, + pDb->vgVersion, pDb->stateTs, numOfTable); } usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); @@ -1306,6 +1346,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); usedbRsp.uid = pDb->uid; usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.stateTs = pDb->stateTs; usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); usedbRsp.hashMethod = pDb->cfg.hashMethod; usedbRsp.hashPrefix = pDb->cfg.hashPrefix; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 04f340b0ff..521f924fad 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" +#include "mndDb.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -356,7 +357,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pDnode->lastAccessTime = curMs; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, - pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); + pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) { SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v); @@ -376,6 +377,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) { if (pVgroup->vnodeGid[vg].syncState != pVload->syncState || pVgroup->vnodeGid[vg].syncRestore != pVload->syncRestore) { + mInfo("vgId:%d, state changed by status msg, old state:%s restored:%d new state:%s restored:%d", + pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore, + syncStr(pVload->syncState), pVload->syncRestore); pVgroup->vnodeGid[vg].syncState = pVload->syncState; pVgroup->vnodeGid[vg].syncRestore = pVload->syncRestore; roleChanged = true; @@ -384,7 +388,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } } if (roleChanged) { - // notify scheduler role has changed + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb != NULL && pDb->stateTs != curMs) { + mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, pDb->stateTs, curMs); + pDb->stateTs = curMs; + } + mndReleaseDb(pMnode, pDb); } } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1b2d85bd29..dcfc046a1e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -139,6 +139,63 @@ static void mndIncreaseUpTime(SMnode *pMnode) { } } +static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + bool roleChanged = false; + for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { + if (pVgroup->vnodeGid[vg].dnodeId == dnodeId) { + if (pVgroup->vnodeGid[vg].syncState != TAOS_SYNC_STATE_ERROR) { + mInfo("vgId:%d, state changed by offline check, old state:%s restored:%d new state:error restored:0", + pVgroup->vgId, syncStr(pVgroup->vnodeGid[vg].syncState), pVgroup->vnodeGid[vg].syncRestore); + pVgroup->vnodeGid[vg].syncState = TAOS_SYNC_STATE_ERROR; + pVgroup->vnodeGid[vg].syncRestore = 0; + roleChanged = true; + } + break; + } + } + + if (roleChanged) { + SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); + if (pDb != NULL && pDb->stateTs != curMs) { + mInfo("db:%s, stateTs changed by offline check, old newTs:%" PRId64 " newTs:%" PRId64, pDb->name, pDb->stateTs, + curMs); + pDb->stateTs = curMs; + } + mndReleaseDb(pMnode, pDb); + } + + sdbRelease(pSdb, pVgroup); + } +} + +static void mndCheckDnodeOffline(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + int64_t curMs = taosGetTimestampMs(); + + void *pIter = NULL; + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + bool online = mndIsDnodeOnline(pDnode, curMs); + if (!online) { + mInfo("dnode:%d, in offline state", pDnode->id); + mndSetVgroupOffline(pMnode, pDnode->id, curMs); + } + + sdbRelease(pSdb, pDnode); + } +} + static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -174,6 +231,10 @@ static void *mndThreadFp(void *param) { if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } + + if (sec % (tsStatusInterval * 5) == 0) { + mndCheckDnodeOffline(pMnode); + } } return NULL; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 637a3818e6..b0af98b933 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -303,8 +303,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pStart += sizeof(SSysTableSchema); } - int32_t len = 0; - blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 36ba0aaf87..594c13f957 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -287,9 +287,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, memcpy(pObj->sourceDb, pCreate->sourceDB, TSDB_DB_FNAME_LEN); SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB); if (pSourceDb == NULL) { - /*ASSERT(0);*/ - mInfo("stream:%s failed to create, source db %s not exist", pCreate->name, pObj->sourceDb); - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mInfo("stream:%s failed to create, source db %s not exist since %s", pCreate->name, pObj->sourceDb, terrstr()); return -1; } pObj->sourceDbUid = pSourceDb->uid; @@ -298,8 +296,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SDbObj *pTargetDb = mndAcquireDbByStb(pMnode, pObj->targetSTbName); if (pTargetDb == NULL) { - mInfo("stream:%s failed to create, target db %s not exist", pCreate->name, pObj->targetDb); - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mInfo("stream:%s failed to create, target db %s not exist since %s", pCreate->name, pObj->targetDb, terrstr()); return -1; } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index ebab83cf6d..450d7a30ed 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -205,8 +205,23 @@ static void mndBecomeLeader(const SSyncFSM *pFsm) { static bool mndApplyQueueEmpty(const SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; - int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); - return (itemSize == 0); + if (pMnode != NULL && pMnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); + return (itemSize == 0); + } else { + return true; + } +} + +static int32_t mndApplyQueueItems(const SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + + if (pMnode != NULL && pMnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pMnode->msgCb, 1, APPLY_QUEUE); + return itemSize; + } else { + return -1; + } } SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { @@ -218,6 +233,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpLeaderTransferCb = NULL; pFsm->FpApplyQueueEmptyCb = mndApplyQueueEmpty; + pFsm->FpApplyQueueItems = mndApplyQueueItems; pFsm->FpReConfigCb = NULL; pFsm->FpBecomeLeaderCb = mndBecomeLeader; pFsm->FpBecomeFollowerCb = mndBecomeFollower; @@ -291,7 +307,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; if (req.contLen <= 0) return -1; - + req.pCont = rpcMallocCont(req.contLen); if (req.pCont == NULL) return -1; memcpy(req.pCont, pRaw, req.contLen); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ac05598bdc..db878d72b9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -938,11 +938,15 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { for (int32_t i = 0; i < size; ++i) { SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); if (pInfo->handle != NULL) { - mInfo("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pInfo->ahandle); if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } + if (i != 0 && code == 0) { + code = TSDB_CODE_RPC_REDIRECT; + } + mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; if (pTrans->originRpcType == TDMT_MND_CREATE_DB) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bfdc71bceb..ba9d68ee7f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -110,7 +110,6 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { #define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // SRowIter void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); @@ -191,7 +190,6 @@ int32_t tsdbKeyFid(TSKEY key, int32_t minutes, int8_t precision); void tsdbFidKeyRange(int32_t fid, int32_t minutes, int8_t precision, TSKEY *minKey, TSKEY *maxKey); int32_t tsdbFidLevel(int32_t fid, STsdbKeepCfg *pKeepCfg, int64_t now); int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline); -void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg); int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, @@ -211,11 +209,10 @@ void tsdbRefMemTable(SMemTable *pMemTable); void tsdbUnrefMemTable(SMemTable *pMemTable); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter -int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); -void *tsdbTbDataIterDestroy(STbDataIter *pIter); -void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter); -bool tsdbTbDataIterNext(STbDataIter *pIter); +int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); +void *tsdbTbDataIterDestroy(STbDataIter *pIter); +void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); +bool tsdbTbDataIterNext(STbDataIter *pIter); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== @@ -773,6 +770,40 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +static FORCE_INLINE int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { + int32_t n = tGetI64(p, &pRow->version); + pRow->pTSRow = (STSRow *)(p + n); + n += pRow->pTSRow->len; + return n; +} + +static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { + if (pIter == NULL) return NULL; + + if (pIter->pRow) { + return pIter->pRow; + } + + if (pIter->backward) { + if (pIter->pNode == pIter->pTbData->sl.pHead) { + return NULL; + } + } else { + if (pIter->pNode == pIter->pTbData->sl.pTail) { + return NULL; + } + } + + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + pIter->pRow = &pIter->row; + + return pIter->pRow; +} + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 44ecf64419..ac9fabf052 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -102,6 +102,7 @@ int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); int metaCommit(SMeta* pMeta); int metaFinishCommit(SMeta* pMeta); +int metaPrepareAsyncCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 01ad833d20..0be0c3e407 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -35,6 +35,7 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { // commit the meta txn int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); } int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, &pMeta->txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, &pMeta->txn); } // abort the meta txn int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, &pMeta->txn); } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a6e8767a4d..48c14bc758 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -27,9 +27,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->completed = 1; pRetrieve->numOfRows = htonl(pBlock->info.rows); - // TODO enable compress - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index b852acb936..27c491c86b 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -125,6 +125,10 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ return -1; } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + return -1; + } + return 0; } @@ -147,6 +151,10 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + return 0; } @@ -226,6 +234,10 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + tEncoderClear(&encoder); taosMemoryFree(buf); return 0; @@ -250,6 +262,10 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { ASSERT(0); } + if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + ASSERT(0); + } + return 0; } diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index c52e0e2c09..b68763867e 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 92e5f8df7a..08d5931bc3 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index b4a7ce7737..31e44a5b6d 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -169,6 +169,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { } else { code = tdbCommit(pWriter->pTq->pMetaStore, &pWriter->txn); if (code) goto _err; + code = tdbPostCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 76236e5078..291a5ab1eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -475,7 +475,7 @@ _err: } return code; } - +/* static int32_t getTableDelIdx(SDelFReader *pDelFReader, tb_uid_t suid, tb_uid_t uid, SDelIdx *pDelIdx) { int32_t code = 0; SArray *pDelIdxArray = NULL; @@ -499,7 +499,7 @@ _err: } return code; } - +*/ typedef enum { SFSLASTNEXTROW_FS, SFSLASTNEXTROW_FILESET, @@ -997,8 +997,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); - SDelIdx delIdx; - SDelFile *pDelFile = pReadSnap->fs.pDelFile; if (pDelFile) { SDelFReader *pDelFReader; @@ -1006,18 +1004,20 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); if (code) goto _err; - code = getTableDelIdx(pDelFReader, suid, uid, &delIdx); - if (code) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - code = getTableDelSkyline(pMem, pIMem, pDelFReader, &delIdx, pIter->pSkyline); + SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); + + code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); + if (code) goto _err; + + SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); + + code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); if (code) { tsdbDelFReaderClose(&pDelFReader); goto _err; } + taosArrayDestroy(pDelIdxArray); tsdbDelFReaderClose(&pDelFReader); } else { code = getTableDelSkyline(pMem, pIMem, NULL, NULL, pIter->pSkyline); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 43be51a694..9f59707ddc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -427,13 +427,13 @@ static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *p {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; -extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); +// extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (pBuilder->calcSma) { if (COL_VAL_IS_VALUE(pColVal)) { - tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); + // tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); } else { pBuilder->sma.numOfNull++; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index c663e2b526..52c7a07c49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -294,31 +294,6 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } -TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { - // we add here for commit usage - if (pIter == NULL) return NULL; - - if (pIter->pRow) { - goto _exit; - } - - if (pIter->backward) { - if (pIter->pNode == pIter->pTbData->sl.pHead) { - goto _exit; - } - } else { - if (pIter->pNode == pIter->pTbData->sl.pTail) { - goto _exit; - } - } - - tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); - pIter->pRow = &pIter->row; - -_exit: - return pIter->pRow; -} - static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 631cbd415f..c157faecb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -366,9 +366,9 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyAllBlockScanInfo(SHashObj* pTableMap) { +static void destroyAllBlockScanInfo(SHashObj* pTableMap, bool clearEntry) { void* p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { + while (clearEntry && ((p = taosHashIterate(pTableMap, p)) != NULL)) { clearBlockScanInfo(*(STableBlockScanInfo**)p); } @@ -893,7 +893,7 @@ static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int or } } -int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { +static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; bool asc = ASCENDING_TRAVERSE(pReader->order); @@ -910,6 +910,117 @@ int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SData return endPos; } +static void copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t)); + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + memcpy(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t)); + + // todo: opt perf by extract the loop + // reverse the array list + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + } +} + +// a faster version of copy procedure. +static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData, + int32_t dumpedRows, bool asc) { + uint8_t* p = NULL; + if (asc) { + p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + } else { + int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1; + p = pData->pData + tDataTypes[pData->type].bytes * startIndex; + } + + int32_t step = asc? 1:-1; + + // make sure it is aligned to 8bit + ASSERT((((uint64_t)pColData->pData) & (0x8 - 1)) == 0); + + // 1. copy data in a batch model + memcpy(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes); + + // 2. reverse the array list in case of descending order scan data block + if (!asc) { + switch(pColData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + { + int32_t mid = dumpedRows >> 1u; + int64_t* pts = (int64_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + int32_t mid = dumpedRows >> 1u; + int8_t* pts = (int8_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int8_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + int32_t mid = dumpedRows >> 1u; + int16_t* pts = (int16_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int64_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t mid = dumpedRows >> 1u; + int32_t* pts = (int32_t*)pColData->pData; + for (int32_t j = 0; j < mid; ++j) { + int32_t t = pts[j]; + pts[j] = pts[dumpedRows - j - 1]; + pts[dumpedRows - j - 1] = t; + } + break; + } + } + } + + // 3. if the null value exists, check items one-by-one + if (pData->flag != HAS_VALUE) { + int32_t rowIndex = 0; + + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) { + uint8_t v = tColDataGetBitValue(pData, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + pColData->hasNull = true; + } + } + } +} + static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -949,24 +1060,17 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } endIndex += step; - int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); - if (remain > pReader->capacity) { // output buffer check - remain = pReader->capacity; + int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (dumpedRows > pReader->capacity) { // output buffer check + dumpedRows = pReader->capacity; } + int32_t i = 0; int32_t rowIndex = 0; - int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - if (asc) { - memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); - } else { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { - colDataAppendInt64(pColData, rowIndex++, &pBlockData->aTSKEY[j]); - } - } - + copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -981,23 +1085,12 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; } else if (pData->cid == pColData->info.colId) { if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); } else { - if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { - uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; - memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); - - // null value exists, check one-by-one - if (pData->flag != HAS_VALUE) { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { - uint8_t v = tColDataGetBitValue(pData, j); - if (v == 0 || v == 1) { - colDataSetNull_f(pColData->nullbitmap, rowIndex); - } - } - } - } else { - for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + if (IS_MATHABLE_TYPE(pColData->info.type)) { + copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc); + } else { // varchar/nchar type + for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); } @@ -1007,7 +1100,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } } @@ -1015,12 +1108,12 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, i); - colDataAppendNNULL(pColData, 0, remain); + colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } - pResBlock->info.rows = remain; - pDumpInfo->rowIndex += step * remain; + pResBlock->info.rows = dumpedRows; + pDumpInfo->rowIndex += step * dumpedRows; // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { @@ -1039,7 +1132,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, dumpedRows, unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3768,7 +3861,7 @@ void tsdbReaderClose(STsdbReader* pReader) { cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); - destroyAllBlockScanInfo(pReader->status.pTableMap); + destroyAllBlockScanInfo(pReader->status.pTableMap, (pReader->innerReader[0] == NULL) ? true : false); blockDataDestroy(pReader->pResBlock); clearBlockScanInfoBuf(&pReader->blockInfoBuf); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d7e5fba5cd..8c1b0004a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -519,8 +519,8 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - SColumnDataAgg sma; - tsdbCalcColDataSMA(pColData, &sma); + SColumnDataAgg sma = {.colId = pColData->cid}; + tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e34e305570..9ac9a7d66d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -575,16 +575,6 @@ int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { return n; } -int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) { - int32_t n = 0; - - n += tGetI64(p, &pRow->version); - pRow->pTSRow = (STSRow *)(p + n); - n += pRow->pTSRow->len; - - return n; -} - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } @@ -607,7 +597,7 @@ void tRowIterInit(SRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { SColVal *tRowIterNext(SRowIter *pIter) { if (pIter->pRow->type == 0) { if (pIter->i < pIter->pTSchema->numOfCols) { - tsdbRowGetColVal(pIter->pRow, pIter->pTSchema, pIter->i, &pIter->colVal); + tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); pIter->i++; return &pIter->colVal; @@ -1053,7 +1043,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS tRowIterInit(&rIter, pRow, pTSchema); pColVal = tRowIterNext(&rIter); for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; while (pColVal && pColVal->cid < pColData->cid) { pColVal = tRowIterNext(&rIter); @@ -1437,111 +1427,6 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { return n; } -#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \ - do { \ - (SUM_V) += (VAL); \ - if (!(MINSET)) { \ - (MIN_V) = (VAL); \ - (MINSET) = 1; \ - } else if ((MIN_V) > (VAL)) { \ - (MIN_V) = (VAL); \ - } \ - if (!(MAXSET)) { \ - (MAX_V) = (VAL); \ - (MAXSET) = 1; \ - } else if ((MAX_V) < (VAL)) { \ - (MAX_V) = (VAL); \ - } \ - } while (0) - -static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - int8_t val = *(int8_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - int16_t val = *(int16_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int32_t val = *(int32_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - int64_t val = *(int64_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - float val = *(float *)&pColVal->value.val; - SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - double val = *(double *)&pColVal->value.val; - SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint8_t val = *(uint8_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint16_t val = *(uint16_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { - uint32_t val = *(uint32_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, - uint8_t *maxSet) { - uint64_t val = *(uint64_t *)&pColVal->value.val; - SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); -} -void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = { - NULL, - tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL - tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT - tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT - tSmaUpdateInt, // TSDB_DATA_TYPE_INT - tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT - tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT - tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR - tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR - tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT - tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT - tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT - tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON - NULL, // TSDB_DATA_TYPE_VARBINARY - NULL, // TSDB_DATA_TYPE_DECIMAL - NULL, // TSDB_DATA_TYPE_BLOB - NULL, // TSDB_DATA_TYPE_MEDIUMBLOB -}; -void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { - *pColAgg = (SColumnDataAgg){.colId = pColData->cid}; - uint8_t minSet = 0; - uint8_t maxSet = 0; - - SColVal cv; - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - tColDataGetValue(pColData, iVal, &cv); - - if (COL_VAL_IS_VALUE(&cv)) { - tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); - } else { - pColAgg->numOfNull++; - } - } -} - int32_t tsdbCmprData(uint8_t *pIn, int32_t szIn, int8_t type, int8_t cmprAlg, uint8_t **ppOut, int32_t nOut, int32_t *szOut, uint8_t **ppBuf) { int32_t code = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 9fe37505ad..782ffd788d 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -37,6 +37,12 @@ struct SVnodeGlobal vnodeGlobal; static void* loop(void* arg); +static tsem_t canCommit = {0}; + +static void vnodeInitCommit() { tsem_init(&canCommit, 0, 4); }; +void vnode_wait_commit() { tsem_wait(&canCommit); } +void vnode_done_commit() { tsem_wait(&canCommit); } + int vnodeInit(int nthreads) { int8_t init; int ret; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e7f8c9f562..38cb534d7f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -438,8 +438,24 @@ static void vnodeBecomeLeader(const SSyncFSM *pFsm) { static bool vnodeApplyQueueEmpty(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; - int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); - return (itemSize == 0); + + if (pVnode != NULL && pVnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); + return (itemSize == 0); + } else { + return true; + } +} + +static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + + if (pVnode != NULL && pVnode->msgCb.qsizeFp != NULL) { + int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); + return itemSize; + } else { + return -1; + } } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { @@ -452,6 +468,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = NULL; pFsm->FpApplyQueueEmptyCb = vnodeApplyQueueEmpty; + pFsm->FpApplyQueueItems = vnodeApplyQueueItems; pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; pFsm->FpReConfigCb = NULL; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8c699bb59b..4d3a3a1ab4 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -538,7 +538,8 @@ typedef struct SCtgOperation { (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) -#define CTG_DB_NOT_EXIST(code) (code == TSDB_CODE_MND_DB_NOT_EXIST) +#define CTG_DB_NOT_EXIST(code) \ + (code == TSDB_CODE_MND_DB_NOT_EXIST || code == TSDB_CODE_MND_DB_IN_CREATING || code == TSDB_CODE_MND_DB_IN_DROPPING) #define ctgFatal(param, ...) qFatal("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgError(param, ...) qError("CTG:%p " param, pCtg, __VA_ARGS__) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1f87066c82..2dcd681205 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -715,10 +715,10 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { +int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum, int64_t* pStateTs) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { + if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId || NULL == tableNum || NULL == pStateTs) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b601865306..ff3bd4d33c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1998,6 +1998,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + pInfo->stateTs = dbCache->vgCache.vgInfo->stateTs; ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 51807a145a..e99beca69a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1231,14 +1231,16 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); - SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0}; tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); + if (!IS_SYS_DBNAME(dbFName)) { + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); + } return TSDB_CODE_SUCCESS; @@ -1563,7 +1565,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); @@ -1579,15 +1581,15 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SDBVgInfo *vgInfo = vgCache->vgInfo; if (dbInfo->vgVersion < vgInfo->vgVersion) { - ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgDebug("db updateVgroup is ignored, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); goto _return; } - if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { - ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, - dbInfo->numOfTable); + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable && dbInfo->stateTs == vgInfo->stateTs) { + ctgDebug("no new db vgroup update info, dbFName:%s, vgVer:%d, numOfTable:%d, stateTs:%" PRId64, dbFName, dbInfo->vgVersion, + dbInfo->numOfTable, dbInfo->stateTs); ctgWUnlockVgInfo(dbCache); goto _return; @@ -1599,15 +1601,17 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, stateTs:%" PRId64 ", dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.stateTs, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + } _return: @@ -2170,7 +2174,6 @@ void *ctgUpdateThreadFunc(void *param) { CTG_RT_STAT_INC(numOfOpDequeue, 1); ctgdShowCacheInfo(); - ctgdShowClusterCache(pCtg); } qInfo("catalog update thread stopped"); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 26b5903bb0..b6ff1c8b98 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -78,7 +78,7 @@ void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { num = taosArrayGetSize(pResult->pDbInfo); for (int32_t i = 0; i < num; ++i) { SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i); - qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); + qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, stateTs:%" PRId64 " dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->stateTs, pDb->dbId); } } else { qDebug("empty db info"); @@ -462,6 +462,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { int32_t hashMethod = -1; int16_t hashPrefix = 0; int16_t hashSuffix = 0; + int64_t stateTs = 0; int32_t vgNum = 0; if (dbCache->vgCache.vgInfo) { @@ -469,16 +470,35 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) { hashMethod = dbCache->vgCache.vgInfo->hashMethod; hashPrefix = dbCache->vgCache.vgInfo->hashPrefix; hashSuffix = dbCache->vgCache.vgInfo->hashSuffix; + stateTs = dbCache->vgCache.vgInfo->stateTs; if (dbCache->vgCache.vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } ctgDebug("[%d] db [%.*s][0x%" PRIx64 - "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, + "] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs, hashMethod, hashPrefix, hashSuffix, vgNum); + if (dbCache->vgCache.vgInfo) { + int32_t i = 0; + void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL); + while (pVgIter) { + SVgroupInfo * pVg = (SVgroupInfo *)pVgIter; + + ctgDebug("The %04dth VG [id:%d, hashBegin:%u, hashEnd:%u, numOfTable:%d, epNum:%d, inUse:%d]", + i++, pVg->vgId, pVg->hashBegin, pVg->hashEnd, pVg->numOfTable, pVg->epSet.numOfEps, pVg->epSet.inUse); + + for (int32_t n = 0; n < pVg->epSet.numOfEps; ++n) { + SEp *pEp = &pVg->epSet.eps[n]; + ctgDebug("\tEp %d [fqdn:%s, port:%d]", n, pEp->fqdn, pEp->port); + } + + pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter); + } + } + pIter = taosHashIterate(dbHash, pIter); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebf7c7baeb..5e543384ac 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2486,7 +2486,8 @@ TEST(dbVgroup, getSetDbVgroupCase) { int32_t dbVer = 0; int64_t dbId = 0; int32_t tbNum = 0; - code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum); + int64_t stateTs = 0; + code = catalogGetDBVgVersion(pCtg, ctgTestDbname, &dbVer, &dbId, &tbNum, &stateTs); ASSERT_EQ(code, 0); ASSERT_EQ(dbVer, ctgTestVgVersion); ASSERT_EQ(dbId, ctgTestDbId); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 76c84b3be9..1c2d7e1f66 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -39,8 +39,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htonl(pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = 0; - blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); + int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 80a524496c..915dc08c14 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1610,8 +1610,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - int32_t len = 0; - blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 771ab10140..62146b6048 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -264,6 +264,7 @@ typedef struct SExchangeInfo { SLoadRemoteDataInfo loadInfo; uint64_t self; SLimitInfo limitInfo; + int64_t openedTs; // start exec time stamp } SExchangeInfo; typedef struct SScanInfo { @@ -841,10 +842,8 @@ typedef struct SJoinOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -void doDestroyExchangeOperatorInfo(void* param); - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain); +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -880,7 +879,11 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); -void doSetOperatorCompleted(SOperatorInfo* pOperator); +void doDestroyExchangeOperatorInfo(void* param); + +void setOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, + SExecTaskInfo* pTaskInfo); void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo * pCache); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 76866eedb7..6f9084ce52 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -93,16 +93,11 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - pOperator->name = "LastrowScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, NULL, destroyLastrowScanOperator, NULL); + createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyLastrowScanOperator, NULL); pOperator->cost.openCost = 0; return pOperator; @@ -126,7 +121,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); if (size == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -182,7 +177,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes += 1; return pRes; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } else { @@ -234,7 +229,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index bc4ab9c468..3d9757c96f 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c new file mode 100644 index 0000000000..049de727df --- /dev/null +++ b/source/libs/executor/src/exchangeoperator.c @@ -0,0 +1,638 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" +#include "tref.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "tsort.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef struct SFetchRspHandleWrapper { + uint32_t exchangeId; + int32_t sourceIndex; +} SFetchRspHandleWrapper; + +static void destroyExchangeOperatorInfo(void* param); +static void freeBlock(void* pParam); +static void freeSourceDataInfo(void* param); +static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs); + +static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); +static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); +static int32_t getCompletedSources(const SArray* pArray); +static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); +static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); +static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); + +static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, + SExecTaskInfo* pTaskInfo) { + int32_t code = 0; + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); + int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (completed == totalSources) { + setAllSourcesCompleted(pOperator, pExchangeInfo->openedTs); + return; + } + + while (1) { + tsem_wait(&pExchangeInfo->ready); + + for (int32_t i = 0; i < totalSources; ++i) { + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { + continue; + } + + if (pDataInfo->status != EX_SOURCE_DATA_READY) { + continue; + } + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); + + // todo + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, + pExchangeInfo->loadInfo.totalRows, i + 1, totalSources); + taosMemoryFreeClear(pDataInfo->pRsp); + break; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + int32_t index = 0; + char* pStart = pRetrieveRsp->data; + while (index++ < pRetrieveRsp->numOfBlocks) { + SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); + if (code != 0) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + + taosArrayPush(pExchangeInfo->pResultBlockList, &pb); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, pExchangeInfo->openedTs, pOperator); + + if (pRsp->completed == 1) { + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + ", total:%.2f Kb, try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, + pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, + i + 1, totalSources); + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 + " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, + pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); + } + + taosMemoryFreeClear(pDataInfo->pRsp); + + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { + pDataInfo->status = EX_SOURCE_DATA_NOT_READY; + code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pDataInfo->pRsp); + goto _error; + } + } + return; + } // end loop + + int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + if (complete1 == totalSources) { + qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); + return; + } + } + + _error: + pTaskInfo->code = code; +} + +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + return NULL; + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pOperator->status == OP_EXEC_DONE) { + qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + return NULL; + } + + size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); + if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { + pExchangeInfo->rspBlockIndex = 0; + taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); + if (pExchangeInfo->seqLoadData) { + seqLoadRemoteData(pOperator); + } else { + concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { + return NULL; + } + } + + // we have buffered retrieved datablock, return it directly + return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); +} + +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pBlock->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + setOperatorCompleted(pOperator); + return NULL; + } else { + return pBlock; + } + } + } else { + return pBlock; + } + } +} + +static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = EX_SOURCE_DATA_NOT_READY; + dataInfo.taskId = id; + dataInfo.index = i; + SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + if (pDs == NULL) { + taosArrayDestroy(pInfo->pSourceDataInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); + + if (numOfSources == 0) { + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); + return TSDB_CODE_INVALID_PARA; + } + + pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); + if (pInfo->pSources == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfSources; ++i) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); + taosArrayPush(pInfo->pSources, pNode); + } + + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); + pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); + + return initDataSource(numOfSources, pInfo, id); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + tsem_init(&pInfo->ready, 0, 0); + pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); + + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + + setOperatorInfo(pOperator, "ExchangeOperator", QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + doDestroyExchangeOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + taosRemoveRef(exchangeObjRefPool, pExInfo->self); +} + +void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + +void freeSourceDataInfo(void* p) { + SSourceDataInfo* pInfo = (SSourceDataInfo*)p; + taosMemoryFreeClear(pInfo->pRsp); +} + +void doDestroyExchangeOperatorInfo(void* param) { + SExchangeInfo* pExInfo = (SExchangeInfo*)param; + + taosArrayDestroy(pExInfo->pSources); + taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); + + if (pExInfo->pResultBlockList != NULL) { + taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); + pExInfo->pResultBlockList = NULL; + } + + blockDataDestroy(pExInfo->pDummyBlock); + + tsem_destroy(&pExInfo->ready); + taosMemoryFreeClear(param); +} + +int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { + SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; + + SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); + if (pExchangeInfo == NULL) { + qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); + return TSDB_CODE_SUCCESS; + } + + int32_t index = pWrapper->sourceIndex; + SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + + if (code == TSDB_CODE_SUCCESS) { + pSourceDataInfo->pRsp = pMsg->pData; + + SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->compLen = htonl(pRsp->compLen); + pRsp->numOfCols = htonl(pRsp->numOfCols); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); + + ASSERT(pRsp != NULL); + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + pRsp->numOfRows); + } else { + taosMemoryFree(pMsg->pData); + pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); + } + + pSourceDataInfo->status = EX_SOURCE_DATA_READY; + + tsem_post(&pExchangeInfo->ready); + taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); + + return TSDB_CODE_SUCCESS; +} + +int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + + SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); + pWrapper->exchangeId = pExchangeInfo->self; + pWrapper->sourceIndex = sourceIndex; + + if (pSource->localExec) { + SDataBuf pBuf = {0}; + int32_t code = + (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, + pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); + loadRemoteDataCallback(pWrapper, &pBuf, code); + taosMemoryFree(pWrapper); + } else { + SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, + pSource->execId, sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pMsg->execId = htonl(pSource->execId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = pSource->fetchMsgType; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); + } + + return TSDB_CODE_SUCCESS; +} + +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator) { + pInfo->totalRows += numOfRows; + pInfo->totalSize += dataLen; + pInfo->totalElapsed += (taosGetTimestampUs() - startTs); + pOperator->resultInfo.totalRows += numOfRows; +} + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { + if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + *pNextStart = (char*)blockDecode(pRes, pData); + } else { // extract data according to pColList + char* pStart = pData; + + int32_t numOfCols = htonl(*(int32_t*)pStart); + pStart += sizeof(int32_t); + + // todo refactor:extract method + SSysTableSchema* pSchema = (SSysTableSchema*)pStart; + for (int32_t i = 0; i < numOfCols; ++i) { + SSysTableSchema* p = (SSysTableSchema*)pStart; + + p->colId = htons(p->colId); + p->bytes = htonl(p->bytes); + pStart += sizeof(SSysTableSchema); + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); + } + + blockDecode(pBlock, pStart); + blockDataEnsureCapacity(pRes, pBlock->info.rows); + + // data from mnode + pRes->info.rows = pBlock->info.rows; + relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); + blockDataDestroy(pBlock); + } + + // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. + blockDataUpdateTsWindow(pRes, 0); + return TSDB_CODE_SUCCESS; +} + +void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int64_t el = taosGetTimestampUs() - startTs; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + + pLoadInfo->totalElapsed += el; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", + GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, + pLoadInfo->totalElapsed / 1000.0); + + setOperatorCompleted(pOperator); + return NULL; +} + +int32_t getCompletedSources(const SArray* pArray) { + size_t total = taosArrayGetSize(pArray); + + int32_t completed = 0; + for (int32_t k = 0; k < total; ++k) { + SSourceDataInfo* p = taosArrayGet(pArray, k); + if (p->status == EX_SOURCE_DATA_EXHAUSTED) { + completed += 1; + } + } + + return completed; +} + +int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for (int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return code; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), + totalSources, (endTs - startTs) / 1000.0); + + pOperator->status = OP_RES_TO_RETURN; + pOperator->cost.openCost = taosGetTimestampUs() - startTs; + + tsem_wait(&pExchangeInfo->ready); + tsem_post(&pExchangeInfo->ready); + return TSDB_CODE_SUCCESS; +} + +int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + while (1) { + if (pExchangeInfo->current >= totalSources) { + setAllSourcesCompleted(pOperator, startTs); + return TSDB_CODE_SUCCESS; + } + + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); + tsem_wait(&pExchangeInfo->ready); + + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), + pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); + pOperator->pTaskInfo->code = pDataInfo->code; + return pOperator->pTaskInfo->code; + } + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, + pDataInfo->totalRows, pLoadInfo->totalRows); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + taosMemoryFreeClear(pDataInfo->pRsp); + continue; + } + + SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; + + char* pStart = pRetrieveRsp->data; + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); + + if (pRsp->completed == 1) { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, + totalSources); + + pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; + pExchangeInfo->current += 1; + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, + pLoadInfo->totalRows, pLoadInfo->totalSize); + } + + updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); + pDataInfo->totalRows += pRetrieveRsp->numOfRows; + + taosMemoryFreeClear(pDataInfo->pRsp); + return TSDB_CODE_SUCCESS; + } +} + +int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + int64_t st = taosGetTimestampUs(); + + SExchangeInfo* pExchangeInfo = pOperator->info; + if (!pExchangeInfo->seqLoadData) { + int32_t code = prepareConcurrentlyLoad(pOperator); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pExchangeInfo->openedTs = taosGetTimestampUs(); + } + + OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1cf01c8661..1aa9a3c613 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1106,3 +1106,24 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } return 0; } + +void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; + assert(pMsg->info.ahandle != NULL); + + SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; + + if (pMsg->contLen > 0) { + buf.pData = taosMemoryCalloc(1, pMsg->contLen); + if (buf.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + memcpy(buf.pData, pMsg->pCont, pMsg->contLen); + } + } + + pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); + rpcFreeCont(pMsg->pCont); + destroySendMsgInfo(pSendInfo); +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index efb9a49df2..709e981a1f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -32,7 +32,6 @@ #include "index.h" #include "query.h" #include "tcompare.h" -#include "tcompression.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" @@ -91,11 +90,11 @@ static void destroySortOperatorInfo(void* param); static void destroyAggOperatorInfo(void* param); static void destroyIntervalOperatorInfo(void* param); -static void destroyExchangeOperatorInfo(void* param); + static void destroyOperatorInfo(SOperatorInfo* pOperator); -void doSetOperatorCompleted(SOperatorInfo* pOperator) { +void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); @@ -103,14 +102,24 @@ void doSetOperatorCompleted(SOperatorInfo* pOperator) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo) { + pOperator->name = (char*)name; + pOperator->operatorType = type; + pOperator->blocking = blocking; + pOperator->status = status; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; +} + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, @@ -1652,623 +1661,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t return TSDB_CODE_SUCCESS; } -typedef struct SFetchRspHandleWrapper { - uint32_t exchangeId; - int32_t sourceIndex; -} SFetchRspHandleWrapper; - -int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { - SFetchRspHandleWrapper* pWrapper = (SFetchRspHandleWrapper*)param; - - SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); - if (pExchangeInfo == NULL) { - qWarn("failed to acquire exchange operator, since it may have been released"); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; - } - - int32_t index = pWrapper->sourceIndex; - SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); - - if (code == TSDB_CODE_SUCCESS) { - pSourceDataInfo->pRsp = pMsg->pData; - - SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->compLen = htonl(pRsp->compLen); - pRsp->numOfCols = htonl(pRsp->numOfCols); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); - - ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, - pRsp->numOfRows); - } else { - taosMemoryFree(pMsg->pData); - pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%s", pSourceDataInfo->taskId, index, tstrerror(code)); - } - - pSourceDataInfo->status = EX_SOURCE_DATA_READY; - - tsem_post(&pExchangeInfo->ready); - taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - - return TSDB_CODE_SUCCESS; -} - -void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); - - SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; - - if (pMsg->contLen > 0) { - buf.pData = taosMemoryCalloc(1, pMsg->contLen); - if (buf.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - memcpy(buf.pData, pMsg->pCont, pMsg->contLen); - } - } - - pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); - rpcFreeCont(pMsg->pCont); - destroySendMsgInfo(pSendInfo); -} - -static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex) { - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); - - ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); - - SFetchRspHandleWrapper* pWrapper = taosMemoryCalloc(1, sizeof(SFetchRspHandleWrapper)); - pWrapper->exchangeId = pExchangeInfo->self; - pWrapper->sourceIndex = sourceIndex; - - if (pSource->localExec) { - SDataBuf pBuf = {0}; - int32_t code = - (*pTaskInfo->localFetch.fp)(pTaskInfo->localFetch.handle, pSource->schedId, pTaskInfo->id.queryId, - pSource->taskId, 0, pSource->execId, &pBuf.pData, pTaskInfo->localFetch.explainRes); - loadRemoteDataCallback(pWrapper, &pBuf, code); - taosMemoryFree(pWrapper); - } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - taosMemoryFree(pWrapper); - return pTaskInfo->code; - } - - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, - pSource->execId, sourceIndex, totalSources); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); - taosMemoryFree(pWrapper); - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return pTaskInfo->code; - } - - pMsgSendInfo->param = pWrapper; - pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = pSource->fetchMsgType; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); - } - - return TSDB_CODE_SUCCESS; -} - -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator) { - pInfo->totalRows += numOfRows; - pInfo->totalSize += dataLen; - pInfo->totalElapsed += (taosGetTimestampUs() - startTs); - pOperator->resultInfo.totalRows += numOfRows; -} - -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { - if (pColList == NULL) { // data from other sources - blockDataCleanup(pRes); - *pNextStart = (char*)blockDecode(pRes, pData); - } else { // extract data according to pColList - char* pStart = pData; - - int32_t numOfCols = htonl(*(int32_t*)pStart); - pStart += sizeof(int32_t); - - // todo refactor:extract method - SSysTableSchema* pSchema = (SSysTableSchema*)pStart; - for (int32_t i = 0; i < numOfCols; ++i) { - SSysTableSchema* p = (SSysTableSchema*)pStart; - - p->colId = htons(p->colId); - p->bytes = htonl(p->bytes); - pStart += sizeof(SSysTableSchema); - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); - blockDataAppendColInfo(pBlock, &idata); - } - - blockDecode(pBlock, pStart); - blockDataEnsureCapacity(pRes, pBlock->info.rows); - - // data from mnode - pRes->info.rows = pBlock->info.rows; - relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - blockDataDestroy(pBlock); - } - - // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. - blockDataUpdateTsWindow(pRes, 0); - return TSDB_CODE_SUCCESS; -} - -static void* setAllSourcesCompleted(SOperatorInfo* pOperator, int64_t startTs) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int64_t el = taosGetTimestampUs() - startTs; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - - pLoadInfo->totalElapsed += el; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - qDebug("%s all %" PRIzu " sources are exhausted, total rows: %" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - - doSetOperatorCompleted(pOperator); - return NULL; -} - -static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, - SExecTaskInfo* pTaskInfo) { - int32_t code = 0; - int64_t startTs = taosGetTimestampUs(); - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - int32_t completed = 0; - for (int32_t k = 0; k < totalSources; ++k) { - SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - if (completed == totalSources) { - setAllSourcesCompleted(pOperator, startTs); - return; - } - - while (1) { -// printf("1\n"); - tsem_wait(&pExchangeInfo->ready); -// printf("2\n"); - - for (int32_t i = 0; i < totalSources; ++i) { - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { -// printf("========:%d is completed\n", i); - continue; - } - -// printf("index:%d - status:%d\n", i, pDataInfo->status); - if (pDataInfo->status != EX_SOURCE_DATA_READY) { -// printf("-----------%d, status:%d, continue\n", i, pDataInfo->status); - continue; - } - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - code = pDataInfo->code; - goto _error; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); - - // todo - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; -// printf("%d completed, try next\n", i); - - qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows, - pExchangeInfo->loadInfo.totalRows, completed, i + 1, totalSources); - taosMemoryFreeClear(pDataInfo->pRsp); - -// if (completed == totalSources) { -// return; -// } else { -// break; -// } - break; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - int32_t index = 0; - char* pStart = pRetrieveRsp->data; - while (index++ < pRetrieveRsp->numOfBlocks) { - printf("results, numOfBLock: %d\n", pRetrieveRsp->numOfBlocks); - SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); - if (code != 0) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - - taosArrayPush(pExchangeInfo->pResultBlockList, &pb); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); - -// int32_t completed = 0; - if (pRsp->completed == 1) { - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - -// for (int32_t k = 0; k < totalSources; ++k) { -// SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); -// if (p->status == EX_SOURCE_DATA_EXHAUSTED) { -// completed += 1; -// } -// } - - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 - ", total:%.2f Kb, completed:%d try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, - pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, - completed, i + 1, totalSources); - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, - pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); - } - - taosMemoryFreeClear(pDataInfo->pRsp); - - if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { - pDataInfo->status = EX_SOURCE_DATA_NOT_READY; - code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pDataInfo->pRsp); - goto _error; - } - } - -// if (completed == totalSources) { -// setAllSourcesCompleted(pOperator, startTs); -// } - - return; - } - - int32_t completed = 0; - for (int32_t k = 0; k < totalSources; ++k) { - SSourceDataInfo* p = taosArrayGet(pExchangeInfo->pSourceDataInfo, k); - if (p->status == EX_SOURCE_DATA_EXHAUSTED) { - completed += 1; - } - } - - if (completed == totalSources) { - return; - } - } - -_error: - pTaskInfo->code = code; -} - -static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return code; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%.2fms", GET_TASKID(pTaskInfo), - totalSources, (endTs - startTs) / 1000.0); - - pOperator->status = OP_RES_TO_RETURN; - pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); - tsem_post(&pExchangeInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - while (1) { - if (pExchangeInfo->current >= totalSources) { - setAllSourcesCompleted(pOperator, startTs); - return TSDB_CODE_SUCCESS; - } - - doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); - tsem_wait(&pExchangeInfo->ready); - - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); - SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - - if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), - pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code)); - pOperator->pTaskInfo->code = pDataInfo->code; - return pOperator->pTaskInfo->code; - } - - SRetrieveTableRsp* pRsp = pDataInfo->pRsp; - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 " try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1, - pDataInfo->totalRows, pLoadInfo->totalRows); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - taosMemoryFreeClear(pDataInfo->pRsp); - continue; - } - - SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - - char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); - - if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, - totalSources); - - pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; - pExchangeInfo->current += 1; - } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 - ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, - pLoadInfo->totalRows, pLoadInfo->totalSize); - } - - updateLoadRemoteInfo(pLoadInfo, pRetrieveRsp->numOfRows, pRetrieveRsp->compLen, startTs, pOperator); - pDataInfo->totalRows += pRetrieveRsp->numOfRows; - - taosMemoryFreeClear(pDataInfo->pRsp); - return TSDB_CODE_SUCCESS; - } -} - -static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - - SExchangeInfo* pExchangeInfo = pOperator->info; - if (!pExchangeInfo->seqLoadData) { - int32_t code = prepareConcurrentlyLoad(pOperator); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - OPTR_SET_OPENED(pOperator); - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return TSDB_CODE_SUCCESS; -} - -static void freeBlock(void* pParam) { - SSDataBlock* pBlock = *(SSDataBlock**)pParam; - blockDataDestroy(pBlock); -} - -static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - return NULL; - } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - - SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; - if (pOperator->status == OP_EXEC_DONE) { - qDebug("%s all %" PRIzu " source(s) are exhausted, total rows:%" PRIu64 " bytes:%" PRIu64 ", elapsed:%.2f ms", - GET_TASKID(pTaskInfo), totalSources, pLoadInfo->totalRows, pLoadInfo->totalSize, - pLoadInfo->totalElapsed / 1000.0); - return NULL; - } - - size_t size = taosArrayGetSize(pExchangeInfo->pResultBlockList); - if (size == 0 || pExchangeInfo->rspBlockIndex >= size) { - pExchangeInfo->rspBlockIndex = 0; - taosArrayClearEx(pExchangeInfo->pResultBlockList, freeBlock); - if (pExchangeInfo->seqLoadData) { - seqLoadRemoteData(pOperator); - } else { - concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } - - if (taosArrayGetSize(pExchangeInfo->pResultBlockList) == 0) { - return NULL; - } - } - - // we have buffered retrieved datablock, return it directly - return taosArrayGetP(pExchangeInfo->pResultBlockList, pExchangeInfo->rspBlockIndex++); -} - -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); - if (pBlock == NULL) { - return NULL; - } - - SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; - if (hasLimitOffsetInfo(pLimitInfo)) { - int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); - if (status == PROJECT_RETRIEVE_CONTINUE) { - continue; - } else if (status == PROJECT_RETRIEVE_DONE) { - size_t rows = pBlock->info.rows; - pExchangeInfo->limitInfo.numOfOutputRows += rows; - - if (rows == 0) { - doSetOperatorCompleted(pOperator); - return NULL; - } else { - return pBlock; - } - } - } else { - return pBlock; - } - } -} - -static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { - pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); - if (pInfo->pSourceDataInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SSourceDataInfo dataInfo = {0}; - dataInfo.status = EX_SOURCE_DATA_NOT_READY; - dataInfo.taskId = id; - dataInfo.index = i; - SSourceDataInfo* pDs = taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); - if (pDs == NULL) { - taosArrayDestroy(pInfo->pSourceDataInfo); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) { - size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); - - if (numOfSources == 0) { - qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); - return TSDB_CODE_INVALID_PARA; - } - - pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); - if (pInfo->pSources == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfSources; ++i) { - SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); - taosArrayPush(pInfo->pSources, pNode); - } - - initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); - pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); - - return initDataSource(numOfSources, pInfo, id); -} - -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - tsem_init(&pInfo->ready, 0, 0); - pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pInfo->pResultBlockList = taosArrayInit(1, POINTER_BYTES); - - pInfo->seqLoadData = false; - pInfo->pTransporter = pTransporter; - - pOperator->name = "ExchangeOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - doDestroyExchangeOperatorInfo(pInfo); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); @@ -2413,7 +1805,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2423,7 +1815,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pAggInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2672,7 +2064,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { if (pInfo->totalInputRows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -2749,7 +2141,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { while (true) { fillResult = doFillImpl(pOperator); if (fillResult == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2979,15 +2371,10 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->groupId = UINT64_MAX; - pOperator->name = "TableAggregate"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_AGG; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, NULL); + createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -3051,33 +2438,6 @@ void destroyFillOperatorInfo(void* param) { taosMemoryFreeClear(param); } -void destroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - taosRemoveRef(exchangeObjRefPool, pExInfo->self); -} - -void freeSourceDataInfo(void* p) { - SSourceDataInfo* pInfo = (SSourceDataInfo*)p; - taosMemoryFreeClear(pInfo->pRsp); -} - -void doDestroyExchangeOperatorInfo(void* param) { - SExchangeInfo* pExInfo = (SExchangeInfo*)param; - - taosArrayDestroy(pExInfo->pSources); - taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); - - if (pExInfo->pResultBlockList != NULL) { - taosArrayDestroyEx(pExInfo->pResultBlockList, freeBlock); - pExInfo->pResultBlockList = NULL; - } - - blockDataDestroy(pExInfo->pDummyBlock); - - tsem_destroy(&pExInfo->ready); - taosMemoryFreeClear(param); -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { @@ -3209,15 +2569,9 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pOperator->name = "FillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; + setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -3383,6 +2737,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + return NULL; + } + STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { @@ -3403,6 +2762,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + return NULL; + } STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 46a8baa682..47ee663abe 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -316,7 +316,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); if (!hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -438,15 +438,10 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -654,7 +649,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { // try next group data ++pInfo->groupIndex; if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); clearPartitionOperator(pInfo); return NULL; } @@ -821,17 +816,12 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; + setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -921,6 +911,8 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { blockDataDestroy(pResBlock); } } + taosArrayDestroy(pParInfo->rowIds); + pParInfo->rowIds = NULL; blockDataUpdateTsWindow(pDest, pInfo->tsColIndex); pDest->info.groupId = pParInfo->groupId; pOperator->resultInfo.totalRows += pDest->info.rows; @@ -966,7 +958,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->pInputDataBlock = NULL; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } printDataBlock(pBlock, "stream partitionby recv"); @@ -1019,6 +1011,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->tbnameCalSup); cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); + taosHashCleanup(pInfo->pPartitions); taosMemoryFreeClear(param); } @@ -1106,15 +1099,10 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); - pOperator->name = "StreamPartitionOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; + setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 45d76dce74..4e1daac643 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -73,14 +73,10 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; + + setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); @@ -121,8 +117,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -372,13 +367,13 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) if (leftTs == rightTs) { mergeJoinJoinDownstreamTsRanges(pOperator, leftTs, pRes, &nrows); - } else if (asc && leftTs < rightTs || !asc && leftTs > rightTs) { + } else if ((asc && leftTs < rightTs) || (!asc && leftTs > rightTs)) { pJoinInfo->leftPos += 1; if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { continue; } - } else if (asc && leftTs > rightTs || !asc && leftTs < rightTs) { + } else if ((asc && leftTs > rightTs) || (!asc && leftTs < rightTs)) { pJoinInfo->rightPos += 1; if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { continue; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index b2865d15f0..ce1d13775c 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -98,13 +98,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys } pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -153,7 +149,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return PROJECT_RETRIEVE_DONE; } @@ -187,7 +183,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS // TODO: optimize it later when partition by + limit if ((pLimitInfo->slimit.limit == -1 && pLimitInfo->currentGroupId == 0) || (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -252,7 +248,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { @@ -400,14 +396,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - pOperator->name = "IndefinitOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, - destroyIndefinitOperatorInfo, NULL); + setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -499,7 +489,7 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -628,7 +618,7 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (pOperator->cost.openCost == 0) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 34d6b3016d..b83fac9fa6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -819,7 +819,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -842,7 +842,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -864,7 +864,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -946,13 +946,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; - pOperator->name = "TableScanOperator"; // for debug purpose - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->metaCache.pTableMetaEntryCache == NULL) { @@ -961,7 +956,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 @@ -985,14 +980,8 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - pOperator->name = "TableSeqScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL); + setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1147,15 +1136,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - pOperator->name = "DataBlockDistScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL); + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; _error: @@ -1437,7 +1419,7 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. SSessionKey startWin = {0}; - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin); + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin); if (IS_INVALID_SESSION_WIN_KEY(startWin)) { // window has been closed. continue; @@ -2369,11 +2351,9 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; - pOperator->name = "RawScanOperator"; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; _end: @@ -2557,16 +2537,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->partitionSup.needCalc = false; - pOperator->name = "StreamScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); return pOperator; @@ -2901,7 +2876,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2954,7 +2929,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3744,7 +3719,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } if (i >= taosArrayGetSize(pIdx->uids)) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } else { pIdx->lastIdx = i + 1; } @@ -3926,7 +3901,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { if (ret != 0) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; @@ -3948,7 +3923,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { if (pInfo->showRewrite == false) { @@ -4200,15 +4175,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; _error: @@ -4284,7 +4253,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { count += 1; if (++pInfo->curPos >= size) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } } @@ -4336,18 +4305,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - + setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); return pOperator; @@ -4714,7 +4676,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; if (tableListSize == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->tableStartIndex = 0; @@ -4733,7 +4695,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } else { stopGroupTableMergeScan(pOperator); if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; @@ -4853,15 +4815,10 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - pOperator->name = "TableMergeScanOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index af87c4b2eb..fc53623d44 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -53,11 +53,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); - pOperator->name = "SortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfCols; @@ -67,7 +63,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -214,7 +210,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pInfo); if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -428,7 +424,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); if (pInfo->prefetchedSortInput == NULL) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; @@ -453,7 +449,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { beginSortGroup(pOperator); } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { finishSortGroup(pOperator); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -509,15 +505,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort } pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - - pOperator->name = "GroupSortOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, + setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); @@ -705,7 +694,7 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; } else { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } return pBlock; @@ -774,14 +763,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->bufPageSize = getProperSortPageSize(rowSize); pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. - pOperator->name = "MultiwayMerge"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL, + setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 9f06e639b3..7c9d73ad13 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1443,7 +1443,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRes, "stream fill"); return pInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1512,7 +1512,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } if (pInfo->pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); resetStreamFillInfo(pInfo); return NULL; } @@ -1690,15 +1690,9 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->srcRowIndex = 0; - - pOperator->name = "StreamFillOperator"; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 46e87527f1..4c369e8802 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1221,7 +1221,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -1232,7 +1232,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1269,7 +1269,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1739,7 +1739,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); - pOperator->pTaskInfo = pTaskInfo; pInfo->win = pTaskInfo->window; pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -1777,15 +1776,11 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh } initResultRowInfo(&pInfo->binfo.resultRowInfo); - - pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1890,7 +1885,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -1933,7 +1928,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { bool hasRemain = hasRemainResults(&pInfo->groupResInfo); if (!hasRemain) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2281,7 +2276,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -2330,7 +2325,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2342,7 +2337,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2365,7 +2360,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2386,7 +2381,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } else { @@ -2448,7 +2443,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2463,7 +2458,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } } @@ -2557,15 +2552,9 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pScanInfo->cond.twindows = pInfo->win; pScanInfo->cond.type = TIMEWINDOW_RANGE_EXTERNAL; - pOperator->name = "TimeSliceOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyTimeSliceOperatorInfo, NULL); + setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -2633,15 +2622,11 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2711,14 +2696,10 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW goto _error; } - pOperator->name = "SessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - + setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3134,7 +3115,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -3403,7 +3384,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -3550,7 +3531,7 @@ void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - int32_t code = streamStateSessionGetKey(pAggSup->pState, pKey, pKey); + int32_t code = streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } @@ -3561,10 +3542,11 @@ bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->session void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SResultWindowInfo* pCurWin) { pCurWin->sessionWin.groupId = groupId; - pCurWin->sessionWin.win.skey = startTs - pAggSup->gap; - pCurWin->sessionWin.win.ekey = endTs + pAggSup->gap; + pCurWin->sessionWin.win.skey = startTs; + pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, &pCurWin->pOutputBuf, &size); + int32_t code = + streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, &pCurWin->pOutputBuf, &size); if (code == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; } else { @@ -3575,7 +3557,7 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, (const void**)&pWinInfo->pOutputBuf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, &pWinInfo->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3680,7 +3662,7 @@ SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* setSessionWinOutputInfo(pStUpdated, pNextWin); int32_t size = 0; pNextWin->sessionWin = pCurWin->sessionWin; - int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, (const void**)&pNextWin->pOutputBuf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, &pNextWin->pOutputBuf, &size); if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(*pNextWin); } @@ -3870,7 +3852,7 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataAppendNULL(pCalEdCol, pBlock->info.rows); - SHashObj* pGroupIdTbNameMap; + SHashObj* pGroupIdTbNameMap = NULL; if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOp->info; @@ -3914,9 +3896,11 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; - SStreamStateCur* pCur = streamStateSessionGetCur(pChAggSup->pState, pWinKey); - SResultRow* pResult = NULL; - SResultRow* pChResult = NULL; + SSessionKey chWinKey = *pWinKey; + chWinKey.win.ekey = chWinKey.win.skey; + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); + SResultRow* pResult = NULL; + SResultRow* pChResult = NULL; while (1) { SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; @@ -4044,7 +4028,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4132,6 +4116,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); +#if 0 + char* pBuf = streamStateSessionDump(pAggSup->pState); + qDebug("===stream===final session%s", pBuf); + taosMemoryFree(pBuf); +#endif + doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); @@ -4144,7 +4134,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4211,13 +4201,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); @@ -4268,7 +4256,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } } @@ -4326,6 +4314,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); +#if 0 + char* pBuf = streamStateSessionDump(pAggSup->pState); + qDebug("===stream===semi session%s", pBuf); + taosMemoryFree(pBuf); +#endif + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); @@ -4341,7 +4335,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4352,20 +4346,21 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pOperator == NULL) { goto _error; } + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { - pInfo->isFinal = true; - pOperator->name = "StreamSessionFinalAggOperator"; - } else { - pInfo->isFinal = false; + pInfo->isFinal = (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION); + char* name = (pInfo->isFinal) ? "StreamSessionFinalAggOperator" : "StreamSessionSemiAggOperator"; + + if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->name = "StreamSessionSemiAggOperator"; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); } + setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); + pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); @@ -4595,7 +4590,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4661,7 +4656,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { printDataBlock(pBInfo->pRes, "single state"); return pBInfo->pRes; } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); return NULL; } @@ -4726,14 +4721,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamStateAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); @@ -4881,7 +4872,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { cleanupAfterGroupResultGen(pMiaInfo, pRes); } - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); break; } @@ -5006,16 +4997,11 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, + false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5259,7 +5245,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pRes->info.rows == 0) { - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); } size_t rows = pRes->info.rows; @@ -5318,16 +5304,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge } initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - - pOperator->name = "TimeMergeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; - + setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, + OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5371,7 +5351,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, &pInfo->delKey); - doSetOperatorCompleted(pOperator); + setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); return NULL; } @@ -5555,13 +5535,10 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pGroupIdTbNameMap = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - pOperator->name = "StreamIntervalOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, - destroyStreamFinalIntervalOperatorInfo, NULL); + setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index afe639b856..d3f03e8e9c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2734,7 +2734,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mode", .type = FUNCTION_TYPE_MODE, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, .translateFunc = translateMode, .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1501bb6d67..079e553b07 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -255,8 +255,9 @@ typedef struct SUniqueInfo { } SUniqueInfo; typedef struct SModeItem { - int64_t count; - char data[]; + int64_t count; + STuplePos tuplePos; + char data[]; } SModeItem; typedef struct SModeInfo { @@ -264,6 +265,10 @@ typedef struct SModeInfo { uint8_t colType; int16_t colBytes; SHashObj* pHash; + + STuplePos nullTuplePos; + bool nullTupleSaved; + char pItems[]; } SModeInfo; @@ -906,6 +911,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_FLOAT: { float* plist = (float*)pCol->pData; +// float val = 0; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -913,8 +919,9 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { numOfElem += 1; pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; + pAvgRes->sum.dsum += plist[i]; } +// pAvgRes->sum.dsum = val; break; } @@ -1273,14 +1280,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1304,14 +1317,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1335,14 +1354,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1366,14 +1391,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1399,14 +1430,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1430,14 +1467,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1461,14 +1504,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1492,14 +1541,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1524,14 +1579,20 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->assign = true; } else { // ignore the equivalent data value - if ((*val) == pData[i]) { - continue; - } - - if ((*val < pData[i]) ^ isMinFunc) { - *val = pData[i]; - if (pCtx->subsidiaries.num > 0) { - updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } } } } @@ -1554,7 +1615,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } pBuf->assign = true; } else { - // ignore the equivalent data value +#if 0 if ((*val) == pData[i]) { continue; } @@ -1565,6 +1626,23 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } +#endif + // NOTE: An faster version to avoid one additional comparison with FPU. + if (isMinFunc) { // min + if (*val > pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } else { // max + if (*val < pData[i]) { + *val = pData[i]; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + } + } + } } numOfElems += 1; @@ -2929,6 +3007,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } #else + int64_t* pts = (int64_t*) pInput->pPTS->pData; for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { continue; @@ -2937,13 +3016,14 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { numOfElems++; char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); + TSKEY cts = pts[i]; if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); pResInfo->numOfRes = 1; } } #endif + if (numOfElems == 0) { // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); @@ -3015,26 +3095,87 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } #else - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - continue; - } + int64_t* pts = (int64_t*)pInput->pPTS->pData; - numOfElems++; +#if 0 + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } - char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveCurrentVal(pCtx, i, cts, type, data); - pResInfo->numOfRes = 1; + numOfElems++; + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } +#else + if (!pInputCol->hasNull) { + numOfElems = 1; + + int32_t round = pInput->numOfRows >> 2; + int32_t reminder = pInput->numOfRows & 0x03; + + int32_t tick = 0; + for (int32_t i = pInput->startRowIndex; tick < round; i += 4, tick += 1) { + int64_t cts = pts[i]; + int32_t chosen = i; + + if (cts < pts[i + 1]) { + cts = pts[i + 1]; + chosen = i + 1; + } + + if (cts < pts[i + 2]) { + cts = pts[i + 2]; + chosen = i + 2; + } + + if (cts < pts[i + 3]) { + cts = pts[i + 3]; + chosen = i + 3; + } + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + char* data = colDataGetData(pInputCol, chosen); + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } + + for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } + } else { + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) { + char* data = colDataGetData(pInputCol, i); + doSaveCurrentVal(pCtx, i, pts[i], type, data); + pResInfo->numOfRes = 1; + } + } } - } #endif + +#endif + + // save selectivity value for column consisted of all null values if (numOfElems == 0) { - // save selectivity value for column consisted of all null values firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); } - SET_VAL(pResInfo, numOfElems, 1); + +// SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3184,8 +3325,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. - // this is NOT always works if project operator exists in downstream. + // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. + // this assumption is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); @@ -3211,11 +3352,13 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { } } #else + + int64_t* pts = (int64_t*)pInput->pPTS->pData; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); - TSKEY cts = getRowPTs(pInput->pPTS, i); - numOfElems++; + TSKEY cts = pts[i]; + numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); pResInfo->numOfRes = 1; @@ -5391,10 +5534,13 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } else { pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } + pInfo->nullTupleSaved = false; + pInfo->nullTuplePos.pageId = -1; + return true; } -static void doModeAdd(SModeInfo* pInfo, char* data) { +static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -5403,10 +5549,17 @@ static void doModeAdd(SModeInfo* pInfo, char* data) { memcpy(pItem->data, data, hashKeyBytes); pItem->count += 1; + if (pCtx->subsidiaries.num > 0) { + pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, NULL); + } + taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); pInfo->numOfPoints++; } else { (*pHashItem)->count += 1; + if (pCtx->subsidiaries.num > 0) { + updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + } } } @@ -5428,7 +5581,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } numOfElems++; - doModeAdd(pInfo, data); + doModeAdd(pInfo, i, pCtx, data); if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { taosHashCleanup(pInfo->pHash); @@ -5436,6 +5589,11 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { + pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); + pInfo->nullTupleSaved = true; + } + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -5461,8 +5619,10 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); colDataAppend(pCol, currentRow, pResItem->data, false); + setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); } else { colDataAppendNULL(pCol, currentRow); + setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } taosHashCleanup(pInfo->pHash); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 1dedfe8364..8715aa0be1 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -131,7 +131,8 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char udfdPathLdLib[1024] = {0}; size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); - strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, tListLen(udfdPathLdLib)); + udfdPathLdLib[udfdLdLibPathLen] = ':'; strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 10a86866d5..c53d3f9320 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -108,7 +108,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum); + int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, bool* pPass); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d3ff787f50..05fc651618 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -466,14 +466,14 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName, } static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SParseContext* pParCxt = pCxt->pParseCxt; int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs); if (TSDB_CODE_SUCCESS == code) { if (pParCxt->async) { - code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum); + code = getDbVgVersionFromCache(pCxt->pMetaCache, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } else { - code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum); + code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum, pStateTs); } } if (TSDB_CODE_SUCCESS != code) { @@ -1252,6 +1252,19 @@ static bool dataTypeEqual(const SDataType* l, const SDataType* r) { return (l->type == r->type && l->bytes == r->bytes && l->precision == r->precision && l->scale == r->scale); } +// 0 means equal, 1 means the left shall prevail, -1 means the right shall prevail +static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { + if (l->type != r->type) { + return 1; + } + + if (l->bytes != r->bytes) { + return l->bytes > r->bytes ? 1 : -1; + } + + return (l->precision == r->precision && l->scale == r->scale) ? 0 : 1; +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); @@ -1876,10 +1889,15 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW } pWhenThen->pWhen = pIsTrue; } - if (first) { - first = false; + if (first || dataTypeComp(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType) < 0) { pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; - } else if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { + } + first = false; + } + + FOREACH(pNode, pCaseWhen->pWhenThenList) { + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pWhenThen->pThen, pCaseWhen->node.resType, &pCastFunc); if (TSDB_CODE_SUCCESS != pCxt->errCode) { @@ -1889,6 +1907,7 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW pWhenThen->node.resType = pCaseWhen->node.resType; } } + if (NULL != pCaseWhen->pElse && !dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pCaseWhen->pElse)->resType)) { SNode* pCastFunc = NULL; pCxt->errCode = createCastFunc(pCxt, pCaseWhen->pElse, pCaseWhen->node.resType, &pCastFunc); @@ -2181,7 +2200,8 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge if (TSDB_DB_NAME_T == pTargetName->type) { int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); - if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { code = TSDB_CODE_SUCCESS; } return code; @@ -2196,7 +2216,8 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge } else { taosArrayPush(*pVgroupList, &vgInfo); } - } else if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + } else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code) { code = TSDB_CODE_SUCCESS; } return code; @@ -2391,6 +2412,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { pCxt->stableQuery = true; } + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { + ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + } code = addNamespace(pCxt, pRealTable); } break; @@ -3021,20 +3045,17 @@ static int32_t translateIntervalWindow(STranslateContext* pCxt, SSelectStmt* pSe return code; } -static EDealRes checkStateExpr(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - STranslateContext* pCxt = pContext; - SColumnNode* pCol = (SColumnNode*)pNode; - - int32_t type = pCol->node.resType.type; - if (!IS_INTEGER_TYPE(type) && type != TSDB_DATA_TYPE_BOOL && !IS_VAR_DATA_TYPE(type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); - } - if (COLUMN_TYPE_TAG == pCol->colType) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); - } +static int32_t checkStateExpr(STranslateContext* pCxt, SNode* pNode) { + int32_t type = ((SExprNode*)pNode)->resType.type; + if (!IS_INTEGER_TYPE(type) && type != TSDB_DATA_TYPE_BOOL && !IS_VAR_DATA_TYPE(type)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); } - return DEAL_RES_CONTINUE; + + if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); + } + + return TSDB_CODE_SUCCESS; } static bool hasPartitionByTbname(SNodeList* pPartitionByList) { @@ -3066,11 +3087,11 @@ static int32_t translateStateWindow(STranslateContext* pCxt, SSelectStmt* pSelec } SStateWindowNode* pState = (SStateWindowNode*)pSelect->pWindow; - nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = checkStateWindowForStream(pCxt, pSelect); + int32_t code = checkStateExpr(pCxt, pState->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = checkStateWindowForStream(pCxt, pSelect); } - return pCxt->errCode; + return code; } static int32_t translateSessionWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -3441,7 +3462,8 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) { SExprNode* pLeftExpr = (SExprNode*)pLeft; SExprNode* pRightExpr = (SExprNode*)pRight; - if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { + int32_t comp = dataTypeComp(&pLeftExpr->resType, &pRightExpr->resType); + if (comp > 0) { SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); if (TSDB_CODE_SUCCESS != code) { @@ -3449,9 +3471,20 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } REPLACE_LIST2_NODE(pRightFunc); pRightExpr = (SExprNode*)pRightFunc; + } else if (comp < 0) { + SNode* pLeftFunc = NULL; + int32_t code = createCastFunc(pCxt, pLeft, pRightExpr->resType, &pLeftFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST1_NODE(pLeftFunc); + SExprNode* pLeftFuncExpr = (SExprNode*)pLeftFunc; + snprintf(pLeftFuncExpr->aliasName, sizeof(pLeftFuncExpr->aliasName), "%s", pLeftExpr->aliasName); + snprintf(pLeftFuncExpr->userAlias, sizeof(pLeftFuncExpr->userAlias), "%s", pLeftExpr->userAlias); + pLeft = pLeftFunc; + pLeftExpr = pLeftFuncExpr; } - strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); - pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; + snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pSetOperator->stmtName, pLeft))) { return TSDB_CODE_OUT_OF_MEMORY; @@ -4942,7 +4975,8 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameExtractFullName(&name, usedbReq.db); - int32_t code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable); + int32_t code = + getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 466c6edd24..00a72a1946 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -876,13 +876,14 @@ int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCac } int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, - int32_t* pTableNum) { + int32_t* pTableNum, int64_t* pStateTs) { SDbInfo* pDbInfo = NULL; int32_t code = getMetaDataFromHash(pDbFName, strlen(pDbFName), pMetaCache->pDbInfo, (void**)&pDbInfo); if (TSDB_CODE_SUCCESS == code) { *pVersion = pDbInfo->vgVer; *pDbId = pDbInfo->dbId; *pTableNum = pDbInfo->tbNum; + *pStateTs = pDbInfo->stateTs; } return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 4f5ddd9a51..8f051c67a0 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -249,7 +249,7 @@ int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, con } int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, - int32_t* tableNum) { + int32_t* tableNum, int64_t* stateTs) { return 0; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 0027ac9ca1..fcd8dd1f26 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -425,6 +425,8 @@ TEST_F(ParserSelectTest, informationSchema) { run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); + + run("SELECT * FROM (SELECT table_name FROM ins_tables) t WHERE table_name = 'a'"); } TEST_F(ParserSelectTest, withoutFrom) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0af8f5075e..249ba1815d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -66,16 +66,36 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SLogicNode* pNod return pSubplan; } -static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, ESubplanType subplanType) { +static bool splHasScan(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return true; + } + + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) { + return true; + } + return splHasScan((SLogicNode*)pChild); + } + + return false; +} + +static void splSetSubplanType(SLogicSubplan* pSubplan) { + pSubplan->subplanType = splHasScan(pSubplan->pNode) ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE; +} + +static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; } pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; - pSubplan->subplanType = subplanType; pSubplan->pNode = pNode; pNode->pParent = NULL; + splSetSubplanType(pSubplan); return pSubplan; } @@ -1204,7 +1224,7 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl SNode* pChild = NULL; FOREACH(pChild, pSplitNode->pChildren) { - SLogicSubplan* pNewSubplan = splCreateSubplan(pCxt, (SLogicNode*)pChild, pUnionSubplan->subplanType); + SLogicSubplan* pNewSubplan = splCreateSubplan(pCxt, (SLogicNode*)pChild); code = nodesListMakeStrictAppend(&pUnionSubplan->pChildren, (SNode*)pNewSubplan); if (TSDB_CODE_SUCCESS == code) { REPLACE_NODE(NULL); @@ -1390,10 +1410,9 @@ static int32_t insertSelectSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SLogicSubplan* pNewSubplan = NULL; SNodeList* pSubplanChildren = info.pSubplan->pChildren; - ESubplanType subplanType = info.pSubplan->subplanType; int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pQueryRoot, SUBPLAN_TYPE_MODIFY); if (TSDB_CODE_SUCCESS == code) { - pNewSubplan = splCreateSubplan(pCxt, info.pQueryRoot, subplanType); + pNewSubplan = splCreateSubplan(pCxt, info.pQueryRoot); if (NULL == pNewSubplan) { code = TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 6985bc8807..3dafaa8bf5 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -29,7 +29,7 @@ TEST_F(PlanStateTest, basic) { TEST_F(PlanStateTest, stateExpr) { useDb("root", "test"); - run("SELECT COUNT(*) FROM t1 STATE_WINDOW(c1 + 10)"); + run("SELECT COUNT(*) FROM t1 STATE_WINDOW(CASE WHEN c1 > 10 THEN 1 ELSE 0 END)"); } TEST_F(PlanStateTest, selectFunc) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cf064881c2..6eadf80e3d 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -146,6 +146,12 @@ void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { } taosMemoryFreeClear(pMsgBody); } +void destroyAhandle(void *ahandle) { + SMsgSendInfo *pSendInfo = ahandle; + if (pSendInfo == NULL) return; + + destroySendMsgInfo(pSendInfo); +} int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 953abd5956..fadc39f21d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -41,8 +41,9 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix; pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix; + pOut->dbVgroup->stateTs = usedbRsp->stateTs; - qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); + qDebug("Got %d vgroup for db %s, vgVersion:%d, stateTs:%" PRId64, usedbRsp->vgNum, usedbRsp->db, usedbRsp->vgVersion, usedbRsp->stateTs); if (usedbRsp->vgNum <= 0) { return TSDB_CODE_SUCCESS; @@ -103,6 +104,7 @@ int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *ms usedbReq.vgVersion = pInput->vgVersion; usedbReq.dbId = pInput->dbId; usedbReq.numOfTable = pInput->numOfTable; + usedbReq.stateTs = pInput->stateTs; int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); void *pBuf = (*mallcFp)(bufLen); @@ -306,6 +308,15 @@ int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_USEDB_OVER; } + qTrace("db:%s, usedbRsp received, numOfVgroups:%d", usedbRsp.db, usedbRsp.vgNum); + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo *pInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); + qTrace("vgId:%d, numOfEps:%d inUse:%d ", pInfo->vgId, pInfo->epSet.numOfEps, pInfo->epSet.inUse); + for (int32_t j = 0; j < pInfo->epSet.numOfEps; ++j) { + qTrace("vgId:%d, index:%d epset:%s:%u", pInfo->vgId, j, pInfo->epSet.eps[j].fqdn, pInfo->epSet.eps[j].port); + } + } + code = queryBuildUseDbOutput(pOut, &usedbRsp); PROCESS_USEDB_OVER: diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 939be1809a..df9a818fee 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -912,8 +912,8 @@ int32_t filterDetachCnfGroups(SArray *group, SArray *left, SArray *right) { if (taosArrayGetSize(left) <= 0) { if (taosArrayGetSize(right) <= 0) { - fltError("both groups are empty"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + fltDebug("both groups are empty"); + return TSDB_CODE_SUCCESS; } SFilterGroup *gp = NULL; @@ -1169,7 +1169,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; - out.columnData->info.bytes = tDataTypes[type].bytes; + out.columnData->info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; //reserved space for simple_copy for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { SValueNode *valueNode = (SValueNode *)cell->pNode; @@ -1191,7 +1191,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); out.columnData->pData = NULL; } else { - void *data = taosMemoryCalloc(1, tDataTypes[type].bytes); + void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); //reserved space for simple_copy if (NULL == data) { FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c89072233f..22fa99d75d 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1028,11 +1028,11 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t type = GET_PARAM_TYPE(pInput); bool tzPresent = (inputNum == 2) ? true : false; - char *tz; - int32_t tzLen; + char tz[20] = {0}; + int32_t tzLen = 0; if (tzPresent) { - tz = varDataVal(pInput[1].columnData->pData); tzLen = varDataLen(pInput[1].columnData->pData); + memcpy(tz, varDataVal(pInput[1].columnData->pData), tzLen); } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -1071,8 +1071,10 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t len = (int32_t)strlen(buf); // add timezone string - snprintf(buf + len, tzLen + 1, "%s", tz); - len += tzLen; + if (tzLen > 0) { + snprintf(buf + len, tzLen + 1, "%s", tz); + len += tzLen; + } if (hasFraction) { int32_t fracLen = (int32_t)strlen(fraction) + 1; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 801f38bd1a..22fb66d92f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -425,6 +425,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { _return: taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -438,6 +439,7 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { code); if (pMsg) { taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); } return TSDB_CODE_SUCCESS; } @@ -492,6 +494,7 @@ _return: tFreeSSchedulerHbRsp(&rsp); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); SCH_RET(code); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index dbdccff302..7e5b3faedb 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -439,6 +439,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(code); @@ -446,6 +448,8 @@ _return: taosMemoryFree(pData->pData); taosMemoryFree(pData->pEpSet); + pData->pData = NULL; + pData->pEpSet = NULL; SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -942,7 +946,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &qwMsg, explainRes)); + pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); @@ -1115,7 +1119,7 @@ int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { } SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, - pTask->execId, &pRsp, explainRes)); + pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index ad342edfa0..573a1ea31f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -118,8 +118,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -200,8 +199,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = 0; - blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 98e4e77cb0..682bfbaf8d 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -70,6 +70,7 @@ _err: void streamMetaClose(SStreamMeta* pMeta) { tdbCommit(pMeta->db, &pMeta->txn); + tdbPostCommit(pMeta->db, &pMeta->txn); tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pCheckpointDb); tdbClose(pMeta->db); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 9829955baf..ccb0dd4a92 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -29,7 +29,7 @@ typedef struct SStateSessionKey { int64_t opNum; } SStateSessionKey; -static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { +static inline int sessionRangeKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { if (pWin1->groupId > pWin2->groupId) { return 1; } else if (pWin1->groupId < pWin2->groupId) { @@ -45,6 +45,28 @@ static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pW return 0; } +static inline int sessionWinKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; + } + + if (pWin1->win.skey > pWin2->win.skey) { + return 1; + } else if (pWin1->win.skey < pWin2->win.skey) { + return -1; + } + + if (pWin1->win.ekey > pWin2->win.ekey) { + return 1; + } else if (pWin1->win.ekey < pWin2->win.ekey) { + return -1; + } + + return 0; +} + static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStateSessionKey* pWin1 = (SStateSessionKey*)pKey1; SStateSessionKey* pWin2 = (SStateSessionKey*)pKey2; @@ -55,7 +77,7 @@ static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* return -1; } - return sessionKeyCmpr(&pWin1->key, &pWin2->key); + return sessionWinKeyCmpr(&pWin1->key, &pWin2->key); } static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { @@ -142,6 +164,7 @@ _err: void streamStateClose(SStreamState* pState) { tdbCommit(pState->db, &pState->txn); + tdbPostCommit(pState->db, &pState->txn); tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); @@ -168,6 +191,9 @@ int32_t streamStateCommit(SStreamState* pState) { if (tdbCommit(pState->db, &pState->txn) < 0) { return -1; } + if (tdbPostCommit(pState->db, &pState->txn) < 0) { + return -1; + } memset(&pState->txn, 0, sizeof(TXN)); if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { @@ -396,7 +422,6 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key SStateKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -422,7 +447,6 @@ SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -448,7 +472,6 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -492,33 +515,18 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, cons return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); } -SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSessionKey* key) { - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL); - - int32_t c = -2; - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c); - if (c != 0) { - streamStateFreeCur(pCur); - return NULL; - } - pCur->number = pState->number; - return pCur; -} - int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); + SSessionKey resKey = *key; void* tmp = NULL; - if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); + if (code == 0) { + *key = resKey; *pVal = tdbRealloc(NULL, *pVLen); memcpy(*pVal, tmp, *pVLen); - streamStateFreeCur(pCur); - return 0; } streamStateFreeCur(pCur); - return -1; + return code; } int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { @@ -540,7 +548,6 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -554,6 +561,34 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, cons return pCur; } +SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + if (c <= 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { @@ -568,7 +603,6 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - tdbTbcClose(pCur->pCur); streamStateFreeCur(pCur); return NULL; } @@ -582,13 +616,13 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess return pCur; } -int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen) { +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) { if (!pCur) { return -1; } - const SStateSessionKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + SStateSessionKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { return -1; } if (pKTmp->opNum != pCur->number) { @@ -602,15 +636,15 @@ int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, c } int32_t streamStateSessionClear(SStreamState* pState) { - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; - streamStateSessionPut(pState, &key, NULL, 0); - SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pState, &key); + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); while (1) { SSessionKey delKey = {0}; void* buf = NULL; int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, buf, &size); + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); if (code == 0) { + ASSERT(size > 0); memset(buf, 0, size); streamStateSessionPut(pState, &delKey, buf, size); } else { @@ -619,63 +653,106 @@ int32_t streamStateSessionClear(SStreamState* pState) { streamStateCurNext(pState, pCur); } streamStateFreeCur(pCur); - streamStateSessionDel(pState, &key); return 0; } -SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - SSessionKey resKey = *key; - while (1) { +int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return -1; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return -1; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + streamStateFreeCur(pCur); + return -1; + } + + SSessionKey resKey = *key; + int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; + } + + if (c > 0) { + streamStateCurNext(pState, pCur); + code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; + } + } else if (c < 0) { streamStateCurPrev(pState, pCur); - SSessionKey tmpKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); - if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { - resKey = tmpKey; - } else { - break; + code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); + if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { + *curKey = resKey; + streamStateFreeCur(pCur); + return code; } } + streamStateFreeCur(pCur); - return streamStateSessionGetRanomCur(pState, &resKey); + return -1; } -int32_t streamStateSessionGetKey(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - SSessionKey resKey = *key; - int32_t res = -1; - while (1) { - SSessionKey tmpKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); - if (code == 0 && sessionKeyCmpr(key, &tmpKey) == 0) { - res = 0; - resKey = tmpKey; - streamStateCurPrev(pState, pCur); - } else { - break; +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, + int32_t* pVLen) { + // todo refactor + int32_t res = 0; + SSessionKey originKey = *key; + SSessionKey searchKey = *key; + searchKey.win.skey = key->win.skey - gap; + searchKey.win.ekey = key->win.ekey + gap; + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); + if (!tmp) { + return -1; + } + + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + if (code == 0) { + if (sessionRangeKeyCmpr(&searchKey, key) == 0) { + memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); + goto _end; + } + streamStateCurNext(pState, pCur); + } else { + *key = originKey; + streamStateFreeCur(pCur); + pCur = streamStateSessionSeekKeyNext(pState, key); + } + + code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); + if (code == 0) { + if (sessionRangeKeyCmpr(&searchKey, key) == 0) { + memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); + goto _end; } } - *curKey = resKey; + + *key = originKey; + res = 1; + memset(tmp, 0, valSize); + +_end: + + *pVal = tmp; streamStateFreeCur(pCur); return res; } -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { - // todo refactor - SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); - int32_t size = *pVLen; - void* tmp = NULL; - *pVal = tdbRealloc(NULL, size); - memset(*pVal, 0, size); - if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { - memcpy(*pVal, tmp, *pVLen); - streamStateFreeCur(pCur); - return 0; - } - streamStateFreeCur(pCur); - return 1; -} - int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { // todo refactor @@ -688,16 +765,18 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch } SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } @@ -708,11 +787,12 @@ int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, ch pCur = streamStateSessionSeekKeyNext(pState, key); } - code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); if (code == 0) { void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); if (fn(pKeyData, stateKey) == true) { memcpy(tmp, *pVal, valSize); + streamStateSessionDel(pState, key); goto _end; } } @@ -742,8 +822,11 @@ char* streamStateSessionDump(SStreamState* pState) { tdbTbcMoveToFirst(pCur->pCur); SSessionKey key = {0}; - int32_t code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0); + void* buf = NULL; + int32_t bufSize = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &key, &buf, &bufSize); if (code != 0) { + streamStateFreeCur(pCur); return NULL; } @@ -758,12 +841,14 @@ char* streamStateSessionDump(SStreamState* pState) { key = (SSessionKey){0}; code = streamStateSessionGetKVByCur(pCur, &key, NULL, 0); if (code != 0) { + streamStateFreeCur(pCur); return dumpBuf; } len += snprintf(dumpBuf + len, size - len, "||s:%15" PRId64 ",", key.win.skey); len += snprintf(dumpBuf + len, size - len, "e:%15" PRId64 ",", key.win.ekey); len += snprintf(dumpBuf + len, size - len, "g:%15" PRId64 "||", key.groupId); } + streamStateFreeCur(pCur); return dumpBuf; } #endif diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index dc40e2fc72..0a67939d35 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 0227d832fc..d2dff92d43 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleAppendEntriesResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncCommit.h b/source/libs/sync/inc/syncCommit.h index 7458ce28ab..d3ba556f82 100644 --- a/source/libs/sync/inc/syncCommit.h +++ b/source/libs/sync/inc/syncCommit.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // \* Leader i advances its commitIndex. // \* This is done as a separate step from handling AppendEntries responses, diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 9ccd9dd28f..662c17094e 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // RequestVote(i, j) == diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index e8f17537b4..bd88f5cdce 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3085b6d8f4..8a951ba38d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -268,8 +268,6 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 19079d518d..936081c7b2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 15ca82664a..6b6ca21399 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define CONFIG_FILE_LEN 2048 diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index bab1dcc661..2a73b95e4d 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -20,13 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" -#include "tref.h" #include "tskiplist.h" typedef struct SSyncRaftEntry { @@ -42,25 +37,14 @@ typedef struct SSyncRaftEntry { char data[]; // origin RpcMsg.pCont } SSyncRaftEntry; -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); // step 4 -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuild(int32_t dataLen); +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg); SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId); void syncEntryDestory(SSyncRaftEntry* pEntry); -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); // step 5 -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); // step 6 -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); -char* syncEntry2Str(const SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj); -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); -void syncEntryLog(const SSyncRaftEntry* pObj); -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); - -//----------------------------------- typedef struct SRaftEntryHashCache { SHashObj* pEntryHash; int32_t maxCount; @@ -78,14 +62,6 @@ int32_t raftCacheDelEntry(struct SRaftEntryHashCache* pCache, SyncI int32_t raftCacheGetAndDel(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftCacheClear(struct SRaftEntryHashCache* pCache); -cJSON* raftCache2Json(SRaftEntryHashCache* pObj); -char* raftCache2Str(SRaftEntryHashCache* pObj); -void raftCachePrint(SRaftEntryHashCache* pObj); -void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); -void raftCacheLog(SRaftEntryHashCache* pObj); -void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); - -//----------------------------------- typedef struct SRaftEntryCache { SSkipList* pSkipList; int32_t maxCount; @@ -102,13 +78,6 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncInde int32_t raftEntryCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count); -cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); -char* raftEntryCache2Str(SRaftEntryCache* pObj); -void raftEntryCachePrint(SRaftEntryCache* pObj); -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); -void raftEntryCacheLog(SRaftEntryCache* pObj); -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index ff59189a9d..17ea36c640 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncRaftEntry.h" -#include "taosdef.h" #include "wal.h" typedef struct SSyncLogStoreData { @@ -40,25 +36,19 @@ typedef struct SSyncLogStoreData { SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); void logStoreDestory(SSyncLogStore* pLogStore); -cJSON* logStore2Json(SSyncLogStore* pLogStore); -char* logStore2Str(SSyncLogStore* pLogStore); -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); -char* logStoreSimple2Str(SSyncLogStore* pLogStore); SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore); -// for debug -void logStorePrint(SSyncLogStore* pLogStore); -void logStorePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreLog(SSyncLogStore* pLogStore); -void logStoreLog2(char* s, SSyncLogStore* pLogStore); - -void logStoreSimplePrint(SSyncLogStore* pLogStore); -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); -void logStoreSimpleLog(SSyncLogStore* pLogStore); -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index ac298c5375..077e6a7658 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 #define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2) diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 4f15a45cec..e08430327b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" // TLA+ Spec // AppendEntries(i, j) == diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 73b4a0efae..c51ea4eea6 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 6bef18405c..1adab30aaa 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -20,12 +20,8 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" #include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // HandleRequestVoteResponse(i, j, m) == diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index 22e1005e5c..1d41c0a85f 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -20,11 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" typedef struct SRespStub { SRpcMsg rpcMsg; @@ -41,13 +37,12 @@ typedef struct SSyncRespMgr { SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl); void syncRespMgrDestroy(SSyncRespMgr *pObj); -int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub); -int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index); -int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); -int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); +uint64_t syncRespMgrAdd(SSyncRespMgr *pObj, const SRespStub *pStub); +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t seq); +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub); +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *pInfo); void syncRespClean(SSyncRespMgr *pObj); void syncRespCleanRsp(SSyncRespMgr *pObj); -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 5594fe46ed..963fedce31 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -20,13 +20,7 @@ extern "C" { #endif -#include -#include -#include -#include "cJSON.h" #include "syncInt.h" -#include "syncMessage.h" -#include "taosdef.h" #define SYNC_SNAPSHOT_SEQ_INVALID -2 #define SYNC_SNAPSHOT_SEQ_FORCE_CLOSE -3 @@ -36,7 +30,6 @@ extern "C" { #define SYNC_SNAPSHOT_RETRY_MS 5000 -//--------------------------------------------------- typedef struct SSyncSnapshotSender { bool start; int32_t seq; @@ -65,12 +58,6 @@ int32_t snapshotSenderStop(SSyncSnapshotSender *pSender, bool finis int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); - -int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); - -//--------------------------------------------------- typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; @@ -83,7 +70,6 @@ typedef struct SSyncSnapshotReceiver { // init when create SSyncNode *pSyncNode; - } SSyncSnapshotReceiver; SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId); @@ -93,13 +79,10 @@ int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver); -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); - -//--------------------------------------------------- // on message int32_t syncNodeOnSnapshot(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotReply(SSyncNode *ths, SyncSnapshotRsp *pMsg); +int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 112a3d8610..3139707d54 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -20,12 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "syncMessage.h" -#include "taosdef.h" // TLA+ Spec // Timeout(i) == /\ state[i] \in {Follower, Candidate} diff --git a/source/libs/sync/inc/syncTools.h b/source/libs/sync/inc/syncTools.h index 932432d1f8..3fb4a5ba0c 100644 --- a/source/libs/sync/inc/syncTools.h +++ b/source/libs/sync/inc/syncTools.h @@ -26,9 +26,6 @@ typedef struct SRaftId { SyncGroupId vgId; } SRaftId; -// for compatibility, the same as syncPropose -int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak); - // ------------------ for debug ------------------- void syncRpcMsgPrint(SRpcMsg* pMsg); void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg); @@ -183,17 +180,13 @@ typedef struct SyncClientRequest { } SyncClientRequest; SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen); -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pMsg, uint64_t seqNum, bool isWeak, int32_t vgId); // step 1 -void syncClientRequestDestroy(SyncClientRequest* pMsg); -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len); -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len); -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3 -cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); -char* syncClientRequest2Str(const SyncClientRequest* pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId); +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2 +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); +char* syncClientRequest2Str(const SyncClientRequest* pMsg); // for debug ---------------------- void syncClientRequestPrint(const SyncClientRequest* pMsg); @@ -384,14 +377,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchDeserialize2(const char* buf, uint void syncAppendEntriesBatch2RpcMsg(const SyncAppendEntriesBatch* pMsg, SRpcMsg* pRpcMsg); void syncAppendEntriesBatchFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesBatch* pMsg); SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg); -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); // --------------------------------------------- typedef struct SyncAppendEntriesReply { @@ -742,14 +727,13 @@ int32_t syncNodeOnSnapshotReply(SSyncNode* ths, SyncSnapshotRsp* pMsg); int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg); int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, SyncHeartbeatReply* pMsg); -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); -int32_t syncNodeOnTimer(SSyncNode* ths, SyncTimeout* pMsg); +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg); // ----------------------------------------- typedef int32_t (*FpOnPingCb)(SSyncNode* ths, SyncPing* pMsg); typedef int32_t (*FpOnPingReplyCb)(SSyncNode* ths, SyncPingReply* pMsg); -typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex); +typedef int32_t (*FpOnClientRequestCb)(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex); typedef int32_t (*FpOnRequestVoteCb)(SSyncNode* ths, SyncRequestVote* pMsg); typedef int32_t (*FpOnRequestVoteReplyCb)(SSyncNode* ths, SyncRequestVoteReply* pMsg); typedef int32_t (*FpOnAppendEntriesCb)(SSyncNode* ths, SyncAppendEntries* pMsg); diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 7f241e827d..076101ef43 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -24,17 +24,15 @@ extern "C" { uint64_t syncUtilAddr2U64(const char* host, uint16_t port); void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port); -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet); +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet); +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId); bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2); bool syncUtilEmptyId(const SRaftId* pId); int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); -char* syncUtilRaftId2Str(const SRaftId* p); const char* syncStr(ESyncState state); char* syncUtilPrintBin(char* ptr, uint32_t len); char* syncUtilPrintBin2(char* ptr, uint32_t len); diff --git a/source/libs/sync/inc/syncVoteMgr.h b/source/libs/sync/inc/syncVoteMgr.h index d894e91600..066a4dd76f 100644 --- a/source/libs/sync/inc/syncVoteMgr.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -39,8 +39,6 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncN bool voteGrantedMajority(SVotesGranted *pVotesGranted); void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg); void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term); -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted); -char *voteGranted2Str(SVotesGranted *pVotesGranted); typedef struct SVotesRespond { SRaftId (*replicas)[TSDB_MAX_REPLICA]; @@ -56,8 +54,6 @@ void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSync bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId); void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg); void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term); -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond); -char *votesRespond2Str(SVotesRespond *pVotesRespond); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7d6e358511..792ce67cd4 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -179,7 +179,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->success = true; bool hasAppendEntries = pMsg->dataLen > 0; if (hasAppendEntries) { - SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + SSyncRaftEntry* pAppendEntry = syncEntryBuildFromAppendEntries(pMsg); ASSERT(pAppendEntry != NULL); SyncIndex appendIndex = pMsg->prevLogIndex + 1; @@ -246,7 +246,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) { } else { // error char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "ignore, get local entry error, append-index:%" PRId64, appendIndex); + snprintf(logBuf, sizeof(logBuf), "ignore, get local entry error, append-index:%" PRId64 " err:%d", appendIndex, + terrno); syncLogRecvAppendEntries(ths, pMsg, logBuf); syncEntryDestory(pLocalEntry); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 0d7306e4fd..25f6c5e6f1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -154,9 +154,7 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { code = syncNodeOnPingReply(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote* pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); @@ -225,7 +223,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { sError("sync begin snapshot error"); return -1; } - + int32_t code = 0; if (syncNodeIsMnode(pSyncNode)) { @@ -390,7 +388,7 @@ bool syncIsReadyForRead(int64_t rid) { if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { SSyncRaftEntry* pEntry = NULL; int32_t code = pSyncNode->pLogStore->syncLogGetEntry( - pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); + pSyncNode->pLogStore, pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore), &pEntry); if (code == 0 && pEntry != NULL) { if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { ready = true; @@ -452,11 +450,6 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { return ret; } -int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - int32_t ret = syncPropose(rid, pMsg, isWeak); - return ret; -} - SSyncState syncGetState(int64_t rid) { SSyncState state = {.state = TAOS_SYNC_STATE_ERROR}; @@ -558,109 +551,27 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho return lastIndex; } -#if 0 -SyncTerm syncGetMyTerm(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return TAOS_SYNC_STATE_ERROR; - } - ASSERT(rid == pSyncNode->rid); - SyncTerm term = pSyncNode->pRaftStore->currentTerm; - - syncNodeRelease(pSyncNode); - return term; -} - -SyncIndex syncGetLastIndex(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return SYNC_INDEX_INVALID; - } - ASSERT(rid == pSyncNode->rid); - SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); - - syncNodeRelease(pSyncNode); - return lastIndex; -} - -SyncIndex syncGetCommitIndex(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return SYNC_INDEX_INVALID; - } - ASSERT(rid == pSyncNode->rid); - SyncIndex cmtIndex = pSyncNode->commitIndex; - - syncNodeRelease(pSyncNode); - return cmtIndex; -} - -SyncGroupId syncGetVgId(int64_t rid) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - return TAOS_SYNC_STATE_ERROR; - } - ASSERT(rid == pSyncNode->rid); - SyncGroupId vgId = pSyncNode->vgId; - - syncNodeRelease(pSyncNode); - return vgId; -} - -void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - memset(pEpSet, 0, sizeof(*pEpSet)); - return; - } - ASSERT(rid == pSyncNode->rid); - pEpSet->numOfEps = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); - pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; - (pEpSet->numOfEps)++; - sInfo("vgId:%d, sync get epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; - sInfo("vgId:%d, sync get epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); - - syncNodeRelease(pSyncNode); -} -#endif - void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = syncNodeAcquire(rid); - if (pSyncNode == NULL) { - memset(pEpSet, 0, sizeof(*pEpSet)); - return; - } - pEpSet->numOfEps = 0; + + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) return; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); - pEpSet->eps[i].port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; - (pEpSet->numOfEps)++; - sInfo("vgId:%d, sync get retry epset: index:%d %s:%d", pSyncNode->vgId, i, pEpSet->eps[i].fqdn, - pEpSet->eps[i].port); + SEp* pEp = &pEpSet->eps[i]; + tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); + pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort; + pEpSet->numOfEps++; + sInfo("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port); } if (pEpSet->numOfEps > 0) { pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps; } - sInfo("vgId:%d, sync get retry epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); + sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse); syncNodeRelease(pSyncNode); } -static void syncGetAndDelRespRpc(SSyncNode* pSyncNode, uint64_t index, SRpcHandleInfo* pInfo) { - SRespStub stub; - int32_t ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub); - if (ret == 1) { - *pInfo = stub.rpcMsg.info; - } - - sTrace("vgId:%d, get seq:%" PRIu64 " rpc handle:%p", pSyncNode->vgId, index, pInfo->handle); -} - int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { @@ -688,44 +599,42 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { return -1; } - int32_t ret = 0; - SyncClientRequest* pSyncMsg; - // optimized one replica if (syncNodeIsOptimizedOneReplica(pSyncNode, pMsg)) { - pSyncMsg = syncClientRequestBuild(pMsg, 0, isWeak, pSyncNode->vgId); - SyncIndex retIndex; - int32_t code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, &retIndex); + int32_t code = syncNodeOnClientRequest(pSyncNode, pMsg, &retIndex); if (code == 0) { pMsg->info.conn.applyIndex = retIndex; pMsg->info.conn.applyTerm = pSyncNode->pRaftStore->currentTerm; - ret = 1; - sTrace("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); - } else { - ret = -1; - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, + sTrace("vgId:%d, propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); + return 1; + } else { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + sError("vgId:%d, failed to propose optimized msg, index:%" PRId64 " type:%s", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType)); + return -1; } } else { SRespStub stub = {.createTime = taosGetTimestampMs(), .rpcMsg = *pMsg}; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); - - pSyncMsg = syncClientRequestBuild(pMsg, seqNum, isWeak, pSyncNode->vgId); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - - sNTrace(pSyncNode, "propose message, type:%s", TMSG_INFO(pMsg->msgType)); - ret = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); - if (ret != 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, failed to enqueue msg since %s", pSyncNode->vgId, terrstr()); + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromRpcMsg(&rpcMsg, pMsg, seqNum, isWeak, pSyncNode->vgId); + if (code != 0) { + sError("vgId:%d, failed to propose msg while serialize since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + return -1; } - } - syncClientRequestDestroy(pSyncMsg); - return ret; + sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); + code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); + } + + return code; + } } static int32_t syncHbTimerInit(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer, SRaftId destId) { @@ -848,7 +757,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - if (!syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) { sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId); goto _error; } @@ -863,7 +772,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i])) { sError("vgId:%d, failed to determine raft member id, peer:%d", pSyncNode->vgId, i); goto _error; } @@ -872,7 +781,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - if (!syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { + if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; } @@ -1127,6 +1036,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); + pSyncNode->pRaftStore = NULL; syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -1171,53 +1081,6 @@ void syncNodeClose(SSyncNode* pSyncNode) { ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; } -// ping -------------- -int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { - syncPingLog2((char*)"==syncNodePing==", pMsg); - int32_t ret = 0; - - SRpcMsg rpcMsg; - syncPing2RpcMsg(pMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); - - ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); - return ret; -} - -int32_t syncNodePingSelf(SSyncNode* pSyncNode) { - int32_t ret = 0; - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - ASSERT(ret == 0); - - syncPingDestroy(pMsg); - return ret; -} - -int32_t syncNodePingPeers(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - SRaftId* destId = &(pSyncNode->peersId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - -int32_t syncNodePingAll(SSyncNode* pSyncNode) { - int32_t ret = 0; - for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - SRaftId* destId = &(pSyncNode->replicasId[i]); - SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); - ret = syncNodePing(pSyncNode, destId, pMsg); - ASSERT(ret == 0); - syncPingDestroy(pMsg); - } - return ret; -} - // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; @@ -1350,7 +1213,7 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilraftId2EpSet(destRaftId, &epSet); + syncUtilRaftId2EpSet(destRaftId, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1367,7 +1230,7 @@ int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRp int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; - syncUtilnodeInfo2EpSet(nodeInfo, &epSet); + syncUtilNodeInfo2EpSet(nodeInfo, &epSet); if (pSyncNode->syncSendMSg != NULL) { // htonl syncUtilMsgHtoN(pMsg->pCont); @@ -1380,196 +1243,6 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S return 0; } -cJSON* syncNode2Json(const SSyncNode* pSyncNode) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pSyncNode != NULL) { - // init by SSyncInfo - cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); - cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); - cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); - cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); - cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); - cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); - cJSON_AddStringToObject(pRoot, "queue", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); - cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); - - // init internal - cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); - cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); - cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); - cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); - - cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); - cJSON* pPeers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); - } - cJSON* pPeersId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "peersId", pPeersId); - for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); - } - - cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); - cJSON* pReplicasId = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); - } - - // raft algorithm - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); - cJSON_AddStringToObject(pRoot, "pFsm", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); - cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); - cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); - - // life cycle - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - - // tla+ server vars - cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); - cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); - cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); - - // tla+ candidate vars - cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); - cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); - - // tla+ leader vars - cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); - cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); - - // tla+ log vars - cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - // timer ms init - cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); - cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); - cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); - - // ping timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); - cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); - cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); - cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); - - // elect timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); - cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); - cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); - cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); - cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); - - // heartbeat timer - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); - cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); - cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); - cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); - cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); - cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); - - // callback - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); - cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); - cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); - cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); - cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); - cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); - cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); - - // restoreFinish - cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); - - // snapshot senders - cJSON* pSenders = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "senders", pSenders); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); - } - - // snapshot receivers - cJSON* pReceivers = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); - - // changing - cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); - return pJson; -} - -char* syncNode2Str(const SSyncNode* pSyncNode) { - cJSON* pJson = syncNode2Json(pSyncNode); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { - int32_t len = 256; - char* s = (char*)taosMemoryMalloc(len); - - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); - - snprintf(s, len, - "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 - ", sby:%d, " - "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d", - pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, - logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); - - return s; -} - inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { bool b1 = false; bool b2 = false; @@ -1671,7 +1344,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; - syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); + syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1; @@ -1683,13 +1356,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } } for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]); } // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum; for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { - syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); + syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } // update quorum first @@ -1716,7 +1389,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "snapshot sender reset for: %" PRIu64 ", newIndex:%d, %s:%d, %p", + sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p", (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); (pSyncNode->senders)[i] = oldSenders[j]; @@ -1785,7 +1458,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, term); char tmpBuf[64]; - snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRIu64, term); + snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRId64, term); syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); } @@ -1799,20 +1472,20 @@ void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm) { if (pSyncNode->pRaftStore->currentTerm > newTerm) { - sNTrace(pSyncNode, "step down, ignore, new-term:%" PRIu64 ", current-term:%" PRIu64, newTerm, + sNTrace(pSyncNode, "step down, ignore, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, pSyncNode->pRaftStore->currentTerm); return; } do { - sNTrace(pSyncNode, "step down, new-term:%" PRIu64 ", current-term:%" PRIu64, newTerm, + sNTrace(pSyncNode, "step down, new-term:%" PRId64 ", current-term:%" PRId64, newTerm, pSyncNode->pRaftStore->currentTerm); } while (0); if (pSyncNode->pRaftStore->currentTerm < newTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, newTerm); char tmpBuf[64]; - snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRIu64, newTerm); + snprintf(tmpBuf, sizeof(tmpBuf), "step down, update term to %" PRId64, newTerm); syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); @@ -2131,7 +1804,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { } } - sNError(pSyncNode, "sync node get pre term error, index:%" PRId64 ", snap-index:%" PRId64 ", snap-term:%" PRIu64, + sNError(pSyncNode, "sync node get pre term error, index:%" PRId64 ", snap-index:%" PRId64 ", snap-term:%" PRId64, index, snapshot.lastApplyIndex, snapshot.lastApplyTerm); return SYNC_TERM_INVALID; } @@ -2143,8 +1816,6 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex return 0; } -// ------ local funciton --------- -// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { @@ -2152,7 +1823,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { pSyncNode->pingTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue ping timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2174,7 +1845,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } } else { - sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64, + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRId64 ", pingTimerLogicClockUser:%" PRId64, pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -2196,7 +1867,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { taosMemoryFree(pElectTimer); return; } - sNTrace(pSyncNode, "eq elect timer lc:%" PRIu64, pSyncMsg->logicClock); + sNTrace(pSyncNode, "eq elect timer lc:%" PRId64, pSyncMsg->logicClock); } else { sTrace("syncNodeEqElectTimer syncEqMsg is NULL"); } @@ -2228,7 +1899,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); + sNTrace(pSyncNode, "enqueue heartbeat timer"); if (pSyncNode->syncEqMsg != NULL) { int32_t code = pSyncNode->syncEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { @@ -2249,7 +1920,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { sError("sync env is stop, syncNodeEqHeartbeatTimer"); } } else { - sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRId64 ", heartbeatTimerLogicClockUser:%" PRId64 "", pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } @@ -2261,10 +1932,18 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = pData->pSyncNode; SSyncTimer* pSyncTimer = pData->pTimer; + if (pSyncNode == NULL) { + return; + } + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return; } + if (pSyncNode->pRaftStore == NULL) { + return; + } + // sNTrace(pSyncNode, "eq peer hb timer"); int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); @@ -2311,40 +1990,34 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { } } else { - sTrace("==syncNodeEqPeerHeartbeatTimer== timerLogicClock:%" PRIu64 ", msgLogicClock:%" PRIu64 "", timerLogicClock, + sTrace("==syncNodeEqPeerHeartbeatTimer== timerLogicClock:%" PRId64 ", msgLogicClock:%" PRId64 "", timerLogicClock, msgLogicClock); } } } -static int32_t syncNodeEqNoop(SSyncNode* ths) { - int32_t ret = 0; - ASSERT(ths->state == TAOS_SYNC_STATE_LEADER); - - SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); - SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - ASSERT(pEntry != NULL); - - uint32_t entryLen; - char* serialized = syncEntrySerialize(pEntry, &entryLen); - SyncClientRequest* pSyncMsg = syncClientRequestAlloc(entryLen); - ASSERT(pSyncMsg->dataLen == entryLen); - memcpy(pSyncMsg->data, serialized, entryLen); - - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); - if (ths->syncEqMsg != NULL) { - ths->syncEqMsg(ths->msgcb, &rpcMsg); - } else { - sTrace("syncNodeEqNoop pSyncNode->syncEqMsg is NULL"); +static int32_t syncNodeEqNoop(SSyncNode* pNode) { + if (pNode->state == TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + return -1; } - syncEntryDestory(pEntry); - taosMemoryFree(serialized); - syncClientRequestDestroy(pSyncMsg); + SyncIndex index = pNode->pLogStore->syncLogWriteIndex(pNode->pLogStore); + SyncTerm term = pNode->pRaftStore->currentTerm; + SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, pNode->vgId); + if (pEntry == NULL) return -1; - return ret; + SRpcMsg rpcMsg = {0}; + int32_t code = syncClientRequestBuildFromNoopEntry(&rpcMsg, pEntry, pNode->vgId); + syncEntryDestory(pEntry); + + sNTrace(pNode, "propose msg, type:noop"); + code = (*pNode->syncEqMsg)(pNode->msgcb, &rpcMsg); + if (code != 0) { + sNError(pNode, "failed to propose noop msg while enqueue since %s", terrstr()); + } + + return code; } static void deleteCacheEntry(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } @@ -2447,7 +2120,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg) { sError("vgId:%d, sync enqueue fc-commit msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index: %" PRIu64, ths->vgId, pSyncMsg->fcIndex); + sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index:%" PRId64, ths->vgId, pSyncMsg->fcIndex); } } } @@ -2468,7 +2141,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, SyncHeartbeat* pMsg) { sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRIu64, ths->vgId, pSyncMsg->sdNewTerm); + sTrace("vgId:%d, sync enqueue step-down msg, new-term: %" PRId64, ths->vgId, pSyncMsg->sdNewTerm); } } @@ -2525,7 +2198,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, SyncLocalCmd* pMsg) { // leaderVars, commitIndex>> // -int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { +int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIndex) { sNTrace(ths, "on client request"); int32_t ret = 0; @@ -2533,8 +2206,13 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; - SSyncRaftEntry* pEntry = syncEntryBuild2(pMsg, term, index); - ASSERT(pEntry != NULL); + SSyncRaftEntry* pEntry; + + if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + pEntry = syncEntryBuildFromClientRequest(pMsg->pCont, term, index); + } else { + pEntry = syncEntryBuildFromRpcMsg(pMsg, term, index); + } LRUHandle* h = NULL; syncCacheEntry(ths->pLogStore, pEntry, &h); @@ -2549,11 +2227,30 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SyncClientRequest* pMsg, SyncInd } else { syncEntryDestory(pEntry); } + return -1; } else { // del resp mgr, call FpCommitCb - ASSERT(0); + SFsmCbMeta cbMeta = { + .index = pEntry->index, + .lastConfigIndex = SYNC_INDEX_INVALID, + .isWeak = pEntry->isWeak, + .code = -1, + .state = ths->state, + .seqNum = pEntry->seqNum, + .term = pEntry->term, + .currentTerm = ths->pRaftStore->currentTerm, + .flag = 0, + }; + ths->pFsm->FpCommitCb(ths->pFsm, pMsg, &cbMeta); + + if (h) { + taosLRUCacheRelease(ths->pLogStore->pCache, h, false); + } else { + syncEntryDestory(pEntry); + } + return -1; } } @@ -2615,7 +2312,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p } if (pEntry->term < ths->pRaftStore->currentTerm) { - sNTrace(ths, "little term:%" PRIu64 ", can not do leader transfer", pEntry->term); + sNTrace(ths, "little term:%" PRId64 ", can not do leader transfer", pEntry->term); return 0; } @@ -2645,7 +2342,7 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p int32_t ret = syncNodeRestartElectTimer(ths, electMS); ASSERT(ret == 0); - sNTrace(ths, "maybe leader transfer to %s:%d %" PRIu64, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, + sNTrace(ths, "maybe leader transfer to %s:%d %" PRId64, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); } @@ -2733,7 +2430,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde } } - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); // user commit @@ -2759,7 +2456,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde .flag = flag, }; - syncGetAndDelRespRpc(ths, cbMeta.seqNum, &rpcMsg.info); + syncRespMgrGetAndDel(ths->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, &cbMeta); } } @@ -2893,7 +2590,7 @@ const char* syncTimerTypeStr(enum ESyncTimeoutType timerType) { } void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char* s) { - sNTrace(pSyncNode, "recv sync-timer {type:%s, lc:%" PRIu64 ", ms:%d, data:%p}, %s", + sNTrace(pSyncNode, "recv sync-timer {type:%s, lc:%" PRId64 ", ms:%d, data:%p}, %s", syncTimerTypeStr(pMsg->timeoutType), pMsg->logicClock, pMsg->timerMS, pMsg->data, s); } @@ -2901,7 +2598,7 @@ void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, c char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", + sNTrace(pSyncNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } @@ -2910,7 +2607,7 @@ void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, c char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", + sNTrace(pSyncNode, "recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); } @@ -2918,7 +2615,7 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, pMsg->term, + sNTrace(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, pMsg->voteGranted, s); } @@ -2926,7 +2623,7 @@ void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, pMsg->term, + sNTrace(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term, pMsg->voteGranted, s); } @@ -2935,8 +2632,8 @@ void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d}, %s", + "send sync-append-entries to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, s); } @@ -2947,8 +2644,8 @@ void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 - ", cmt:%" PRIu64 ", pterm:%" PRIu64 ", datalen:%d}, %s", + "recv sync-append-entries from %s:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", cmt:%" PRId64 ", pterm:%" PRId64 ", datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, pMsg->dataLen, s); } @@ -2959,8 +2656,8 @@ void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + "send sync-append-entries-batch to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); } @@ -2971,8 +2668,8 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + "recv sync-append-entries-batch from %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64 + ", pterm:%" PRId64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); } @@ -2983,7 +2680,7 @@ void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "send sync-append-entries-reply to %s:%d, {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, match:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); } @@ -2994,7 +2691,7 @@ void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntries syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "recv sync-append-entries-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 ", success:%d, match:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); } @@ -3005,7 +2702,7 @@ void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-heartbeat to %s:%d {term:%" PRIu64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRIu64 + "send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); } @@ -3016,7 +2713,7 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, const syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "recv sync-heartbeat from %s:%d {term:%" PRIu64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRIu64 + "recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->privateTerm, s); } @@ -3026,7 +2723,7 @@ void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 "}, %s", host, port, + sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, s); } @@ -3034,34 +2731,34 @@ void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* p char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 "}, %s", host, port, + sNTrace(pSyncNode, "recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->privateTerm, s); } void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s) { - sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRIu64 "}, %s", pMsg->cmd, - syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, s); + sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRId64 ", fc-index:%" PRId64 "}, %s", pMsg->cmd, + syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, pMsg->fcIndex, s); } void syncLogSendSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-pre-snapshot to %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s); + sNTrace(pSyncNode, "send sync-pre-snapshot to %s:%d {term:%" PRId64 "}, %s", host, port, pMsg->term, s); } void syncLogRecvSyncPreSnapshot(SSyncNode* pSyncNode, const SyncPreSnapshot* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-pre-snapshot from %s:%d {term:%" PRIu64 "}, %s", host, port, pMsg->term, s); + sNTrace(pSyncNode, "recv sync-pre-snapshot from %s:%d {term:%" PRId64 "}, %s", host, port, pMsg->term, s); } void syncLogSendSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshotReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "send sync-pre-snapshot-reply to %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, port, + sNTrace(pSyncNode, "send sync-pre-snapshot-reply to %s:%d {term:%" PRId64 ", snap-start:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->snapStart, s); } @@ -3069,7 +2766,7 @@ void syncLogRecvSyncPreSnapshotReply(SSyncNode* pSyncNode, const SyncPreSnapshot char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - sNTrace(pSyncNode, "recv sync-pre-snapshot-reply from %s:%d {term:%" PRIu64 ", snap-start:%" PRId64 "}, %s", host, + sNTrace(pSyncNode, "recv sync-pre-snapshot-reply from %s:%d {term:%" PRId64 ", snap-start:%" PRId64 "}, %s", host, port, pMsg->term, pMsg->snapStart, s); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 6453693576..3fcb563f3b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -19,146 +19,6 @@ #include "syncUtil.h" #include "tcoding.h" -// --------------------------------------------- -cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { - cJSON* pRoot; - - // in compiler optimization, switch case = if else constants - if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncTimeout2Json(pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { - SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPing2Json(pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncPingReply2Json(pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncClientRequest2Json(pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { - pRoot = syncRpcUnknownMsg2Json(); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVote2Json(pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncRequestVoteReply2Json(pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntries2Json(pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncAppendEntriesReply2Json(pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotSend2Json(pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncSnapshotRsp2Json(pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { - SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - pRoot = syncLeaderTransfer2Json(pSyncMsg); - syncLeaderTransferDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - - } else { - pRoot = cJSON_CreateObject(); - char* s; - s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); - cJSON_AddStringToObject(pRoot, "pCont2", s); - taosMemoryFree(s); - } - - cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); - cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); - cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); - // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); - return pJson; -} - -cJSON* syncRpcUnknownMsg2Json() { - cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); - cJSON_AddStringToObject(pRoot, "data", "unknown message"); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); - return pJson; -} - -char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { - cJSON* pJson = syncRpcMsg2Json(pRpcMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncRpcMsgPrint(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog(SRpcMsg* pMsg) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); @@ -840,69 +700,49 @@ SyncClientRequest* syncClientRequestAlloc(uint32_t dataLen) { return pMsg; } -// step 1. original SRpcMsg => SyncClientRequest, add seqNum, isWeak -SyncClientRequest* syncClientRequestBuild(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak, int32_t vgId) { - SyncClientRequest* pMsg = syncClientRequestAlloc(pOriginalRpcMsg->contLen); - pMsg->vgId = vgId; - pMsg->originalRpcType = pOriginalRpcMsg->msgType; - pMsg->seqNum = seqNum; - pMsg->isWeak = isWeak; - memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); - return pMsg; -} - -void syncClientRequestDestroy(SyncClientRequest* pMsg) { - if (pMsg != NULL) { - taosMemoryFree(pMsg); +int32_t syncClientRequestBuildFromRpcMsg(SRpcMsg* pClientRequestRpcMsg, const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, + bool isWeak, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pOriginalRpcMsg->contLen; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = pOriginalRpcMsg->msgType; + pClientRequest->seqNum = seqNum; + pClientRequest->isWeak = isWeak; + pClientRequest->dataLen = pOriginalRpcMsg->contLen; + memcpy(pClientRequest->data, (char*)pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); + + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } -void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) { - ASSERT(pMsg->bytes <= bufLen); - memcpy(buf, pMsg, pMsg->bytes); -} - -void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) { - memcpy(pMsg, buf, len); - ASSERT(len == pMsg->bytes); -} - -char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { - char* buf = taosMemoryMalloc(pMsg->bytes); - ASSERT(buf != NULL); - syncClientRequestSerialize(pMsg, buf, pMsg->bytes); - if (len != NULL) { - *len = pMsg->bytes; +int32_t syncClientRequestBuildFromNoopEntry(SRpcMsg* pClientRequestRpcMsg, const SSyncRaftEntry* pEntry, int32_t vgId) { + int32_t bytes = sizeof(SyncClientRequest) + pEntry->bytes; + pClientRequestRpcMsg->pCont = rpcMallocCont(bytes); + if (pClientRequestRpcMsg->pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - return buf; -} -SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SyncClientRequest* pMsg = taosMemoryMalloc(bytes); - ASSERT(pMsg != NULL); - syncClientRequestDeserialize(buf, len, pMsg); - ASSERT(len == pMsg->bytes); - return pMsg; -} + SyncClientRequest* pClientRequest = pClientRequestRpcMsg->pCont; + pClientRequest->bytes = bytes; + pClientRequest->vgId = vgId; + pClientRequest->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequest->originalRpcType = TDMT_SYNC_NOOP; + pClientRequest->dataLen = pEntry->bytes; + memcpy(pClientRequest->data, (char*)pEntry, pEntry->bytes); -// step 2. SyncClientRequest => RpcMsg, to queue -void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { - pRpcMsg->msgType = pMsg->msgType; - pRpcMsg->contLen = pMsg->bytes; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - syncClientRequestSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); -} - -void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg) { - syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); -} - -// step 3. RpcMsg => SyncClientRequest, from queue -SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { - SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - ASSERT(pMsg != NULL); - return pMsg; + pClientRequestRpcMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; + pClientRequestRpcMsg->contLen = bytes; + return 0; } cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { @@ -940,35 +780,6 @@ char* syncClientRequest2Str(const SyncClientRequest* pMsg) { return serialized; } -// for debug ---------------------- -void syncClientRequestPrint(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestLog(const SyncClientRequest* pMsg) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncClientRequestBatch---- // block1: @@ -1059,86 +870,6 @@ SyncClientRequestBatch* syncClientRequestBatchFromRpcMsg(const SRpcMsg* pRpcMsg) return pSyncMsg; } -cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - - SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); - SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); - cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pMsgArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); - cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); - return pJson; -} - -char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { - cJSON* pJson = syncClientRequestBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncClientRequestBatch2Str(pMsg); - sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncRequestVote); @@ -1716,138 +1447,6 @@ SyncAppendEntriesBatch* syncAppendEntriesBatchFromRpcMsg2(const SRpcMsg* pRpcMsg return pMsg; } -cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); - cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); - cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); - cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - - int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // - int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; - - cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); - cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); - - SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); - - cJSON* pMetaArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - cJSON* pMeta = cJSON_CreateObject(); - cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); - cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); - cJSON_AddItemToArray(pMetaArr, pMeta); - } - - cJSON* pEntryArr = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); - for (int i = 0; i < pMsg->dataCount; ++i) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); - cJSON* pEntryJson = syncEntry2Json(pEntry); - cJSON_AddItemToArray(pEntryArr, pEntryJson); - } - - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); - return pJson; -} - -char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { - cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncAppendEntriesBatch2Str(pMsg); - sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - // ---- message process SyncAppendEntriesReply---- SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncAppendEntriesReply); @@ -2827,118 +2426,6 @@ SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { return pMsg; } -cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { - char u64buf[128]; - cJSON* pRoot = cJSON_CreateObject(); - - if (pMsg != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); - cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); - cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); - - cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); - cJSON_AddStringToObject(pSrcId, "addr", u64buf); - { - uint64_t u64 = pMsg->srcId.addr; - cJSON* pTmp = pSrcId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); - cJSON_AddItemToObject(pRoot, "srcId", pSrcId); - - cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); - cJSON_AddStringToObject(pDestId, "addr", u64buf); - { - uint64_t u64 = pMsg->destId.addr; - cJSON* pTmp = pDestId; - char host[128]; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); - cJSON_AddItemToObject(pRoot, "destId", pDestId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); - cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); - cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); - cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); - cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); - - cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); - char* s; - s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); - return pJson; -} - -char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { - cJSON* pJson = syncSnapshotSend2Json(pMsg); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// for debug ---------------------- -void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { - if (gRaftDetailLog) { - char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// --------------------------------------------- SyncSnapshotRsp* syncSnapshotRspBuild(int32_t vgId) { uint32_t bytes = sizeof(SyncSnapshotRsp); SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 520ecd95db..262a7b0879 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,31 +13,29 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncRaftEntry.h" #include "syncUtil.h" +#include "tref.h" -SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { - uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; +SSyncRaftEntry* syncEntryBuild(int32_t dataLen) { + int32_t bytes = sizeof(SSyncRaftEntry) + dataLen; SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memset(pEntry, 0, bytes); + if (pEntry == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pEntry->bytes = bytes; pEntry->dataLen = dataLen; pEntry->rid = -1; - return pEntry; -} - -// step 4. SyncClientRequest => SSyncRaftEntry, add term, index -SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index); - ASSERT(pEntry != NULL); return pEntry; } -SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { +SSyncRaftEntry* syncEntryBuildFromClientRequest(const SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); - ASSERT(pEntry != NULL); + if (pEntry == NULL) return NULL; pEntry->msgType = pMsg->msgType; pEntry->originalRpcType = pMsg->originalRpcType; @@ -45,42 +43,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde pEntry->isWeak = pMsg->isWeak; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pMsg->dataLen; memcpy(pEntry->data, pMsg->data, pMsg->dataLen); return pEntry; } -SSyncRaftEntry* syncEntryBuild4(SRpcMsg* pOriginalMsg, SyncTerm term, SyncIndex index) { - SSyncRaftEntry* pEntry = syncEntryBuild(pOriginalMsg->contLen); - ASSERT(pEntry != NULL); +SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->contLen); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; - pEntry->originalRpcType = pOriginalMsg->msgType; + pEntry->originalRpcType = pMsg->msgType; pEntry->seqNum = 0; pEntry->isWeak = 0; pEntry->term = term; pEntry->index = index; - pEntry->dataLen = pOriginalMsg->contLen; - memcpy(pEntry->data, pOriginalMsg->pCont, pOriginalMsg->contLen); + memcpy(pEntry->data, pMsg->pCont, pMsg->contLen); return pEntry; } -SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { - // init rpcMsg - SMsgHead head; - head.vgId = vgId; - head.contLen = sizeof(SMsgHead); - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(SRpcMsg)); - rpcMsg.contLen = head.contLen; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - rpcMsg.msgType = TDMT_SYNC_NOOP; - memcpy(rpcMsg.pCont, &head, sizeof(head)); +SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + if (pEntry == NULL) return NULL; - SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); - ASSERT(pEntry != NULL); + memcpy(pEntry, pMsg->data, pMsg->dataLen); + return pEntry; +} + +SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) { + SSyncRaftEntry* pEntry = syncEntryBuild(sizeof(SMsgHead)); + if (pEntry == NULL) return NULL; pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = TDMT_SYNC_NOOP; @@ -89,9 +82,9 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) pEntry->term = term; pEntry->index = index; - ASSERT(pEntry->dataLen == rpcMsg.contLen); - memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen); - rpcFreeCont(rpcMsg.pCont); + SMsgHead* pHead = (SMsgHead*)pEntry->data; + pHead->vgId = vgId; + pHead->contLen = sizeof(SMsgHead); return pEntry; } @@ -102,104 +95,13 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { } } -// step 5. SSyncRaftEntry => bin, to raft log -char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { - char* buf = taosMemoryMalloc(pEntry->bytes); - ASSERT(buf != NULL); - memcpy(buf, pEntry, pEntry->bytes); - if (len != NULL) { - *len = pEntry->bytes; - } - return buf; -} - -// step 6. bin => SSyncRaftEntry, from raft log -SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { - uint32_t bytes = *((uint32_t*)buf); - SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - ASSERT(pEntry != NULL); - memcpy(pEntry, buf, len); - ASSERT(len == pEntry->bytes); - return pEntry; -} - -cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pEntry != NULL) { - cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); - cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); - cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); - cJSON_AddStringToObject(pRoot, "seqNum", u64buf); - cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); - cJSON_AddStringToObject(pRoot, "index", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); - cJSON_AddStringToObject(pRoot, "rid", u64buf); - cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); - - char* s; - s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data", s); - taosMemoryFree(s); - - s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); - cJSON_AddStringToObject(pRoot, "data2", s); - taosMemoryFree(s); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); - return pJson; -} - -char* syncEntry2Str(const SSyncRaftEntry* pEntry) { - cJSON* pJson = syncEntry2Json(pEntry); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -// step 7. SSyncRaftEntry => original SRpcMsg, commit to user, delete seqNum, isWeak, term, index void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { - memset(pRpcMsg, 0, sizeof(*pRpcMsg)); pRpcMsg->msgType = pEntry->originalRpcType; pRpcMsg->contLen = pEntry->dataLen; pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } -// for debug ---------------------- -void syncEntryPrint(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void syncEntryLog(const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { - char* serialized = syncEntry2Str(pObj); - sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); - taosMemoryFree(serialized); -} - //----------------------------------- SRaftEntryHashCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { SRaftEntryHashCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryHashCache)); @@ -354,76 +256,6 @@ int32_t raftCacheClear(struct SRaftEntryHashCache* pCache) { return 0; } -//----------------------------------- -cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - while (pIter) { - pIter = taosHashIterate(pCache->pEntryHash, pIter); - if (pIter != NULL) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - } - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); - return pJson; -} - -char* raftCache2Str(SRaftEntryHashCache* pCache) { - cJSON* pJson = raftCache2Json(pCache); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftCachePrint(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftCacheLog(SRaftEntryHashCache* pCache) { - char* serialized = raftCache2Str(pCache); - sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { - if (gRaftDetailLog) { - char* serialized = raftCache2Str(pCache); - sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} //----------------------------------- static char* keyFn(const void* pData) { @@ -612,69 +444,3 @@ int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) { taosThreadMutexUnlock(&pCache->mutex); return returnCnt; } - -cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - - snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); - cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "entries", pEntries); - - SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); - while (tSkipListIterNext(pIter)) { - SSkipListNode* pNode = tSkipListIterGet(pIter); - ASSERT(pNode != NULL); - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - } - tSkipListDestroyIter(pIter); - - taosThreadMutexUnlock(&pCache->mutex); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); - return pJson; -} - -char* raftEntryCache2Str(SRaftEntryCache* pObj) { - cJSON* pJson = raftEntryCache2Json(pObj); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -void raftEntryCachePrint(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog(SRaftEntryCache* pObj) { - char* serialized = raftEntryCache2Str(pObj); - sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { - if (gRaftDetailLog) { - char* serialized = raftEntryCache2Str(pObj); - sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index d3d69b288b..d4a1d35c74 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -22,15 +22,8 @@ // public function static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex); -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); + static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry); static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index); static int32_t raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); @@ -126,29 +119,29 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI return 0; } -static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex firstVer = walGetFirstVer(pWal); return firstVer; } -static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } +SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } -static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { +bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; return walIsEmpty(pWal); } -static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { +int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); int32_t count = endIndex - beginIndex + 1; return count > 0 ? count : 0; } -static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SyncIndex lastIndex; SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -157,7 +150,7 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { return lastVer; } -static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { +SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); @@ -174,7 +167,7 @@ static bool raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index) { // if success, return last term // if not log, return 0 // if error, return SYNC_TERM_INVALID -static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { +SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (walIsEmpty(pWal)) { @@ -225,7 +218,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr // entry found, return 0 // entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST // other error, return -1 -static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { +int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; int32_t code = 0; @@ -301,6 +294,12 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return 0; } + // need not truncate + SyncIndex walCommitVer = walGetCommittedVer(pWal); + if (fromIndex <= walCommitVer) { + return 0; + } + int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; @@ -364,111 +363,6 @@ SyncIndex raftlogCommitIndex(SSyncLogStore* pLogStore) { return pData->pSyncNode->commitIndex; } -cJSON* logStore2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - - cJSON* pEntries = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "pEntries", pEntries); - - if (!raftLogIsEmpty(pLogStore)) { - for (SyncIndex i = beginIndex; i <= endIndex; ++i) { - SSyncRaftEntry* pEntry = NULL; - raftLogGetEntry(pLogStore, i, &pEntry); - - cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); - syncEntryDestory(pEntry); - } - } - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); - return pJson; -} - -char* logStore2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStore2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - -cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { - char u64buf[128] = {0}; - SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; - cJSON* pRoot = cJSON_CreateObject(); - - if (pData != NULL && pData->pWal != NULL) { - snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); - cJSON_AddStringToObject(pRoot, "pWal", u64buf); - - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); - cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - - SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); - cJSON_AddStringToObject(pRoot, "endIndex", u64buf); - - int32_t count = raftLogEntryCount(pLogStore); - cJSON_AddNumberToObject(pRoot, "entryCount", count); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); - cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); - cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); - } - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); - return pJson; -} - -char* logStoreSimple2Str(SSyncLogStore* pLogStore) { - cJSON* pJson = logStoreSimple2Json(pLogStore); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -480,63 +374,3 @@ SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore) { SWal* pWal = pData->pWal; return walGetCommittedVer(pWal); } - -// for debug ----------------- -void logStorePrint(SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStorePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStore2Str(pLogStore); - printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreLog(SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); - } -} - -void logStoreLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStore2Str(pLogStore); - sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} - -// for debug ----------------- -void logStoreSimplePrint(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); - fflush(NULL); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog(SSyncLogStore* pLogStore) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); - taosMemoryFree(serialized); -} - -void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { - if (gRaftDetailLog) { - char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); - taosMemoryFree(serialized); - } -} diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 7b4ef63406..3f22a1f2cd 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -78,14 +78,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); - - // add pEntry into msg - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - ASSERT(len == pEntry->bytes); - memcpy(pMsg->data, serialized, len); - - taosMemoryFree(serialized); + memcpy(pMsg->data, pEntry, pEntry->bytes); syncEntryDestory(pEntry); } else { diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 56d5e944b8..de8f1927ae 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -13,21 +13,22 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncRespMgr.h" #include "syncRaftEntry.h" #include "syncRaftStore.h" SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { - SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); + SSyncRespMgr *pObj = taosMemoryCalloc(1, sizeof(SSyncRespMgr)); if (pObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pObj, 0, sizeof(SSyncRespMgr)); pObj->pRespHash = taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - ASSERT(pObj->pRespHash != NULL); + if (pObj->pRespHash == NULL) return NULL; + pObj->ttl = ttl; pObj->data = data; pObj->seqNum = 0; @@ -38,93 +39,84 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { void syncRespMgrDestroy(SSyncRespMgr *pObj) { if (pObj != NULL) { - taosThreadMutexLock(&(pObj->mutex)); + taosThreadMutexLock(&pObj->mutex); taosHashCleanup(pObj->pRespHash); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexDestroy(&(pObj->mutex)); taosMemoryFree(pObj); } } -int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +uint64_t syncRespMgrAdd(SSyncRespMgr *pObj, const SRespStub *pStub) { + taosThreadMutexLock(&pObj->mutex); - uint64_t keyCode = ++(pObj->seqNum); - taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); + uint64_t seq = ++(pObj->seqNum); + int32_t code = taosHashPut(pObj->pRespHash, &seq, sizeof(uint64_t), pStub, sizeof(SRespStub)); + sNTrace(pObj->data, "save message handle:%p, type:%s seq:%" PRIu64 " code:0x%x", pStub->rpcMsg.info.handle, + TMSG_INFO(pStub->rpcMsg.msgType), seq, code); - sNTrace(pObj->data, "save message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), - keyCode, pStub->rpcMsg.info.handle); - taosThreadMutexUnlock(&(pObj->mutex)); - return keyCode; + taosThreadMutexUnlock(&pObj->mutex); + return seq; } -int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t seq) { + taosThreadMutexLock(&pObj->mutex); - taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + int32_t code = taosHashRemove(pObj->pRespHash, &seq, sizeof(seq)); + sNTrace(pObj->data, "remove message handle, seq:%" PRIu64 " code:%d", seq, code); - taosThreadMutexUnlock(&(pObj->mutex)); - return 0; + taosThreadMutexUnlock(&pObj->mutex); + return code; } -int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub) { + taosThreadMutexLock(&pObj->mutex); - void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); + SRespStub *pTmp = taosHashGet(pObj->pRespHash, &seq, sizeof(uint64_t)); if (pTmp != NULL) { memcpy(pStub, pTmp, sizeof(SRespStub)); + sNTrace(pObj->data, "get message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), seq, + pStub->rpcMsg.info.handle); - sNTrace(pObj->data, "get message handle, type:%s seq:%" PRIu64 " handle:%p", TMSG_INFO(pStub->rpcMsg.msgType), - index, pStub->rpcMsg.info.handle); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object } - taosThreadMutexUnlock(&(pObj->mutex)); + + taosThreadMutexUnlock(&pObj->mutex); return 0; // get none object } -int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { - taosThreadMutexLock(&(pObj->mutex)); +int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *pInfo) { + taosThreadMutexLock(&pObj->mutex); - void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); - if (pTmp != NULL) { - memcpy(pStub, pTmp, sizeof(SRespStub)); + SRespStub *pStub = taosHashGet(pObj->pRespHash, &seq, sizeof(uint64_t)); + if (pStub != NULL) { + *pInfo = pStub->rpcMsg.info; + sNTrace(pObj->data, "get-and-del message handle:%p, type:%s seq:%" PRIu64, pStub->rpcMsg.info.handle, + TMSG_INFO(pStub->rpcMsg.msgType), seq); + taosHashRemove(pObj->pRespHash, &seq, sizeof(uint64_t)); - sNTrace(pObj->data, "get-and-del message handle, type:%s seq:%" PRIu64 " handle:%p", - TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle); - taosHashRemove(pObj->pRespHash, &index, sizeof(index)); - taosThreadMutexUnlock(&(pObj->mutex)); + taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object } - taosThreadMutexUnlock(&(pObj->mutex)); + + taosThreadMutexUnlock(&pObj->mutex); return 0; // get none object } -void syncRespCleanRsp(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, -1, true); - taosThreadMutexUnlock(&(pObj->mutex)); -} - -void syncRespClean(SSyncRespMgr *pObj) { - taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, pObj->ttl, false); - taosThreadMutexUnlock(&(pObj->mutex)); -} - -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { +static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; int sum = 0; SSyncNode *pSyncNode = pObj->data; - SArray *delIndexArray = taosArrayInit(0, sizeof(uint64_t)); - ASSERT(delIndexArray != NULL); - sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); + SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t)); + if (delIndexArray == NULL) return; + sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); while (pStub) { size_t len; - void * key = taosHashGetKey(pStub, &len); + void *key = taosHashGetKey(pStub, &len); uint64_t *pSeqNum = (uint64_t *)key; sum++; @@ -149,15 +141,15 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { pStub->rpcMsg.contLen = 0; // TODO: and make rpcMsg body, call commit cb - // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta); pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; if (pStub->rpcMsg.info.handle != NULL) { - tmsgSendRsp(&(pStub->rpcMsg)); + tmsgSendRsp(&pStub->rpcMsg); } } - pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub); + pStub = taosHashIterate(pObj->pRespHash, pStub); } int32_t arraySize = taosArrayGetSize(delIndexArray); @@ -170,3 +162,15 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { } taosArrayDestroy(delIndexArray); } + +void syncRespCleanRsp(SSyncRespMgr *pObj) { + taosThreadMutexLock(&pObj->mutex); + syncRespCleanByTTL(pObj, -1, true); + taosThreadMutexUnlock(&pObj->mutex); +} + +void syncRespClean(SSyncRespMgr *pObj) { + taosThreadMutexLock(&pObj->mutex); + syncRespCleanByTTL(pObj, pObj->ttl, false); + taosThreadMutexUnlock(&pObj->mutex); +} diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 78413bbeff..3273718f7d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -13,33 +13,25 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncSnapshot.h" #include "syncIndexMgr.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncUtil.h" -#include "wal.h" -//---------------------------------- -static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg); -static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg); -static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); -static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); - -//---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { bool condition = (pSyncNode->pFsm->FpSnapshotStartRead != NULL) && (pSyncNode->pFsm->FpSnapshotStopRead != NULL) && (pSyncNode->pFsm->FpSnapshotDoRead != NULL); SSyncSnapshotSender *pSender = NULL; if (condition) { - pSender = taosMemoryMalloc(sizeof(SSyncSnapshotSender)); + pSender = taosMemoryCalloc(1, sizeof(SSyncSnapshotSender)); if (pSender == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pSender, 0, sizeof(*pSender)); pSender->start = false; pSender->seq = SYNC_SNAPSHOT_SEQ_INVALID; @@ -249,64 +241,6 @@ static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnaps ++(pSender->seq); } -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pSender != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pSender->start); - cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); - cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); - cJSON_AddStringToObject(pRoot, "pReader", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); - cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); - - if (pSender->pCurrentBlock != NULL) { - char *s; - s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); - taosMemoryFree(s); - s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); - cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); - taosMemoryFree(s); - } - - cJSON *pSnapshot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); - cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); - cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); - // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - - cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); - return pJson; -} - -char *snapshotSender2Str(SSyncSnapshotSender *pSender) { - cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { sNTrace(pSyncNode, "starting snapshot ..."); @@ -335,16 +269,17 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) { return 0; } -// ------------------------------------- SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) { bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) && (pSyncNode->pFsm->FpSnapshotDoWrite != NULL); SSyncSnapshotReceiver *pReceiver = NULL; if (condition) { - pReceiver = taosMemoryMalloc(sizeof(SSyncSnapshotReceiver)); - ASSERT(pReceiver != NULL); - memset(pReceiver, 0, sizeof(*pReceiver)); + pReceiver = taosMemoryCalloc(1, sizeof(SSyncSnapshotReceiver)); + if (pReceiver == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pReceiver->start = false; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; @@ -530,63 +465,6 @@ static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapsh } } -cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { - char u64buf[128]; - cJSON *pRoot = cJSON_CreateObject(); - - if (pReceiver != NULL) { - cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); - cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); - cJSON_AddStringToObject(pRoot, "pWriter", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - cJSON *pFromId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); - cJSON_AddStringToObject(pFromId, "addr", u64buf); - { - uint64_t u64 = pReceiver->fromId.addr; - cJSON *pTmp = pFromId; - char host[128] = {0}; - uint16_t port; - syncUtilU642Addr(u64, host, sizeof(host), &port); - cJSON_AddStringToObject(pTmp, "addr_host", host); - cJSON_AddNumberToObject(pTmp, "addr_port", port); - } - cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); - cJSON_AddItemToObject(pRoot, "fromId", pFromId); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); - cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); - cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - - snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); - cJSON_AddStringToObject(pRoot, "startTime", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); - return pJson; -} - -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { - cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) { SyncIndex snapStart = SYNC_INDEX_INVALID; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 30e0617f43..91d807319b 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -13,28 +13,23 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncTimeout.h" #include "syncElection.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncReplication.h" -#include "syncRespMgr.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { int32_t newArrIndex = 0; - SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT]; - memset(newConfigIndexArr, 0, sizeof(newConfigIndexArr)); - + SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0}; SSnapshot snapshot = {0}; - if (ths->pFsm != NULL && ths->pFsm->FpGetSnapshotInfo != NULL) { - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - } - + + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) { - for (int i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { + for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) { if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) { // pass - ; } else { // save newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i]; @@ -47,13 +42,15 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr)); int32_t code = raftCfgPersist(ths->pRaftCfg); - ASSERT(code == 0); - - sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + if (code != 0) { + sNFatal(ths, "failed to persist cfg"); + } else { + sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount); + } } } -int32_t syncNodeTimerRoutine(SSyncNode* ths) { +static int32_t syncNodeTimerRoutine(SSyncNode* ths) { sNTrace(ths, "timer routines"); // timer replicate @@ -71,7 +68,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); + sNError(ths, "timer wal snapshot end error since:%s", terrstr()); return -1; } else { sNTrace(ths, "wal snapshot end, index:%" PRId64, atomic_load_64(&ths->snapshottingIndex)); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index e82578ef4d..894aa05aad 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -41,15 +41,15 @@ void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) { *port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16); } -void syncUtilnodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { +void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) { pEpSet->inUse = 0; pEpSet->numOfEps = 0; addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort); } -void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { +void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { char host[TSDB_FQDN_LEN] = {0}; - uint16_t port; + uint16_t port = 0; syncUtilU642Addr(raftId->addr, host, sizeof(host), &port); pEpSet->inUse = 0; @@ -57,7 +57,7 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { addEpIntoEpSet(pEpSet, host, port); } -bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { +bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ipv4 == 0xFFFFFFFF || ipv4 == 1) { sError("failed to resolve ipv4 addr, fqdn: %s", pInfo->nodeFqdn); @@ -73,8 +73,7 @@ bool syncUtilnodeInfo2raftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* } bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) { - bool ret = pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; - return ret; + return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId; } bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); } @@ -90,18 +89,6 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } -cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { - char u64buf[128] = {0}; - cJSON* pRoot = cJSON_CreateObject(); - - cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); - cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); - - cJSON* pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); - return pJson; -} - cJSON* syncUtilRaftId2Json(const SRaftId* p) { char u64buf[128] = {0}; cJSON* pRoot = cJSON_CreateObject(); @@ -120,13 +107,6 @@ cJSON* syncUtilRaftId2Json(const SRaftId* p) { return pJson; } -char* syncUtilRaftId2Str(const SRaftId* p) { - cJSON* pJson = syncUtilRaftId2Json(p); - char* serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - static inline bool syncUtilCanPrint(char c) { if (c >= 32 && c <= 126) { return true; @@ -165,14 +145,12 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len) { } void syncUtilMsgHtoN(void* msg) { - // htonl SMsgHead* pHead = msg; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); } void syncUtilMsgNtoH(void* msg) { - // ntohl SMsgHead* pHead = msg; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); @@ -205,7 +183,7 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { if (i < pSyncNode->replicaNum - 1) { len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 ", ", i, pState->lastSendIndex, - pState->lastSendTime); + pState->lastSendTime); } else { len += snprintf(buf + len, bufLen - len, "%d:%" PRId64 " %" PRId64 "}", i, pState->lastSendIndex, pState->lastSendTime); @@ -216,6 +194,9 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) { if (pNode == NULL || pNode->pRaftCfg != NULL && pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return; + // save error code, otherwise it will be overwritten + int32_t errCode = terrno; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) { pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot); @@ -242,16 +223,21 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); + int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + + // restore error code + terrno = errCode; + taosPrintLog(flags, level, dflag, "vgId:%d, sync %s " "%s" ", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 - ", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64 + ", snap-tm:%" PRIu64 ", sby:%d, aq:%d, bch:%d, r-num:%d, lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s", pNode->vgId, syncStr(pNode->state), eventLog, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - pNode->pRaftCfg->isStandBy, pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize, - pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, + pNode->pRaftCfg->isStandBy, aqItems, pNode->pRaftCfg->batchSize, pNode->replicaNum, + pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum, pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr); } diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index ee1f83ee6a..4ca4e26bec 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -23,12 +23,11 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { } SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { - SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); + SVotesGranted *pVotesGranted = taosMemoryCalloc(1, sizeof(SVotesGranted)); if (pVotesGranted == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); pVotesGranted->replicaNum = pSyncNode->replicaNum; @@ -59,20 +58,24 @@ void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode) { pVotesGranted->pSyncNode = pSyncNode; } -bool voteGrantedMajority(SVotesGranted *pVotesGranted) { - bool ret = pVotesGranted->votes >= pVotesGranted->quorum; - return ret; -} +bool voteGrantedMajority(SVotesGranted *pVotesGranted) { return pVotesGranted->votes >= pVotesGranted->quorum; } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { - ASSERT(pMsg->voteGranted == true); - - if (pMsg->term != pVotesGranted->term) { - sNTrace(pVotesGranted->pSyncNode, "vote grant vnode error"); + if (!pMsg->voteGranted) { + sNFatal(pVotesGranted->pSyncNode, "vote granted should be true"); return; } - ASSERT(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + if (pMsg->term != pVotesGranted->term) { + sNTrace(pVotesGranted->pSyncNode, "vote grant term:%" PRId64 " not matched with msg term:%" PRId64, + pVotesGranted->term, pMsg->term); + return; + } + + if (!syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)) { + sNFatal(pVotesGranted->pSyncNode, "vote granted raftId not matched with msg"); + return; + } int32_t j = -1; for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -81,14 +84,21 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - ASSERT(j != -1); - ASSERT(j >= 0 && j < pVotesGranted->replicaNum); + if ((j == -1) || !(j >= 0 && j < pVotesGranted->replicaNum)) { + sNFatal(pVotesGranted->pSyncNode, "invalid msg srcId, index:%d", j); + return; + } if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); pVotesGranted->isGranted[j] = true; } - ASSERT(pVotesGranted->votes <= pVotesGranted->replicaNum); + + if (pVotesGranted->votes > pVotesGranted->replicaNum) { + sNFatal(pVotesGranted->pSyncNode, "votes:%d not matched with replicaNum:%d", pVotesGranted->votes, + pVotesGranted->replicaNum); + return; + } } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { @@ -97,53 +107,12 @@ void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { pVotesGranted->toLeader = false; } -cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesGranted != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); - } - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); - for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { - arr[i] = pVotesGranted->isGranted[i]; - } - cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); - - cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); - cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - - bool majority = voteGrantedMajority(pVotesGranted); - cJSON_AddNumberToObject(pRoot, "majority", majority); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); - return pJson; -} - -char *voteGranted2Str(SVotesGranted *pVotesGranted) { - cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; -} - SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { - SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); - ASSERT(pVotesRespond != NULL); - memset(pVotesRespond, 0, sizeof(SVotesRespond)); + SVotesRespond *pVotesRespond = taosMemoryCalloc(1, sizeof(SVotesRespond)); + if (pVotesRespond == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } pVotesRespond->replicas = &(pSyncNode->replicasId); pVotesRespond->replicaNum = pSyncNode->replicaNum; @@ -185,62 +154,15 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { - // ASSERT(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } } - ASSERT(0); + + sNFatal(pVotesRespond->pSyncNode, "votes respond not found"); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { pVotesRespond->term = term; memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond)); - /* - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - pVotesRespond->isRespond[i] = false; - } - */ -} - -cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { - char u64buf[128] = {0}; - cJSON *pRoot = cJSON_CreateObject(); - - if (pVotesRespond != NULL) { - cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); - cJSON *pReplicas = cJSON_CreateArray(); - cJSON_AddItemToObject(pRoot, "replicas", pReplicas); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); - } - int32_t respondNum = 0; - int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); - for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { - arr[i] = pVotesRespond->isRespond[i]; - if (pVotesRespond->isRespond[i]) { - respondNum++; - } - } - cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); - taosMemoryFree(arr); - cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); - cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); - cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); - cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); - } - - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); - return pJson; -} - -char *votesRespond2Str(SVotesRespond *pVotesRespond) { - cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); - cJSON_Delete(pJson); - return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index e584893ae0..7a22d96972 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(sync_test_lib) add_executable(syncTest "") add_executable(syncRaftIdCheck "") add_executable(syncEnvTest "") @@ -24,8 +25,6 @@ add_executable(syncRequestVoteReplyTest "") add_executable(syncAppendEntriesTest "") add_executable(syncAppendEntriesBatchTest "") add_executable(syncAppendEntriesReplyTest "") -add_executable(syncClientRequestTest "") -add_executable(syncClientRequestBatchTest "") add_executable(syncTimeoutTest "") add_executable(syncPingTest "") add_executable(syncPingReplyTest "") @@ -167,14 +166,6 @@ target_sources(syncAppendEntriesReplyTest PRIVATE "syncAppendEntriesReplyTest.cpp" ) -target_sources(syncClientRequestTest - PRIVATE - "syncClientRequestTest.cpp" -) -target_sources(syncClientRequestBatchTest - PRIVATE - "syncClientRequestBatchTest.cpp" -) target_sources(syncTimeoutTest PRIVATE "syncTimeoutTest.cpp" @@ -287,10 +278,6 @@ target_sources(syncLeaderTransferTest PRIVATE "syncLeaderTransferTest.cpp" ) -target_sources(syncReconfigFinishTest - PRIVATE - "syncReconfigFinishTest.cpp" -) target_sources(syncRestoreFromSnapshot PRIVATE "syncRestoreFromSnapshot.cpp" @@ -451,16 +438,6 @@ target_include_directories(syncAppendEntriesReplyTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncClientRequestTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_include_directories(syncClientRequestBatchTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncTimeoutTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -601,11 +578,6 @@ target_include_directories(syncLeaderTransferTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncReconfigFinishTest - PUBLIC - "${TD_SOURCE_DIR}/include/libs/sync" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) target_include_directories(syncRestoreFromSnapshot PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -644,259 +616,247 @@ target_include_directories(syncPreSnapshotReplyTest target_link_libraries(syncTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftIdCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnvTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickQTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOTickPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOSendMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOClientTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIOServerTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEnqTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncInitTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncUtilTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesGrantedTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncVotesRespondTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncIndexMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEntryCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHashCacheTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRequestVoteReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncAppendEntriesReplyTest - sync - gtest_main -) -target_link_libraries(syncClientRequestTest - sync - gtest_main -) -target_link_libraries(syncClientRequestBatchTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTimeoutTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRpcMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingTimerTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncPingSelfTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncElectTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncEncodeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncWriteTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncReplicateTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRefTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck - sync + sync_test_lib gtest_main ) target_link_libraries(syncLogStoreCheck2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRespMgrTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncApplyMsgTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncConfigChangeSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSendTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotRspTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotSenderTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncSnapshotReceiverTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncTestTool - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest2 - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftLogTest3 - sync + sync_test_lib gtest_main ) target_link_libraries(syncLeaderTransferTest - sync - gtest_main -) -target_link_libraries(syncReconfigFinishTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncRestoreFromSnapshot - sync + sync_test_lib gtest_main ) target_link_libraries(syncRaftCfgIndexTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncHeartbeatReplyTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncLocalCmdTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotTest - sync + sync_test_lib gtest_main ) target_link_libraries(syncPreSnapshotReplyTest - sync + sync_test_lib gtest_main ) diff --git a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp index f2544d8fec..947a2cb05c 100644 --- a/source/libs/sync/test/syncAppendEntriesBatchTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesBatchTest.cpp @@ -1,11 +1,5 @@ //#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncUtil.h" -#include "trpc.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index a5e96233c2..1eb803e846 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncAppendEntriesTest.cpp b/source/libs/sync/test/syncAppendEntriesTest.cpp index 256c13e267..7b9e04a814 100644 --- a/source/libs/sync/test/syncAppendEntriesTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncApplyMsgTest.cpp b/source/libs/sync/test/syncApplyMsgTest.cpp index ce129015c6..a8a1931d1d 100644 --- a/source/libs/sync/test/syncApplyMsgTest.cpp +++ b/source/libs/sync/test/syncApplyMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncClientRequestBatchTest.cpp b/source/libs/sync/test/syncClientRequestBatchTest.cpp deleted file mode 100644 index f07ee08b2b..0000000000 --- a/source/libs/sync/test/syncClientRequestBatchTest.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SRpcMsg *createRpcMsg(int32_t i, int32_t dataLen) { - SyncPing *pSyncMsg = syncPingBuild(20); - snprintf(pSyncMsg->data, pSyncMsg->dataLen, "value_%d", i); - - SRpcMsg *pRpcMsg = (SRpcMsg *)taosMemoryMalloc(sizeof(SRpcMsg)); - memset(pRpcMsg, 0, sizeof(SRpcMsg)); - pRpcMsg->code = 10 * i; - syncPing2RpcMsg(pSyncMsg, pRpcMsg); - - syncPingDestroy(pSyncMsg); - return pRpcMsg; -} - -SyncClientRequestBatch *createMsg() { - SRpcMsg *rpcMsgPArr[5]; - memset(rpcMsgPArr, 0, sizeof(rpcMsgPArr)); - for (int32_t i = 0; i < 5; ++i) { - SRpcMsg *pRpcMsg = createRpcMsg(i, 20); - rpcMsgPArr[i] = pRpcMsg; - // taosMemoryFree(pRpcMsg); - } - - SRaftMeta raftArr[5]; - memset(raftArr, 0, sizeof(raftArr)); - for (int32_t i = 0; i < 5; ++i) { - raftArr[i].seqNum = i * 10; - raftArr[i].isWeak = i % 2; - } - - SyncClientRequestBatch *pMsg = syncClientRequestBatchBuild(rpcMsgPArr, raftArr, 5, 1234); - return pMsg; -} - -void test1() { - SyncClientRequestBatch *pMsg = createMsg(); - syncClientRequestBatchLog2((char *)"==test1==", pMsg); - syncClientRequestBatchDestroyDeep(pMsg); -} - -/* -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char * serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} -*/ - -int main() { - gRaftDetailLog = true; - tsAsyncLog = 0; - sDebugFlag = DEBUG_DEBUG + DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - - /* -test2(); -test3(); -test4(); -test5(); -*/ - - return 0; -} diff --git a/source/libs/sync/test/syncClientRequestTest.cpp b/source/libs/sync/test/syncClientRequestTest.cpp deleted file mode 100644 index b6bfcc2da5..0000000000 --- a/source/libs/sync/test/syncClientRequestTest.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" - -void logTest() { - sTrace("--- sync log test: trace"); - sDebug("--- sync log test: debug"); - sInfo("--- sync log test: info"); - sWarn("--- sync log test: warn"); - sError("--- sync log test: error"); - sFatal("--- sync log test: fatal"); -} - -SyncClientRequest *createMsg() { - SRpcMsg rpcMsg; - memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.msgType = 12345; - rpcMsg.contLen = 20; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); - rpcFreeCont(rpcMsg.pCont); - return pMsg; -} - -void test1() { - SyncClientRequest *pMsg = createMsg(); - syncClientRequestLog2((char *)"test1:", pMsg); - syncClientRequestDestroy(pMsg); -} - -void test2() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char *serialized = (char *)taosMemoryMalloc(len); - syncClientRequestSerialize(pMsg, serialized, len); - SyncClientRequest *pMsg2 = syncClientRequestAlloc(pMsg->dataLen); - syncClientRequestDeserialize(serialized, len, pMsg2); - syncClientRequestLog2((char *)"test2: syncClientRequestSerialize -> syncClientRequestDeserialize ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test3() { - SyncClientRequest *pMsg = createMsg(); - uint32_t len; - char *serialized = syncClientRequestSerialize2(pMsg, &len); - SyncClientRequest *pMsg2 = syncClientRequestDeserialize2(serialized, len); - syncClientRequestLog2((char *)"test3: syncClientRequestSerialize3 -> syncClientRequestDeserialize2 ", pMsg2); - - taosMemoryFree(serialized); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test4() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(rpcMsg.contLen); - syncClientRequestFromRpcMsg(&rpcMsg, pMsg2); - syncClientRequestLog2((char *)"test4: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -void test5() { - SyncClientRequest *pMsg = createMsg(); - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pMsg, &rpcMsg); - SyncClientRequest *pMsg2 = syncClientRequestFromRpcMsg2(&rpcMsg); - syncClientRequestLog2((char *)"test5: syncClientRequest2RpcMsg -> syncClientRequestFromRpcMsg2 ", pMsg2); - - rpcFreeCont(rpcMsg.pCont); - syncClientRequestDestroy(pMsg); - syncClientRequestDestroy(pMsg2); -} - -int main() { - tsAsyncLog = 0; - sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; - logTest(); - - test1(); - test2(); - test3(); - test4(); - test5(); - - return 0; -} diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index dcf7249d30..a6d33aa674 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c5548d25d4..cf498933e1 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 5cdbb2cc88..59d731c823 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index c60176fbf8..528cc1614d 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -1,15 +1,7 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" +#if 0 void logTest() { sTrace("--- sync log test: trace"); sDebug("--- sync log test: debug"); @@ -118,17 +110,7 @@ SyncClientRequest *step3(const SRpcMsg *pMsg) { } SSyncRaftEntry *step4(const SyncClientRequest *pMsg) { - SSyncRaftEntry *pRetMsg = syncEntryBuild2((SyncClientRequest *)pMsg, 100, 0); - return pRetMsg; -} - -char *step5(const SSyncRaftEntry *pMsg, uint32_t *len) { - char *pRetMsg = syncEntrySerialize(pMsg, len); - return pRetMsg; -} - -SSyncRaftEntry *step6(const char *pMsg, uint32_t len) { - SSyncRaftEntry *pRetMsg = syncEntryDeserialize(pMsg, len); + SSyncRaftEntry *pRetMsg = syncEntryBuildFromClientRequest((SyncClientRequest *)pMsg, 100, 0); return pRetMsg; } @@ -137,13 +119,14 @@ SRpcMsg *step7(const SSyncRaftEntry *pMsg) { syncEntry2OriginalRpc(pMsg, pRetMsg); return pRetMsg; } - +#endif int main(int argc, char **argv) { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; void logTest(); +#if 0 myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); @@ -188,20 +171,9 @@ int main(int argc, char **argv) { syncEntryLog2((char *)"==pEntry==", pEntry); - // step5 - uint32_t len; - char *pMsg5 = step5(pMsg4, &len); - char *s = syncUtilPrintBin(pMsg5, len); - printf("==step5== [%s] \n", s); - taosMemoryFree(s); - - // step6 - SSyncRaftEntry *pMsg6 = step6(pMsg5, len); - syncEntryLog2((char *)"==step6==", pMsg6); - // step7 SRpcMsg *pMsg7 = step7(pMsg6); syncRpcMsgLog2((char *)"==step7==", pMsg7); - +#endif return 0; } diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index bb5f968dc6..54bfb1b387 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryCacheTest.cpp b/source/libs/sync/test/syncEntryCacheTest.cpp index 3ee28ce96b..b86422b0b1 100644 --- a/source/libs/sync/test/syncEntryCacheTest.cpp +++ b/source/libs/sync/test/syncEntryCacheTest.cpp @@ -1,12 +1,4 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tref.h" -#include "tskiplist.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp index b274408c01..369306b857 100644 --- a/source/libs/sync/test/syncEntryTest.cpp +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -1,11 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -38,10 +31,10 @@ void test2() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test2"); - SSyncRaftEntry* pEntry = syncEntryBuild2(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -52,10 +45,10 @@ void test3() { pSyncMsg->isWeak = 1; strcpy(pSyncMsg->data, "test3"); - SSyncRaftEntry* pEntry = syncEntryBuild3(pSyncMsg, 100, 200); + SSyncRaftEntry* pEntry = syncEntryBuildFromClientRequest(pSyncMsg, 100, 200); syncEntryPrint(pEntry); - syncClientRequestDestroy(pSyncMsg); + taosMemoryFree(pSyncMsg); syncEntryDestory(pEntry); } @@ -71,14 +64,7 @@ void test4() { strcpy(pEntry->data, "test4"); syncEntryPrint(pEntry); - uint32_t len; - char* serialized = syncEntrySerialize(pEntry, &len); - assert(serialized != NULL); - SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); - syncEntryPrint(pEntry2); - - taosMemoryFree(serialized); - syncEntryDestory(pEntry2); + // syncEntryDestory(pEntry2); syncEntryDestory(pEntry); } diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index a220c2aed4..3cd6048428 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -1,9 +1,4 @@ -#include "syncEnv.h" -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "ttime.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHashCacheTest.cpp b/source/libs/sync/test/syncHashCacheTest.cpp index 7d822971da..14a29c9a1e 100644 --- a/source/libs/sync/test/syncHashCacheTest.cpp +++ b/source/libs/sync/test/syncHashCacheTest.cpp @@ -1,11 +1,4 @@ -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "tskiplist.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp index 1fac03652b..6f7f4d18f3 100644 --- a/source/libs/sync/test/syncHeartbeatReplyTest.cpp +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp index b0c0554355..c8c38c0c38 100644 --- a/source/libs/sync/test/syncHeartbeatTest.cpp +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOClientTest.cpp b/source/libs/sync/test/syncIOClientTest.cpp index bd0221114a..f66a854657 100644 --- a/source/libs/sync/test/syncIOClientTest.cpp +++ b/source/libs/sync/test/syncIOClientTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" -#include "tdatablock.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index f88e4f240d..a082f9373a 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -101,7 +95,7 @@ int main(int argc, char** argv) { syncPingReply2RpcMsg(pSyncMsg, &rpcMsg); SEpSet epSet; - syncUtilnodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); + syncUtilNodeInfo2EpSet(&pSyncNode->myNodeInfo, &epSet); rpcMsg.info.noResp = 1; pSyncNode->syncSendMSg(&epSet, &rpcMsg); diff --git a/source/libs/sync/test/syncIOServerTest.cpp b/source/libs/sync/test/syncIOServerTest.cpp index 1d7402e461..add7d6df60 100644 --- a/source/libs/sync/test/syncIOServerTest.cpp +++ b/source/libs/sync/test/syncIOServerTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 9c2342828e..71274f5ca3 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 64b65f25c8..9ef8a7ad40 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 23b8693a0b..f84119045f 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -1,12 +1,4 @@ -#include "syncIndexMgr.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp index 763117c0c9..a44d9c4e31 100644 --- a/source/libs/sync/test/syncIndexTest.cpp +++ b/source/libs/sync/test/syncIndexTest.cpp @@ -1,8 +1,4 @@ -#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void print(SHashObj *pNextIndex) { printf("----------------\n"); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 2fe5dd7e18..b15a767db5 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -1,10 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLeaderTransferTest.cpp b/source/libs/sync/test/syncLeaderTransferTest.cpp index bd1f22edff..fd8cdfaac0 100644 --- a/source/libs/sync/test/syncLeaderTransferTest.cpp +++ b/source/libs/sync/test/syncLeaderTransferTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLocalCmdTest.cpp b/source/libs/sync/test/syncLocalCmdTest.cpp index b42626df29..fa0d91d45f 100644 --- a/source/libs/sync/test/syncLocalCmdTest.cpp +++ b/source/libs/sync/test/syncLocalCmdTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck.cpp b/source/libs/sync/test/syncLogStoreCheck.cpp index 431b291ca7..e02c484729 100644 --- a/source/libs/sync/test/syncLogStoreCheck.cpp +++ b/source/libs/sync/test/syncLogStoreCheck.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp index 80679bc85c..fce0a21a47 100644 --- a/source/libs/sync/test/syncLogStoreCheck2.cpp +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp index 9ff0ed2089..1b898803ef 100644 --- a/source/libs/sync/test/syncLogStoreTest.cpp +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingReplyTest.cpp b/source/libs/sync/test/syncPingReplyTest.cpp index e5158a7932..661509e9a2 100644 --- a/source/libs/sync/test/syncPingReplyTest.cpp +++ b/source/libs/sync/test/syncPingReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 781ffc31db..19f377e037 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 1777b9b9e2..54cc506b03 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index 5a0b5e9953..ded7e22eac 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index 09f56815cd..4f1bb64798 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp index a30dcc2c54..e5175f1101 100644 --- a/source/libs/sync/test/syncPreSnapshotReplyTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotReplyTest.cpp @@ -1,10 +1,6 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncPreSnapshotTest.cpp b/source/libs/sync/test/syncPreSnapshotTest.cpp index 03894dfa84..2ce95abea3 100644 --- a/source/libs/sync/test/syncPreSnapshotTest.cpp +++ b/source/libs/sync/test/syncPreSnapshotTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgIndexTest.cpp b/source/libs/sync/test/syncRaftCfgIndexTest.cpp index 522da159b5..e6d3f23b58 100644 --- a/source/libs/sync/test/syncRaftCfgIndexTest.cpp +++ b/source/libs/sync/test/syncRaftCfgIndexTest.cpp @@ -1,10 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index b1f820c78e..c841a68fde 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -1,10 +1,4 @@ -#include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index e7ef69da20..08716c876b 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -1,8 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void usage(char* exe) { printf("Usage: %s host port \n", exe); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp index 278113919a..e309a2e432 100644 --- a/source/libs/sync/test/syncRaftLogTest.cpp +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -1,12 +1,5 @@ #include "syncRaftLog.h" -//#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 78c08a6d8d..a7752dcb8b 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index cf862feb2a..31c06625aa 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 070f8c0f07..87798a7d80 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -1,9 +1,5 @@ #include "syncRaftStore.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index c5132018a2..47bc290abe 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "tref.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 7506ae19bd..2419ec0974 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -1,10 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteReplyTest.cpp b/source/libs/sync/test/syncRequestVoteReplyTest.cpp index 3d9db17725..973ccf040c 100644 --- a/source/libs/sync/test/syncRequestVoteReplyTest.cpp +++ b/source/libs/sync/test/syncRequestVoteReplyTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRequestVoteTest.cpp b/source/libs/sync/test/syncRequestVoteTest.cpp index 94e1504add..275f7804bf 100644 --- a/source/libs/sync/test/syncRequestVoteTest.cpp +++ b/source/libs/sync/test/syncRequestVoteTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp index cad6eec91d..8d709e8c81 100644 --- a/source/libs/sync/test/syncRespMgrTest.cpp +++ b/source/libs/sync/test/syncRespMgrTest.cpp @@ -1,8 +1,4 @@ -#include "syncRespMgr.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -64,13 +60,13 @@ void syncRespMgrGetTest(uint64_t i) { void syncRespMgrGetAndDelTest(uint64_t i) { printf("------syncRespMgrGetAndDelTest-------%" PRIu64 "-- \n", i); - SRespStub stub; - int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); - if (ret == 1) { - printStub(&stub); - } else if (ret == 0) { - printf("%" PRId64 " notFound \n", i); - } + // SRespStub stub; + // int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); + // if (ret == 1) { + // printStub(&stub); + // } else if (ret == 0) { + // printf("%" PRId64 " notFound \n", i); + // } } SSyncNode *createSyncNode() { diff --git a/source/libs/sync/test/syncRestoreFromSnapshot.cpp b/source/libs/sync/test/syncRestoreFromSnapshot.cpp index 470dd678b0..44444a7f5e 100644 --- a/source/libs/sync/test/syncRestoreFromSnapshot.cpp +++ b/source/libs/sync/test/syncRestoreFromSnapshot.cpp @@ -1,12 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncRpcMsgTest.cpp b/source/libs/sync/test/syncRpcMsgTest.cpp index 941fa7eab5..3f66cd9d1c 100644 --- a/source/libs/sync/test/syncRpcMsgTest.cpp +++ b/source/libs/sync/test/syncRpcMsgTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -47,7 +43,11 @@ SyncClientRequest *createSyncClientRequest() { rpcMsg.contLen = 20; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); strcpy((char *)rpcMsg.pCont, "hello rpc"); - SyncClientRequest *pMsg = syncClientRequestBuild(&rpcMsg, 123, true, 1000); + + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, &rpcMsg, 123, true, 1000); + SyncClientRequest *pMsg = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg->data, clientRequestMsg.pCont, clientRequestMsg.contLen); return pMsg; } @@ -155,11 +155,13 @@ void test7() { } void test8() { +#if 0 SyncClientRequest *pMsg = createSyncClientRequest(); SRpcMsg rpcMsg = {0}; syncClientRequest2RpcMsg(pMsg, &rpcMsg); syncRpcMsgLog2((char *)"test8", &rpcMsg); - syncClientRequestDestroy(pMsg); + taosMemoryFree(pMsg); +#endif } int main() { diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index 6eb8548740..49b06a7d1b 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotRspTest.cpp b/source/libs/sync/test/syncSnapshotRspTest.cpp index 63905c2182..0a06b512ea 100644 --- a/source/libs/sync/test/syncSnapshotRspTest.cpp +++ b/source/libs/sync/test/syncSnapshotRspTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSendTest.cpp b/source/libs/sync/test/syncSnapshotSendTest.cpp index 83f1dfebb3..46f33e0c56 100644 --- a/source/libs/sync/test/syncSnapshotSendTest.cpp +++ b/source/libs/sync/test/syncSnapshotSendTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotSenderTest.cpp b/source/libs/sync/test/syncSnapshotSenderTest.cpp index 7b2c13463c..9a234d412e 100644 --- a/source/libs/sync/test/syncSnapshotSenderTest.cpp +++ b/source/libs/sync/test/syncSnapshotSenderTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftStore.h" -#include "syncSnapshot.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index a1cedd624a..40eac0b55f 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -1,14 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -162,8 +153,11 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); - return pRetMsg; + SRpcMsg clientRequestMsg; + syncClientRequestBuildFromRpcMsg(&clientRequestMsg, pMsg, 123, true, 1000); + SyncClientRequest *pMsg2 = (SyncClientRequest *)taosMemoryMalloc(clientRequestMsg.contLen); + memcpy(pMsg2->data, clientRequestMsg.pCont, clientRequestMsg.contLen); + return pMsg2; } int main(int argc, char **argv) { @@ -207,8 +201,8 @@ int main(int argc, char **argv) { for (int i = 0; i < 10; ++i) { SyncClientRequest *pSyncClientRequest = pMsg1; SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 97de81572a..55480802da 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" +#include "syncTest.h" /* typedef enum { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index e6f6006410..b0fe612b7f 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -1,12 +1,5 @@ #include -#include -#include "os.h" -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftCfg.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncTimeoutTest.cpp b/source/libs/sync/test/syncTimeoutTest.cpp index bae5e0ea88..ed0742f499 100644 --- a/source/libs/sync/test/syncTimeoutTest.cpp +++ b/source/libs/sync/test/syncTimeoutTest.cpp @@ -1,9 +1,5 @@ #include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncUtil.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 411915fe22..bbd4fa7d18 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -1,9 +1,4 @@ -#include "syncUtil.h" -//#include -#include -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index bbf14c604e..2d2a12c656 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -1,11 +1,4 @@ -#include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index adebfe1be2..712e97edc9 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -1,11 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "syncVoteMgr.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 7c5334f668..56dd0a5845 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -1,14 +1,5 @@ #include -#include -#include "syncEnv.h" -#include "syncIO.h" -#include "syncInt.h" -#include "syncMessage.h" -#include "syncRaftEntry.h" -#include "syncRaftLog.h" -#include "syncRaftStore.h" -#include "syncUtil.h" -#include "wal.h" +#include "syncTest.h" void logTest() { sTrace("--- sync log test: trace"); @@ -140,7 +131,8 @@ SRpcMsg *step0() { } SyncClientRequest *step1(const SRpcMsg *pMsg) { - SyncClientRequest *pRetMsg = syncClientRequestBuild(pMsg, 123, true, 1000); + SyncClientRequest *pRetMsg = NULL; + // syncClientRequestBuild(pMsg, 123, true, 1000); return pRetMsg; } @@ -179,14 +171,14 @@ int main(int argc, char **argv) { SyncClientRequest *pMsg1 = step1(pMsg0); syncClientRequestLog2((char *)"==step1==", pMsg1); - for (int i = 0; i < 10; ++i) { - SyncClientRequest *pSyncClientRequest = pMsg1; - SRpcMsg rpcMsg = {0}; - syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); - gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); + // for (int i = 0; i < 10; ++i) { + // SyncClientRequest *pSyncClientRequest = pMsg1; + // SRpcMsg rpcMsg = {0}; + // syncClientRequest2RpcMsg(pSyncClientRequest, &rpcMsg); + // gSyncNode->syncEqMsg(gSyncNode->msgcb, &rpcMsg); - taosMsleep(1000); - } + // taosMsleep(1000); + // } while (1) { sTrace("while 1 sleep"); diff --git a/source/libs/sync/test/sync_test_lib/CMakeLists.txt b/source/libs/sync/test/sync_test_lib/CMakeLists.txt new file mode 100644 index 0000000000..db775c3662 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SYNC_TEST_SRC) +add_library(sync_test_lib STATIC ${SYNC_TEST_SRC}) + +target_link_libraries( + sync_test_lib + PUBLIC sync +) + +target_include_directories( + sync_test_lib + PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/test/sync_test_lib/inc/syncIO.h similarity index 95% rename from source/libs/sync/inc/syncIO.h rename to source/libs/sync/test/sync_test_lib/inc/syncIO.h index cfc4dd2472..955a832b68 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncIO.h @@ -50,7 +50,7 @@ typedef struct SSyncIO { void *pSyncNode; int32_t (*FpOnSyncPing)(SSyncNode *pSyncNode, SyncPing *pMsg); int32_t (*FpOnSyncPingReply)(SSyncNode *pSyncNode, SyncPingReply *pMsg); - int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SyncClientRequest *pMsg, SyncIndex *pRetIndex); + int32_t (*FpOnSyncClientRequest)(SSyncNode *pSyncNode, SRpcMsg *pMsg, SyncIndex *pRetIndex); int32_t (*FpOnSyncRequestVote)(SSyncNode *pSyncNode, SyncRequestVote *pMsg); int32_t (*FpOnSyncRequestVoteReply)(SSyncNode *pSyncNode, SyncRequestVoteReply *pMsg); int32_t (*FpOnSyncAppendEntries)(SSyncNode *pSyncNode, SyncAppendEntries *pMsg); diff --git a/source/libs/sync/test/sync_test_lib/inc/syncTest.h b/source/libs/sync/test/sync_test_lib/inc/syncTest.h new file mode 100644 index 0000000000..a1881bf1b3 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/inc/syncTest.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_SYNC_TEST_H +#define _TD_LIBS_SYNC_TEST_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "syncInt.h" + +#include "tref.h" +#include "wal.h" + +#include "tref.h" +#include "syncEnv.h" +#include "syncIO.h" +#include "syncIndexMgr.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncRaftCfg.h" +#include "syncRaftEntry.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncRespMgr.h" +#include "syncSnapshot.h" +#include "syncUtil.h" +#include "syncVoteMgr.h" + +extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); +char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntryPrint(const SSyncRaftEntry* pObj); +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj); +void syncEntryLog(const SSyncRaftEntry* pObj); +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj); + +cJSON* raftCache2Json(SRaftEntryHashCache* pObj); +char* raftCache2Str(SRaftEntryHashCache* pObj); +void raftCachePrint(SRaftEntryHashCache* pObj); +void raftCachePrint2(char* s, SRaftEntryHashCache* pObj); +void raftCacheLog(SRaftEntryHashCache* pObj); +void raftCacheLog2(char* s, SRaftEntryHashCache* pObj); + +cJSON* raftEntryCache2Json(SRaftEntryCache* pObj); +char* raftEntryCache2Str(SRaftEntryCache* pObj); +void raftEntryCachePrint(SRaftEntryCache* pObj); +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj); +void raftEntryCacheLog(SRaftEntryCache* pObj); +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj); + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg); +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg); +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg); + +cJSON* logStore2Json(SSyncLogStore* pLogStore); +char* logStore2Str(SSyncLogStore* pLogStore); +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore); +char* logStoreSimple2Str(SSyncLogStore* pLogStore); +void logStorePrint(SSyncLogStore* pLogStore); +void logStorePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreLog(SSyncLogStore* pLogStore); +void logStoreLog2(char* s, SSyncLogStore* pLogStore); +void logStoreSimplePrint(SSyncLogStore* pLogStore); +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore); +void logStoreSimpleLog(SSyncLogStore* pLogStore); +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore); + +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); + +cJSON* voteGranted2Json(SVotesGranted* pVotesGranted); +char* voteGranted2Str(SVotesGranted* pVotesGranted); +cJSON* votesRespond2Json(SVotesRespond* pVotesRespond); +char* votesRespond2Str(SVotesRespond* pVotesRespond); + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); +char* syncUtilRaftId2Str(const SRaftId* p); + +cJSON* snapshotSender2Json(SSyncSnapshotSender* pSender); +char* snapshotSender2Str(SSyncSnapshotSender* pSender); +cJSON* snapshotReceiver2Json(SSyncSnapshotReceiver* pReceiver); +char* snapshotReceiver2Str(SSyncSnapshotReceiver* pReceiver); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c similarity index 98% rename from source/libs/sync/src/syncIO.c rename to source/libs/sync/test/sync_test_lib/src/syncIO.c index afa2d43e13..14adc18c66 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -279,10 +279,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - ASSERT(pSyncMsg != NULL); - io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); + io->FpOnSyncClientRequest(io->pSyncNode, pRpcMsg, NULL); } } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { diff --git a/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c new file mode 100644 index 0000000000..5e1a9be164 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMainDebug.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncNode2Json(const SSyncNode* pSyncNode) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pSyncNode != NULL) { + // init by SSyncInfo + cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); + cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg)); + cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); + cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath); + cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncSendMSg); + cJSON_AddStringToObject(pRoot, "syncSendMSg", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->msgcb); + cJSON_AddStringToObject(pRoot, "queue", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->syncEqMsg); + cJSON_AddStringToObject(pRoot, "syncEqMsg", u64buf); + + // init internal + cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo); + cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe); + cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId); + cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId); + + cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum); + cJSON* pPeers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i])); + } + cJSON* pPeersId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "peersId", pPeersId); + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i])); + } + + cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum); + cJSON* pReplicasId = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId); + for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i])); + } + + // raft algorithm + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm); + cJSON_AddStringToObject(pRoot, "pFsm", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum); + cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache); + cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); + + // life cycle + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + + // tla+ server vars + cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state); + cJSON_AddStringToObject(pRoot, "state_str", syncStr(pSyncNode->state)); + cJSON_AddItemToObject(pRoot, "pRaftStore", raftStore2Json(pSyncNode->pRaftStore)); + + // tla+ candidate vars + cJSON_AddItemToObject(pRoot, "pVotesGranted", voteGranted2Json(pSyncNode->pVotesGranted)); + cJSON_AddItemToObject(pRoot, "pVotesRespond", votesRespond2Json(pSyncNode->pVotesRespond)); + + // tla+ leader vars + cJSON_AddItemToObject(pRoot, "pNextIndex", syncIndexMgr2Json(pSyncNode->pNextIndex)); + cJSON_AddItemToObject(pRoot, "pMatchIndex", syncIndexMgr2Json(pSyncNode->pMatchIndex)); + + // tla+ log vars + cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + // timer ms init + cJSON_AddNumberToObject(pRoot, "pingBaseLine", pSyncNode->pingBaseLine); + cJSON_AddNumberToObject(pRoot, "electBaseLine", pSyncNode->electBaseLine); + cJSON_AddNumberToObject(pRoot, "hbBaseLine", pSyncNode->hbBaseLine); + + // ping timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); + cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerLogicClock); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); + cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->pingTimerCounter); + cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); + + // elect timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); + cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->electTimerLogicClock); + cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); + cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->electTimerCounter); + cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); + + // heartbeat timer + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); + cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); + cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerLogicClock); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerLogicClockUser); + cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); + cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->heartbeatTimerCounter); + cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); + + // callback + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing); + cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply); + cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote); + cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply); + cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply); + cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout); + cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf); + + // restoreFinish + cJSON_AddNumberToObject(pRoot, "restoreFinish", pSyncNode->restoreFinish); + + // snapshot senders + cJSON* pSenders = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "senders", pSenders); + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + cJSON_AddItemToArray(pSenders, snapshotSender2Json((pSyncNode->senders)[i])); + } + + // snapshot receivers + cJSON* pReceivers = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); + + // changing + cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncNode", pRoot); + return pJson; +} + +char* syncNode2Str(const SSyncNode* pSyncNode) { + cJSON* pJson = syncNode2Json(pSyncNode); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { + int32_t len = 256; + char* s = (char*)taosMemoryMalloc(len); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + snprintf(s, len, + "vgId:%d, sync %s, tm:%" PRId64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", sby:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d", + pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + + return s; +} + +// ping -------------- +int32_t syncNodePing(SSyncNode* pSyncNode, const SRaftId* destRaftId, SyncPing* pMsg) { + syncPingLog2((char*)"==syncNodePing==", pMsg); + int32_t ret = 0; + + SRpcMsg rpcMsg; + syncPing2RpcMsg(pMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodePing==", &rpcMsg); + + ret = syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); + return ret; +} + +int32_t syncNodePingSelf(SSyncNode* pSyncNode) { + int32_t ret = 0; + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); + ASSERT(ret == 0); + + syncPingDestroy(pMsg); + return ret; +} + +int32_t syncNodePingPeers(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* destId = &(pSyncNode->peersId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + +int32_t syncNodePingAll(SSyncNode* pSyncNode) { + int32_t ret = 0; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { + SRaftId* destId = &(pSyncNode->replicasId[i]); + SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); + ret = syncNodePing(pSyncNode, destId, pMsg); + ASSERT(ret == 0); + syncPingDestroy(pMsg); + } + return ret; +} + diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c new file mode 100644 index 0000000000..012382d69d --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +// --------------------------------------------- +cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { + cJSON* pRoot; + + // in compiler optimization, switch case = if else constants + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncTimeout2Json(pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { + SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPing2Json(pSyncMsg); + syncPingDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncPingReply2Json(pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest* pSyncMsg = pRpcMsg->pCont; + pRoot = syncClientRequest2Json(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { + pRoot = syncRpcUnknownMsg2Json(); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVote2Json(pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncRequestVoteReply2Json(pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntries2Json(pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncAppendEntriesReply2Json(pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotSend2Json(pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncSnapshotRsp2Json(pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_LEADER_TRANSFER) { + SyncLeaderTransfer* pSyncMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + pRoot = syncLeaderTransfer2Json(pSyncMsg); + syncLeaderTransferDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + + } else { + pRoot = cJSON_CreateObject(); + char* s; + s = syncUtilPrintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pRpcMsg->pCont), pRpcMsg->contLen); + cJSON_AddStringToObject(pRoot, "pCont2", s); + taosMemoryFree(s); + } + + cJSON_AddNumberToObject(pRoot, "msgType", pRpcMsg->msgType); + cJSON_AddNumberToObject(pRoot, "contLen", pRpcMsg->contLen); + cJSON_AddNumberToObject(pRoot, "code", pRpcMsg->code); + // cJSON_AddNumberToObject(pRoot, "persist", pRpcMsg->persist); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "RpcMsg", pRoot); + return pJson; +} + +cJSON* syncRpcUnknownMsg2Json() { + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); + cJSON_AddStringToObject(pRoot, "data", "unknown message"); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncUnknown", pRoot); + return pJson; +} + +char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { + cJSON* pJson = syncRpcMsg2Json(pRpcMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncRpcMsgPrint(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + printf("syncRpcMsgPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog(SRpcMsg* pMsg) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncRpcMsg2Str(pMsg); + sTrace("syncRpcMsgLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); + cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); + cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + + int32_t metaArrayLen = sizeof(SOffsetAndContLen) * pMsg->dataCount; // + int32_t entryArrayLen = pMsg->dataLen - metaArrayLen; + + cJSON_AddNumberToObject(pRoot, "metaArrayLen", metaArrayLen); + cJSON_AddNumberToObject(pRoot, "entryArrayLen", entryArrayLen); + + SOffsetAndContLen* metaArr = (SOffsetAndContLen*)(pMsg->data); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "offset", metaArr[i].offset); + cJSON_AddNumberToObject(pMeta, "contLen", metaArr[i].contLen); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pEntryArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entryArr", pEntryArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)(pMsg->data + metaArr[i].offset); + cJSON* pEntryJson = syncEntry2Json(pEntry); + cJSON_AddItemToArray(pEntryArr, pEntryJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncAppendEntriesBatch", pRoot); + return pJson; +} + +char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { + cJSON* pJson = syncAppendEntriesBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + printf("syncAppendEntriesBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sTrace("syncAppendEntriesBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncAppendEntriesBatch2Str(pMsg); + sLTrace("syncAppendEntriesBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { + char u64buf[128]; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); + cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); + cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); + cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); + cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); + + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncSnapshotSend", pRoot); + return pJson; +} + +char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { + cJSON* pJson = syncSnapshotSend2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + printf("syncSnapshotSendPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncSnapshotSend2Str(pMsg); + sTrace("syncSnapshotSendLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* syncClientRequestBatch2Json(const SyncClientRequestBatch* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); + + SRaftMeta* metaArr = syncClientRequestBatchMetaArr(pMsg); + SRpcMsg* msgArr = syncClientRequestBatchRpcMsgArr(pMsg); + + cJSON* pMetaArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "metaArr", pMetaArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pMeta = cJSON_CreateObject(); + cJSON_AddNumberToObject(pMeta, "seqNum", metaArr[i].seqNum); + cJSON_AddNumberToObject(pMeta, "isWeak", metaArr[i].isWeak); + cJSON_AddItemToArray(pMetaArr, pMeta); + } + + cJSON* pMsgArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "msgArr", pMsgArr); + for (int i = 0; i < pMsg->dataCount; ++i) { + cJSON* pRpcMsgJson = syncRpcMsg2Json(&msgArr[i]); + cJSON_AddItemToArray(pMsgArr, pRpcMsgJson); + } + + char* s; + s = syncUtilPrintBin((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char*)(pMsg->data), pMsg->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncClientRequestBatch", pRoot); + return pJson; +} + +char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { + cJSON* pJson = syncClientRequestBatch2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + printf("syncClientRequestBatchPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sTrace("syncClientRequestBatchLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequestBatch2Str(pMsg); + sLTrace("syncClientRequestBatchLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ---------------------- +void syncClientRequestPrint(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + printf("syncClientRequestPrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncClientRequestLog(const SyncClientRequest* pMsg) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncClientRequest2Str(pMsg); + sTrace("syncClientRequestLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c new file mode 100644 index 0000000000..88b697fbf1 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + + diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c new file mode 100644 index 0000000000..8179b24d29 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftEntryDebug.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pEntry != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); + cJSON_AddStringToObject(pRoot, "index", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->rid); + cJSON_AddStringToObject(pRoot, "rid", u64buf); + cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); + + char* s; + s = syncUtilPrintBin((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + taosMemoryFree(s); + + s = syncUtilPrintBin2((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + taosMemoryFree(s); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); + return pJson; +} + +char* syncEntry2Str(const SSyncRaftEntry* pEntry) { + cJSON* pJson = syncEntry2Json(pEntry); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ---------------------- +void syncEntryPrint(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint | len:%zu | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + printf("syncEntryPrint2 | len:%zu | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncEntryLog(const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog | len:%zu | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) { + char* serialized = syncEntry2Str(pObj); + sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + +//----------------------------------- +cJSON* raftCache2Json(SRaftEntryHashCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + while (pIter) { + pIter = taosHashIterate(pCache->pEntryHash, pIter); + if (pIter != NULL) { + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter; + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + } + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryHashCache", pRoot); + return pJson; +} + +char* raftCache2Str(SRaftEntryHashCache* pCache) { + cJSON* pJson = raftCache2Json(pCache); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftCachePrint(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCachePrint2(char* s, SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + printf("raftCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftCacheLog(SRaftEntryHashCache* pCache) { + char* serialized = raftCache2Str(pCache); + sTrace("raftCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftCacheLog2(char* s, SRaftEntryHashCache* pCache) { + if (gRaftDetailLog) { + char* serialized = raftCache2Str(pCache); + sLTrace("raftCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +cJSON* raftEntryCache2Json(SRaftEntryCache* pCache) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pCache != NULL) { + taosThreadMutexLock(&pCache->mutex); + + snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount); + cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount); + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "entries", pEntries); + + SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); + while (tSkipListIterNext(pIter)) { + SSkipListNode* pNode = tSkipListIterGet(pIter); + ASSERT(pNode != NULL); + SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + } + tSkipListDestroyIter(pIter); + + taosThreadMutexUnlock(&pCache->mutex); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot); + return pJson; +} + +char* raftEntryCache2Str(SRaftEntryCache* pObj) { + cJSON* pJson = raftEntryCache2Json(pObj); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void raftEntryCachePrint(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCachePrint2(char* s, SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + printf("raftEntryCachePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog(SRaftEntryCache* pObj) { + char* serialized = raftEntryCache2Str(pObj); + sTrace("raftEntryCacheLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void raftEntryCacheLog2(char* s, SRaftEntryCache* pObj) { + if (gRaftDetailLog) { + char* serialized = raftEntryCache2Str(pObj); + sLTrace("raftEntryCacheLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c new file mode 100644 index 0000000000..53e414ccfd --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncRaftLogDebug.c @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + + if (!raftLogIsEmpty(pLogStore)) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = NULL; + raftLogGetEntry(pLogStore, i, &pEntry); + + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + } + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} + +char* logStore2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStore2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStoreSimple", pRoot); + return pJson; +} + +char* logStoreSimple2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStoreSimple2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug ----------------- +void logStorePrint(SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStorePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStore2Str(pLogStore); + printf("logStorePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreLog(SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); + } +} + +void logStoreLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStore2Str(pLogStore); + sLTrace("logStoreLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// for debug ----------------- +void logStoreSimplePrint(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint | len:%d | %s \n", (int32_t)strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + printf("logStoreSimplePrint2 | len:%d | %s | %s \n", (int32_t)strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog(SSyncLogStore* pLogStore) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog | len:%d | %s", (int32_t)strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { + if (gRaftDetailLog) { + char* serialized = logStoreSimple2Str(pLogStore); + sTrace("logStoreSimpleLog2 | len:%d | %s | %s", (int32_t)strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c new file mode 100644 index 0000000000..fdaf4b8b34 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncSnapshotDebug.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pSender != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pSender->start); + cJSON_AddNumberToObject(pRoot, "seq", pSender->seq); + cJSON_AddNumberToObject(pRoot, "ack", pSender->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pReader); + cJSON_AddStringToObject(pRoot, "pReader", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pCurrentBlock); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", u64buf); + cJSON_AddNumberToObject(pRoot, "blockLen", pSender->blockLen); + + if (pSender->pCurrentBlock != NULL) { + char *s; + s = syncUtilPrintBin((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock", s); + taosMemoryFree(s); + s = syncUtilPrintBin2((char *)(pSender->pCurrentBlock), pSender->blockLen); + cJSON_AddStringToObject(pRoot, "pCurrentBlock2", s); + taosMemoryFree(s); + } + + cJSON *pSnapshot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); + cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); + cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotSender", pRoot); + return pJson; +} + +char *snapshotSender2Str(SSyncSnapshotSender *pSender) { + cJSON *pJson = snapshotSender2Json(pSender); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { + char u64buf[128]; + cJSON *pRoot = cJSON_CreateObject(); + + if (pReceiver != NULL) { + cJSON_AddNumberToObject(pRoot, "start", pReceiver->start); + cJSON_AddNumberToObject(pRoot, "ack", pReceiver->ack); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pWriter); + cJSON_AddStringToObject(pRoot, "pWriter", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%p", pReceiver->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + cJSON *pFromId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); + cJSON_AddStringToObject(pFromId, "addr", u64buf); + { + uint64_t u64 = pReceiver->fromId.addr; + cJSON *pTmp = pFromId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); + cJSON_AddItemToObject(pRoot, "fromId", pFromId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); + cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); + cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pReceiver->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncSnapshotReceiver", pRoot); + return pJson; +} + +char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { + cJSON *pJson = snapshotReceiver2Json(pReceiver); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c new file mode 100644 index 0000000000..b12f4e76a5 --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn); + cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot); + return pJson; +} + +char* syncUtilRaftId2Str(const SRaftId* p) { + cJSON* pJson = syncUtilRaftId2Json(p); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c new file mode 100644 index 0000000000..f1c26a97aa --- /dev/null +++ b/source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "syncTest.h" + +cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesGranted != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i])); + } + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum); + for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) { + arr[i] = pVotesGranted->isGranted[i]; + } + cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); + + cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); + cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + + bool majority = voteGrantedMajority(pVotesGranted); + cJSON_AddNumberToObject(pRoot, "majority", majority); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot); + return pJson; +} + +char *voteGranted2Str(SVotesGranted *pVotesGranted) { + cJSON *pJson = voteGranted2Json(pVotesGranted); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { + char u64buf[128] = {0}; + cJSON *pRoot = cJSON_CreateObject(); + + if (pVotesRespond != NULL) { + cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum); + cJSON *pReplicas = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "replicas", pReplicas); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i])); + } + int32_t respondNum = 0; + int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum); + for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) { + arr[i] = pVotesRespond->isRespond[i]; + if (pVotesRespond->isRespond[i]) { + respondNum++; + } + } + cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum); + taosMemoryFree(arr); + cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); + cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + } + + cJSON *pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot); + return pJson; +} + +char *votesRespond2Str(SVotesRespond *pVotesRespond) { + cJSON *pJson = votesRespond2Json(pVotesRespond); + char *serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 3d92d164b4..7ab2bc3995 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -36,6 +36,7 @@ int32_t tdbClose(TDB *pDb); int32_t tdbBegin(TDB *pDb, TXN *pTxn); int32_t tdbCommit(TDB *pDb, TXN *pTxn); int32_t tdbPostCommit(TDB *pDb, TXN *pTxn); +int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn); int32_t tdbAbort(TDB *pDb, TXN *pTxn); int32_t tdbAlter(TDB *pDb, int pages); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e4d3cc3488..e3860f85c6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -140,6 +140,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg // tdbUnrefPage(pPage); tdbPCacheRelease(pPager->pCache, pPage, &txn); tdbCommit(pPager->pEnv, &txn); + tdbPostCommit(pPager->pEnv, &txn); tdbTxnClose(&txn); } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 855e3510f2..5aff5b7bb2 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -138,7 +138,24 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerPostCommit(pPager, pTxn); if (ret < 0) { - tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, + pTxn->txnId); + return -1; + } + } + + return 0; +} + +int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; + + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerPrepareAsyncCommit(pPager, pTxn); + if (ret < 0) { + tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, + pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index cee54c1a73..c3ae1dc739 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -287,6 +287,10 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return -1; } + return 0; +} + +int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { // remove the journal file if (tdbOsClose(pPager->jfd) < 0) { tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); @@ -305,15 +309,54 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return 0; } -int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); +int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { + SPage *pPage; + int ret; + + // sync the journal file + ret = tdbOsFSync(pPager->jfd); + if (ret < 0) { + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pPager->inTran = 0; + // loop to write the dirty pages to file + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + if (pPage->isLocal) continue; + ret = tdbPagerWritePageToDB(pPager, pPage); + if (ret < 0) { + tdbError("failed to write page to db since %s", tstrerror(terrno)); + return -1; + } + } + tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); + pPager->dbOrigSize = pPager->dbFileSize; + + // release the page + iter = tRBTreeIterCreate(&pPager->rbt, 1); + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + if (pPage->isLocal) continue; + pPage->isDirty = 0; + + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + /* + tRBTreeCreate(&pPager->rbt, pageCmpFn); + + // sync the db file + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + */ return 0; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index b45747c972..e5ece98b28 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -191,6 +191,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage); int tdbPagerBegin(SPager *pPager, TXN *pTxn); int tdbPagerCommit(SPager *pPager, TXN *pTxn); int tdbPagerPostCommit(SPager *pPager, TXN *pTxn); +int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn); int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index b9167501e2..3b7182f983 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); int index; void* parent; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d3db4879d1..feb55985de 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -53,6 +53,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 98677c64d3..457d3e5cb1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -143,6 +143,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); static void cliHandleResp(SCliConn* conn); // handle except about conn static void cliHandleExcept(SCliConn* conn); +static void cliReleaseUnfinishedMsg(SCliConn* conn); // handle req from app static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); @@ -163,17 +164,6 @@ static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); -static void cliReleaseUnfinishedMsg(SCliConn* conn) { - for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { - SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); - if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { - if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { - conn->ctx.freeFunc(msg->ctx->ahandle); - } - } - destroyCmsg(msg); - } -} #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -266,6 +256,23 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { static void* cliWorkThread(void* arg); +static void cliReleaseUnfinishedMsg(SCliConn* conn) { + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); + if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { + if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { + conn->ctx.freeFunc(msg->ctx->ahandle); + } else if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); + pTransInst->destroyFp(msg->ctx->ahandle); + } + } + destroyCmsg(msg); + } +} bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; @@ -474,7 +481,7 @@ void* destroyConnPool(void* pool) { } static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { - char key[32] = {0}; + char key[TSDB_FQDN_LEN + 64] = {0}; CONN_CONSTRUCT_HASH_KEY(key, ip, port); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); @@ -525,7 +532,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->status = ConnInPool; if (conn->list == NULL) { - char key[32] = {0}; + char key[TSDB_FQDN_LEN + 64] = {0}; CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port); tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); conn->list = taosHashGet((SHashObj*)pool, key, strlen(key)); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 309e6b30ae..95065972a3 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -26,12 +26,14 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { SArray* pArray = taosMemoryMalloc(sizeof(SArray)); if (pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pArray->size = 0; pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); return NULL; } diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 5d73a1464b..d84a3d25c6 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -249,7 +249,7 @@ int32_t compareJsonVal(const void *pLeft, const void *pRight) { } int32_t compareInt8Int16(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int16_t right = GET_INT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -257,7 +257,7 @@ int32_t compareInt8Int16(const void *pLeft, const void *pRight) { } int32_t compareInt8Int32(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int32_t right = GET_INT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -265,7 +265,7 @@ int32_t compareInt8Int32(const void *pLeft, const void *pRight) { } int32_t compareInt8Int64(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); int64_t right = GET_INT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -273,7 +273,7 @@ int32_t compareInt8Int64(const void *pLeft, const void *pRight) { } int32_t compareInt8Float(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); float right = GET_FLOAT_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -281,7 +281,7 @@ int32_t compareInt8Float(const void *pLeft, const void *pRight) { } int32_t compareInt8Double(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); double right = GET_DOUBLE_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -289,7 +289,7 @@ int32_t compareInt8Double(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint8(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint8_t right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -297,7 +297,7 @@ int32_t compareInt8Uint8(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint16(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint16_t right = GET_UINT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -305,7 +305,7 @@ int32_t compareInt8Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint32_t right = GET_UINT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -313,7 +313,7 @@ int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { - int8_t left = GET_INT32_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); uint64_t right = GET_UINT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -321,7 +321,7 @@ int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { } int32_t compareInt16Int8(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int8_t right = GET_INT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -329,7 +329,7 @@ int32_t compareInt16Int8(const void *pLeft, const void *pRight) { } int32_t compareInt16Int32(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int32_t right = GET_INT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -337,7 +337,7 @@ int32_t compareInt16Int32(const void *pLeft, const void *pRight) { } int32_t compareInt16Int64(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); int64_t right = GET_INT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -345,7 +345,7 @@ int32_t compareInt16Int64(const void *pLeft, const void *pRight) { } int32_t compareInt16Float(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); float right = GET_FLOAT_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -353,7 +353,7 @@ int32_t compareInt16Float(const void *pLeft, const void *pRight) { } int32_t compareInt16Double(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); double right = GET_DOUBLE_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -361,7 +361,7 @@ int32_t compareInt16Double(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint8(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint8_t right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -369,7 +369,7 @@ int32_t compareInt16Uint8(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint16(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint16_t right = GET_UINT16_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -377,7 +377,7 @@ int32_t compareInt16Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint32_t right = GET_UINT32_VAL(pRight); if (left > right) return 1; if (left < right) return -1; @@ -385,7 +385,7 @@ int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint64(const void *pLeft, const void *pRight) { - int16_t left = GET_INT32_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); uint64_t right = GET_UINT64_VAL(pRight); if (left > right) return 1; if (left < right) return -1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f7e56f372f..0e6568d692 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -225,11 +225,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exis TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database in dropping status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a03c04ed6e..f7949048ca 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -317,14 +317,14 @@ static void taosGetLogFileName(char *fn) { for (int32_t i = 0; i < tsLogObj.fileNum; i++) { char fileName[LOG_FILE_NAME_LEN]; - sprintf(fileName, "%s%d.0", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.0", fn, i); bool file1open = taosCheckFileIsOpen(fileName); - sprintf(fileName, "%s%d.1", fn, i); + snprintf(fileName, LOG_FILE_NAME_LEN, "%s%d.1", fn, i); bool file2open = taosCheckFileIsOpen(fileName); if (!file1open && !file2open) { - sprintf(tsLogObj.logName, "%s%d", fn, i); + snprintf(tsLogObj.logName, LOG_FILE_NAME_LEN, "%s%d", fn, i); return; } } @@ -586,7 +586,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg int32_t end = 0; int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; + char tmpBuf[128] = {0}; int32_t tmpBufLen = 0; if (pLogBuf == NULL || pLogBuf->stop) return -1; @@ -598,7 +598,7 @@ static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msg remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(pLogBuf) - end - 1); if (lostLine > 0) { - sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine); + snprintf(tmpBuf, tListLen(tmpBuf), "...Lost %" PRId64 " lines here...\n", lostLine); tmpBufLen = (int32_t)strlen(tmpBuf); } diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index b6139cea13..63018aea8c 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -38,7 +38,7 @@ endi print ============= create database #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | WAL_FSYNC_PERIOD value [0 ~ 180000 ms] @@ -78,7 +78,7 @@ endi if $data7_db != 1440000m,1440000m,1440000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 5ac2ee6c4e..7012fbac6c 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -37,7 +37,7 @@ endi print ============= create database with all options #database_option: { -# | BUFFER value [3~16384, default: 96] +# | BUFFER value [3~16384, default: 256] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] # | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] @@ -98,7 +98,7 @@ endi if $data7_db != 5256000m,5256000m,5256000m then # keep return -1 endi -if $data8_db != 96 then # buffer +if $data8_db != 256 then # buffer return -1 endi if $data9_db != 4 then # pagesize diff --git a/tests/script/tsim/parser/having_child.sim b/tests/script/tsim/parser/having_child.sim index ae78c806ca..db9a25365e 100644 --- a/tests/script/tsim/parser/having_child.sim +++ b/tests/script/tsim/parser/having_child.sim @@ -733,6 +733,7 @@ sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 sql select avg(f1),count(tb1.*),sum(f1),stddev(f1),LEASTSQUARES(f1,1,1) from tb1 group by f1 having sum(f1) > 2 order by f1; if $rows != 3 then + print expect 3 , actual: $rows return -1 endi if $data00 != 2.000000000 then diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index b20e2e3592..bb6604687c 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -23,443 +23,516 @@ sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); sql insert into t1 values(1648791243003,4,2,3,3.1); sql insert into t1 values(1648791213004,4,2,3,4.1); -sleep 1000 + +$loop_count = 0 + +loop0: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; -if $rows != 4 then - print ======$rows +$loop_count = $loop_count + 1 +if $loop_count == 20 then return -1 endi +if $rows != 4 then + print =====rows=$rows + goto loop0 +endi + # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop0 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop0 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop0 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop0 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop0 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop0 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop0 endi if $data13 != 2 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop0 endi if $data14 != 2 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop0 endi if $data15 != 3 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop0 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223001,12,14,13,11.1); -sleep 500 + +$loop_count = 0 +loop1: +sleep 200 + sql select * from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + print count(*) , count(d) , sum(a) , max(b) , min(c) print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 if $rows != 4 then print ======$rows - return -1 + goto loop1 endi # row 0 if $data01 != 2 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop1 endi if $data02 != 2 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop1 endi if $data03 != 5 then - print ======$data03 - return -1 + print =====data03=$data03 + goto loop1 endi if $data04 != 2 then - print ======$data04 - return -1 + print =====data04=$data04 + goto loop1 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop1 endi # row 1 if $data11 != 1 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop1 endi if $data12 != 1 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop1 endi if $data13 != 12 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop1 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop1 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop1 endi # row 2 if $data21 != 1 then - print ======$data21 + print =====data21=$data21 return -1 endi if $data22 != 1 then - print ======$data22 + print =====data22=$data22 return -1 endi if $data23 != 3 then - print ======$data23 + print =====data23=$data23 return -1 endi if $data24 != 2 then - print ======$data24 + print =====data24=$data24 return -1 endi if $data25 != 3 then - print ======$data25 + print =====data25=$data25 return -1 endi # row 3 if $data31 != 1 then - print ======$data31 + print =====data31=$data31 return -1 endi if $data32 != 1 then - print ======$data32 + print =====data32=$data32 return -1 endi if $data33 != 4 then - print ======$data33 + print =====data33=$data33 return -1 endi if $data34 != 2 then - print ======$data34 + print =====data34=$data34 return -1 endi if $data35 != 3 then - print ======$data35 + print =====data35=$data35 return -1 endi sql insert into t1 values(1648791223002,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop2: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 2 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop2 endi if $data12 != 2 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop2 endi if $data13 != 24 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop2 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop2 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop2 endi sql insert into t1 values(1648791223003,12,14,13,11.1); -sleep 100 + +$loop_count = 0 +loop3: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop3 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop3 endi if $data13 != 36 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop3 endi if $data14 != 14 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop3 endi if $data15 != 13 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop3 endi sql insert into t1 values(1648791223001,1,1,1,1.1); sql insert into t1 values(1648791223002,2,2,2,2.1); sql insert into t1 values(1648791223003,3,3,3,3.1); -sleep 100 + +$loop_count = 0 +loop4: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 3 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop4 endi if $data12 != 3 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop4 endi if $data13 != 6 then - print ======$data13 - return -1 + print =====data13=$data13 + goto loop4 endi if $data14 != 3 then - print ======$data14 - return -1 + print =====data14=$data14 + goto loop4 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop4 endi sql insert into t1 values(1648791233003,3,2,3,2.1); sql insert into t1 values(1648791233002,5,6,7,8.1); sql insert into t1 values(1648791233002,3,2,3,2.1); -sleep 100 + +$loop_count = 0 +loop5: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 2 if $data21 != 2 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop5 endi if $data22 != 2 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop5 endi if $data23 != 6 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop5 endi if $data24 != 2 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop5 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop5 endi sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop6: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 0 if $data01 != 4 then - print ======$data01 - return -1 + print =====data01=$data01 + goto loop6 endi if $data02 != 4 then - print ======$data02 - return -1 + print =====data02=$data02 + goto loop6 endi if $data03 != 50 then - print ======$data03 != 50 - return -1 + print =====data03=$data03 != 50 + goto loop6 endi if $data04 != 20 then - print ======$data04 != 20 - return -1 + print =====data04=$data04 != 20 + goto loop6 endi if $data05 != 3 then - print ======$data05 - return -1 + print =====data05=$data05 + goto loop6 endi sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); -sleep 100 + +$loop_count = 0 +loop7: +sleep 200 + sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + # row 1 if $data11 != 4 then - print ======$data11 - return -1 + print =====data11=$data11 + goto loop7 endi if $data12 != 4 then - print ======$data12 - return -1 + print =====data12=$data12 + goto loop7 endi if $data13 != 46 then - print ======$data13 != 46 - return -1 + print =====data13=$data13 != 46 + goto loop7 endi if $data14 != 20 then - print ======$data14 != 20 - return -1 + print =====data14=$data14 != 20 + goto loop7 endi if $data15 != 1 then - print ======$data15 - return -1 + print =====data15=$data15 + goto loop7 endi # row 2 if $data21 != 4 then - print ======$data21 - return -1 + print =====data21=$data21 + goto loop7 endi if $data22 != 4 then - print ======$data22 - return -1 + print =====data22=$data22 + goto loop7 endi if $data23 != 15 then - print ======$data23 - return -1 + print =====data23=$data23 + goto loop7 endi if $data24 != 4 then - print ======$data24 - return -1 + print =====data24=$data24 + goto loop7 endi if $data25 != 3 then - print ======$data25 - return -1 + print =====data25=$data25 + goto loop7 endi sql create database test2 vgroups 1; @@ -479,11 +552,11 @@ sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2 $loop_count = 0 -loop0: -sleep 300 +loop8: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -491,7 +564,7 @@ sql select * from streamt; if $rows != 4 then print =====rows=$rows - goto loop0 + goto loop8 endi sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6,6.0) t5 values(1648791213000,7,7,7,7.0); @@ -499,11 +572,11 @@ sql insert into t1 values(1648791213000,5,5,5,5.0) t2 values(1648791213000,6,6,6 $loop_count = 0 -loop1: -sleep 300 +loop9: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -511,51 +584,51 @@ sql select * from streamt order by c4 desc; if $rows != 5 then print =====rows=$rows - goto loop1 + goto loop9 endi # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop1 + goto loop9 endi if $data02 != 7 then print =====data02=$data02 - goto loop1 + goto loop9 endi # row 1 if $data11 != 1 then print =====data11=$data11 - goto loop1 + goto loop9 endi if $data12 != 6 then print =====data12=$data12 - goto loop1 + goto loop9 endi # row 2 if $data21 != 1 then print =====data21=$data21 - goto loop1 + goto loop9 endi if $data22 != 5 then print =====data22=$data22 - goto loop1 + goto loop9 endi sql insert into t1 values(1648791213000,8,8,8,8.0); $loop_count = 0 -loop2: -sleep 300 +loop10: +sleep 200 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi @@ -564,28 +637,29 @@ sql select * from streamt order by c4 desc; # row 0 if $data01 != 1 then print =====data01=$data01 - goto loop2 + goto loop10 endi if $data02 != 8 then print =====data02=$data02 - goto loop2 + goto loop10 endi $loop_count = 0 -loop3: -sleep 300 +loop11: +sleep 200 + +sql select count(*) from streamt3; $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi -sql select count(*) from streamt3; # row 0 if $data00 != 5 then print =====data00=$data00 - goto loop3 + goto loop11 endi #max,min selectivity @@ -601,25 +675,26 @@ sql insert into ts1 values(1648791222001,2,2,3); sleep 50 $loop_count = 0 -loop3: +loop12: +sleep 200 + sql select * from streamtST3; -sleep 300 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 20 then return -1 endi # row 0 if $data02 != 1 then print =====data02=$data02 - goto loop3 + goto loop12 endi # row 1 if $data12 != 2 then print =====data12=$data12 - goto loop3 + goto loop12 endi @@ -629,19 +704,22 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams4 trigger at_once into streamt4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); sql insert into t1 values(1648791213000,1,2,3,1.0); +$loop_count = 0 +loop13: sleep 200 + sql select * from streamt4; # row 0 if $rows != 0 then print =====rows=$rows - return -1 + goto loop13 endi sql insert into t1 values(1648791213000,6,2,3,1.0); $loop_count = 0 -loop4: +loop14: sleep 200 sql select * from streamt4; @@ -652,13 +730,13 @@ endi if $data01 != 1 then print =====data01=$data01 - goto loop4 + goto loop14 endi sql insert into t1 values(1648791213000,2,2,3,1.0); $loop_count = 0 -loop5: +loop15: sleep 200 sql select * from streamt4; @@ -669,7 +747,7 @@ endi if $rows != 0 then print =====rows=$rows - goto loop5 + goto loop15 endi diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index afae8ef5da..5e95428e0a 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -201,7 +201,7 @@ if $loop_count == 10 then endi if $rows != 1 then - print ======$rows + print =====rows=$rows goto loop2 endi diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index ccff06819b..8efba515ae 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -118,7 +118,7 @@ step7: sql select count(*) from db.tb -x step7 print select count(*) from db.tb ==> $data00 $lastRows -if $data00 <= $lastRows then +if $data00 < $lastRows then return -1 endi diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 1f6096dd5a..15253df0c4 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -280,7 +280,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -289,7 +288,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index b49c99de34..869cb098e2 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -282,7 +282,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -291,7 +290,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 9426e7eacb..65dad64000 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -282,7 +282,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -291,7 +290,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index f6134909b4..dae707520f 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -279,7 +279,6 @@ class TDTestCase: tdSql.error("select udf1(num1) , irate(num1) from tb;") tdSql.error("select udf1(num1) , sum(num1) from tb;") tdSql.error("select udf1(num1) , stddev(num1) from tb;") - tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") @@ -288,7 +287,6 @@ class TDTestCase: tdSql.error("select udf1(c1) , irate(c1) from stb1;") tdSql.error("select udf1(c1) , sum(c1) from stb1;") tdSql.error("select udf1(c1) , stddev(c1) from stb1;") - tdSql.error("select udf1(c1) , mode(c1) from stb1;") tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") # regular table with select functions diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index 1918c0ef76..aec18a1cf1 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -10,37 +10,43 @@ from util.sql import * from util.cases import * from util.dnodes import * + class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) - self.buffer_boundary = [3,4097,8193,12289,16384] - self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + tdSql.init(conn.cursor(), logSql) + self.buffer_boundary = [3, 4097, 8193, 12289, 16384] + self.buffer_error = [self.buffer_boundary[0] - + 1, self.buffer_boundary[-1]+1, 12289, 256] # pages_boundary >= 64 - self.pages_boundary = [64,128,512] + self.pages_boundary = [64, 128, 512] self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: tdSql.execute(f'alter database db buffer {buffer}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8], buffer) tdSql.execute('drop database db') tdSql.execute('create database db vgroups 10') for buffer in self.buffer_error: tdSql.error(f'alter database db buffer {buffer}') tdSql.execute('drop database db') - + def alter_pages(self): tdSql.execute('create database db') for pages in self.pages_boundary: tdSql.execute(f'alter database db pages {pages}') - tdSql.query('select * from information_schema.ins_databases where name = "db"') - tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10], pages) tdSql.execute('drop database db') tdSql.execute('create database db') - tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.query( + 'select * from information_schema.ins_databases where name = "db"') self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') @@ -55,5 +61,6 @@ class TDTestCase: tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py new file mode 100644 index 0000000000..584e88330c --- /dev/null +++ b/tests/system-test/1-insert/delete_childtable.py @@ -0,0 +1,232 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.ntbname = 'ntb' + self.rowNum = 5 + self.tbnum = 2 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1,stbname=''): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + tdSql.query(f'select count(*) from {stbname}') + if tb_num <= 1: + if len(tdSql.queryResult) != 0: + tdLog.exit('delete case failure!') + else: + tdSql.checkEqual(tdSql.queryResult[0][0],(tb_num-1)*row_num) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(row_num-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ctb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_one_row(f'{self.stbname}_{i}',col_type,col_name,self.base_data,self.rowNum,self.dbname,) + self.delete_all_data(f'{self.stbname}_{i}',col_type,self.rowNum,self.base_data,self.dbname,i+1,self.stbname) + self.delete_error(f'{self.stbname}_{i}',col_name,col_type,self.base_data) + self.delete_rows(self.dbname,f'{self.stbname}_{i}',col_name,col_type,self.base_data,self.rowNum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}_{i}') + tdSql.execute(f'drop table {self.stbname}') + def run(self): + self.delete_data_ctb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ctb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 718a4497dc..aaad723b89 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), True) + tdSql.init(conn.cursor()) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/1-insert/delete_normaltable.py b/tests/system-test/1-insert/delete_normaltable.py new file mode 100644 index 0000000000..66ef91d955 --- /dev/null +++ b/tests/system-test/1-insert/delete_normaltable.py @@ -0,0 +1,224 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),True) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 3 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1,stbname=''): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(row_num-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num-i) + self.insert_base_data(col_type,tbname,row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.rowNum,self.dbname) + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname) + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + self.delete_rows(self.dbname,self.ntbname,col_name,col_type,self.base_data,self.rowNum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.ntbname}') + tdSql.execute(f'drop table {self.ntbname}') + tdSql.execute(f'drop database {self.dbname}') + def run(self): + self.delete_data_ntb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ntb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/delete_stable.py b/tests/system-test/1-insert/delete_stable.py new file mode 100644 index 0000000000..313b6ce731 --- /dev/null +++ b/tests/system-test/1-insert/delete_stable.py @@ -0,0 +1,233 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.stbname = 'stb' + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 3 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', + + } + + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname,tb_num=1): + tdSql.query(f'select count(*) from {tbname}') + tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + for i in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{i}',row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num*tb_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,row_num,dbname,tb_num=1): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows((row_num-1)*tb_num) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + for i in range(tb_num): + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname}_{i} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname}_{i} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + def delete_rows(self,dbname,tbname,col_name,col_type,base_data,row_num,tb_num=1): + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((i+1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts>={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(i*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<={self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((row_num-i-1)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts<{self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows((row_num-i)*tb_num) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(tb_num*(row_num - i-1)) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + tdSql.checkRows(tb_num*row_num) + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + def delete_data_stb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t1 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags(1)') + self.insert_base_data(col_type,f'{self.stbname}_{i}',self.rowNum,self.base_data) + self.delete_error(self.stbname,col_name,col_type,self.base_data) + self.delete_one_row(self.stbname,col_type,col_name,self.base_data,self.rowNum,self.dbname,self.tbnum) + self.delete_all_data(self.stbname,col_type,self.rowNum,self.base_data,self.dbname,self.tbnum) + self.delete_rows(self.dbname,self.stbname,col_name,col_type,self.base_data,self.rowNum,self.tbnum) + for func in ['first','last']: + tdSql.query(f'select {func}(*) from {self.stbname}') + tdSql.execute(f'drop table {self.stbname}') + tdSql.execute(f'drop database {self.dbname}') + def run(self): + self.delete_data_stb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_stb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/mode.py b/tests/system-test/2-query/mode.py new file mode 100644 index 0000000000..72bae55261 --- /dev/null +++ b/tests/system-test/2-query/mode.py @@ -0,0 +1,208 @@ +from random import randint, random +from numpy import equal +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self, dbname="db"): + tdSql.execute( + f'''create table {dbname}.stb + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + f''' + create table {dbname}.tb + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(2): + tdSql.execute(f'create table {dbname}.ctb{i+1} using {dbname}.stb tags ( {i+1} )') + + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:00.000', 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:01.000', 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:02.000', 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ctb1 values ('2020-12-11 00:00:03.000', 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:00.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ctb2 values ('2020-12-12 00:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into {dbname}.tb values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_mode_function(self, dbname="db"): + + # normal table + tdSql.query(f"select ts,mode(c1) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 9) + + tdSql.query(f"select ts,mode(c2) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -99999999999999999) + + tdSql.query(f"select ts,mode(c3) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -999) + + tdSql.query(f"select ts,mode(c4) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -99) + + tdSql.query(f"select ts,mode(c5) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -9.99) + + tdSql.query(f"select ts,mode(c6) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -1e+21) + + tdSql.query(f"select ts,mode(c7) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, True) + + tdSql.query(f"select ts,mode(c8) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 'binary9') + + tdSql.query(f"select ts,mode(c9) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, 'nchar9') + + tdSql.query(f"select ts,c3,c5,c8,mode(c1) from {dbname}.tb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkData(0, 1, -999) + tdSql.checkData(0, 2, -9.99) + tdSql.checkData(0, 3, 'binary9') + tdSql.checkData(0, 4, 9) + + # super table + tdSql.query(f"select ts,mode(c1) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 9) + + tdSql.query(f"select ts,mode(c2) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -99999) + + tdSql.query(f"select ts,mode(c3) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -999) + + tdSql.query(f"select ts,mode(c4) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 2)) + tdSql.checkData(0, 1, -99) + + tdSql.query(f"select ts,mode(c5) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -9.99) + + tdSql.query(f"select ts,mode(c6) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, -99.99) + + tdSql.query(f"select ts,mode(c7) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, True) + + tdSql.query(f"select ts,mode(c8) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 'binary9') + + tdSql.query(f"select ts,mode(c9) from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 'nchar9') + + tdSql.query(f"select ts,mode(c1),c3,c5,c8 from {dbname}.stb") + tdSql.checkRows(1) + ts = tdSql.getData(0, 0) + tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkData(0, 1, 9) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, -9.99) + tdSql.checkData(0, 4, 'binary9') + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test results ==============") + + self.check_mode_function() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0672c03ee8..0729d36f12 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -76,8 +76,8 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/count_partition.py -R python3 ./test.py -f 2-query/count.py python3 ./test.py -f 2-query/count.py -R -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/db.py -R python3 ./test.py -f 2-query/diff.py @@ -142,6 +142,8 @@ python3 ./test.py -f 2-query/max.py python3 ./test.py -f 2-query/max.py -R python3 ./test.py -f 2-query/min.py python3 ./test.py -f 2-query/min.py -R +python3 ./test.py -f 2-query/mode.py +python3 ./test.py -f 2-query/mode.py -R python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Now.py -R python3 ./test.py -f 2-query/percentile.py @@ -212,6 +214,9 @@ python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/tb_100w_data_order.py # TD-20200 python3 ./test.py -f 1-insert/delete_data.py +python3 ./test.py -f 1-insert/delete_stable.py +python3 ./test.py -f 1-insert/delete_childtable.py +python3 ./test.py -f 1-insert/delete_normaltable.py python3 ./test.py -f 1-insert/keep_expired.py python3 ./test.py -f 2-query/join2.py @@ -386,8 +391,9 @@ python3 ./test.py -f 2-query/Now.py -Q 2 python3 ./test.py -f 2-query/Today.py -Q 2 python3 ./test.py -f 2-query/max.py -Q 2 python3 ./test.py -f 2-query/min.py -Q 2 +python3 ./test.py -f 2-query/mode.py -Q 2 python3 ./test.py -f 2-query/count.py -Q 2 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 python3 ./test.py -f 2-query/last.py -Q 2 python3 ./test.py -f 2-query/first.py -Q 2 python3 ./test.py -f 2-query/To_iso8601.py -Q 2 @@ -482,8 +488,9 @@ python3 ./test.py -f 2-query/Now.py -Q 3 python3 ./test.py -f 2-query/Today.py -Q 3 python3 ./test.py -f 2-query/max.py -Q 3 python3 ./test.py -f 2-query/min.py -Q 3 +python3 ./test.py -f 2-query/mode.py -Q 3 python3 ./test.py -f 2-query/count.py -Q 3 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 python3 ./test.py -f 2-query/last.py -Q 3 python3 ./test.py -f 2-query/first.py -Q 3 python3 ./test.py -f 2-query/To_iso8601.py -Q 3 @@ -580,8 +587,9 @@ python3 ./test.py -f 2-query/Now.py -Q 4 python3 ./test.py -f 2-query/Today.py -Q 4 python3 ./test.py -f 2-query/max.py -Q 4 python3 ./test.py -f 2-query/min.py -Q 4 +python3 ./test.py -f 2-query/mode.py -Q 4 python3 ./test.py -f 2-query/count.py -Q 4 -# python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 python3 ./test.py -f 2-query/last.py -Q 4 python3 ./test.py -f 2-query/first.py -Q 4 python3 ./test.py -f 2-query/To_iso8601.py -Q 4 diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d4f6e15b16..9bb02159f0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -226,7 +226,7 @@ void shellRunSingleCommandImp(char *command) { int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected in set (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d row(s) affected(%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false);