diff --git a/cmake/cmake.define b/cmake/cmake.define index 9c2d5dc04c..639ae9ca3f 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -36,11 +36,13 @@ IF (TD_WINDOWS) ENDIF () ELSE () - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -g3") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + #SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") + #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") -IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") +IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ADD_DEFINITIONS("-D_TD_ARM_") ELSE () ADD_DEFINITIONS("-msse4.2 -mfma") diff --git a/cmake/cmake.platform b/cmake/cmake.platform index dfb79c0fae..0312f92a5b 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -42,7 +42,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") - ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare -Wno-return-type") + ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.") IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 73c06e7cc9..657819ff51 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -128,44 +128,44 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui pColumnInfoData->hasNull = true; } -static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) { +static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(int8_t*)p = *(int8_t*)v; } -static FORCE_INLINE int32_t colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) { - ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || - pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT); +static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) { + ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(int16_t*)p = *(int16_t*)v; } -static FORCE_INLINE int32_t colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) { +static FORCE_INLINE void colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(int32_t*)p = *(int32_t*)v; } -static FORCE_INLINE int32_t colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) { +static FORCE_INLINE void colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) { int32_t type = pColumnInfoData->info.type; ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(int64_t*)p = *(int64_t*)v; } -static FORCE_INLINE int32_t colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) { +static FORCE_INLINE void colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(float*)p = *(float*)v; } -static FORCE_INLINE int32_t colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) { +static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) { ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(double*)p = *(double*)v; } + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); @@ -178,7 +178,7 @@ void colDataTrim(SColumnInfoData* pColumnInfoData); size_t blockDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); -int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap); int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bef323cdba..b60ae60ce6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -717,6 +717,18 @@ typedef struct { char charset[TD_LOCALE_LEN]; // tsCharset } SClusterCfg; +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; + int64_t errors; +} SVnodesStat; + typedef struct { int32_t vgId; int8_t role; @@ -2293,6 +2305,24 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) { return buf; } +typedef struct { + char indexFName[TSDB_INDEX_FNAME_LEN]; +} SUserIndexReq; + +int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); +int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq); + +typedef struct { + char dbFName[TSDB_DB_FNAME_LEN]; + char tblFName[TSDB_TABLE_FNAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + char indexType[TSDB_INDEX_TYPE_LEN]; + char indexExts[TSDB_INDEX_EXTS_LEN]; +} SUserIndexRsp; + +int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp); +int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp); + typedef struct { int8_t mqMsgType; int32_t code; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 63d1fc5014..378ae54e41 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -156,6 +156,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "mnode-get-index", NULL, NULL) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -218,6 +219,15 @@ enum { TD_NEW_MSG_SEG(TDMT_SCH_MSG) TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL) + // Monitor info exchange between processes + TD_NEW_MSG_SEG(TDMT_MON_MSG) + TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_QM_INFO, "monitor-qinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) + #if defined(TD_MSG_NUMBER_) TDMT_MAX #endif diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a0b342fca2..5abda69aa8 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -78,6 +78,7 @@ typedef struct SDbVgVersion { } SDbVgVersion; typedef SDbCfgRsp SDbCfgInfo; +typedef SUserIndexRsp SIndexInfo; int32_t catalogInit(SCatalogCfg *cfg); @@ -221,6 +222,8 @@ int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *n int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); +int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); + /** * Destroy catalog and relase all resources diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index fc7a4bb95d..f5080fbe7b 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -19,6 +19,7 @@ #include "tarray.h" #include "tdef.h" #include "tlog.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -29,6 +30,44 @@ extern "C" { #define MON_VER_LEN 12 #define MON_LOG_LEN 1024 +typedef struct { + int64_t ts; + ELogLevel level; + char content[MON_LOG_LEN]; +} SMonLogItem; + +typedef struct { + SArray *logs; // array of SMonLogItem + int32_t numOfErrorLogs; + int32_t numOfInfoLogs; + int32_t numOfDebugLogs; + int32_t numOfTraceLogs; +} SMonLogs; + +typedef struct { + char name[TSDB_FILENAME_LEN]; + int8_t level; + SDiskSize size; +} SMonDiskDesc; + +typedef struct { + double cpu_engine; + double cpu_system; + float cpu_cores; + int64_t mem_engine; // KB + int64_t mem_system; // KB + int64_t mem_total; // KB + int64_t disk_engine; // Byte + int64_t disk_used; // Byte + int64_t disk_total; // Byte + int64_t net_in; // bytes + int64_t net_out; // bytes + int64_t io_read; // bytes + int64_t io_write; // bytes + int64_t io_read_disk; // bytes + int64_t io_write_disk; // bytes +} SMonSysInfo; + typedef struct { int32_t dnode_id; char dnode_ep[TSDB_EP_LEN]; @@ -36,6 +75,19 @@ typedef struct { int32_t protocol; } SMonBasicInfo; +typedef struct { + float uptime; // day + int8_t has_mnode; + SMonDiskDesc logdir; + SMonDiskDesc tempdir; +} SMonDnodeInfo; + +typedef struct { + SMonBasicInfo basic; + SMonDnodeInfo dnode; + SMonSysInfo sys; +} SMonDmInfo; + typedef struct { int32_t dnode_id; char dnode_ep[TSDB_EP_LEN]; @@ -87,46 +139,65 @@ typedef struct { } SMonGrantInfo; typedef struct { - float uptime; // day - double cpu_engine; - double cpu_system; - float cpu_cores; - int64_t mem_engine; // KB - int64_t mem_system; // KB - int64_t mem_total; // KB - int64_t disk_engine; // Byte - int64_t disk_used; // Byte - int64_t disk_total; // Byte - int64_t net_in; // bytes - int64_t net_out; // bytes - int64_t io_read; // bytes - int64_t io_write; // bytes - int64_t io_read_disk; // bytes - int64_t io_write_disk; // bytes - int64_t req_select; - int64_t req_insert; - int64_t req_insert_success; - int64_t req_insert_batch; - int64_t req_insert_batch_success; - int32_t errors; - int32_t vnodes_num; - int32_t masters; - int8_t has_mnode; -} SMonDnodeInfo; + SMonClusterInfo cluster; + SMonVgroupInfo vgroup; + SMonGrantInfo grant; + SMonSysInfo sys; + SMonLogs log; +} SMonMmInfo; + +int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo); +int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo); +void tFreeSMonMmInfo(SMonMmInfo *pInfo); typedef struct { - char name[TSDB_FILENAME_LEN]; - int8_t level; - SDiskSize size; -} SMonDiskDesc; - -typedef struct { - SArray *datadirs; // array of SMonDiskDesc - SMonDiskDesc logdir; - SMonDiskDesc tempdir; + SArray *datadirs; // array of SMonDiskDesc } SMonDiskInfo; -typedef struct SMonInfo SMonInfo; +typedef struct { + SMonDiskInfo tfs; + SVnodesStat vstat; + SMonSysInfo sys; + SMonLogs log; +} SMonVmInfo; + +int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo); +int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo); +void tFreeSMonVmInfo(SMonVmInfo *pInfo); + +typedef struct { + SMonSysInfo sys; + SMonLogs log; +} SMonQmInfo; + +int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo); +int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo); +void tFreeSMonQmInfo(SMonQmInfo *pInfo); + +typedef struct { + SMonSysInfo sys; + SMonLogs log; +} SMonSmInfo; + +int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo); +int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo); +void tFreeSMonSmInfo(SMonSmInfo *pInfo); +typedef struct { + SMonSysInfo sys; + SMonLogs log; +} SMonBmInfo; + +int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo); +int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo); +void tFreeSMonBmInfo(SMonBmInfo *pInfo); + +typedef struct { + SArray *pVloads; // SVnodeLoad +} SMonVloadInfo; + +int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo); +int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo); +void tFreeSMonVloadInfo(SMonVloadInfo *pInfo); typedef struct { const char *server; @@ -138,16 +209,14 @@ typedef struct { int32_t monInit(const SMonCfg *pCfg); void monCleanup(); void monRecordLog(int64_t ts, ELogLevel level, const char *content); - -SMonInfo *monCreateMonitorInfo(); -void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); -void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); -void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); -void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo); -void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); -void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); -void monSendReport(SMonInfo *pMonitor); -void monCleanupMonitorInfo(SMonInfo *pMonitor); +int32_t monGetLogs(SMonLogs *logs); +void monSetDmInfo(SMonDmInfo *pInfo); +void monSetMmInfo(SMonMmInfo *pInfo); +void monSetVmInfo(SMonVmInfo *pInfo); +void monSetQmInfo(SMonQmInfo *pInfo); +void monSetSmInfo(SMonSmInfo *pInfo); +void monSetBmInfo(SMonBmInfo *pInfo); +void monSendReport(); #ifdef __cplusplus } diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 6b6d11c2fe..f052c9daa7 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -238,6 +238,7 @@ typedef struct SSelectStmt { SNode* pSlimit; char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; + bool isEmptyResult; } SSelectStmt; typedef enum ESetOperatorType { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f10185c44f..fef624288a 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -44,8 +44,15 @@ typedef struct SCmdMsgInfo { void* pExtension; // todo remove it soon } SCmdMsgInfo; +typedef enum EQueryExecMode { + QUERY_EXEC_MODE_LOCAL = 1, + QUERY_EXEC_MODE_RPC, + QUERY_EXEC_MODE_SCHEDULE, + QUERY_EXEC_MODE_EMPTY_RESULT +} EQueryExecMode; + typedef struct SQuery { - bool directRpc; + EQueryExecMode execMode; bool haveResultSet; SNode* pRoot; int32_t numOfResCols; @@ -55,7 +62,6 @@ typedef struct SQuery { SArray* pDbList; SArray* pTableList; bool showRewrite; - bool localCmd; } SQuery; int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index e1cc6f7829..022f11bb0e 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -44,8 +44,8 @@ int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); -int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); -int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes); +void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); +void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 37e05bf3d3..5b7aa51881 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -217,6 +217,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385) #define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0386) #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0387) +#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388) // mnode-vgroup #define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390) diff --git a/include/util/tdef.h b/include/util/tdef.h index 2a634e2327..98fbe94d66 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -109,6 +109,9 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_VGROUPS "vgroups" +#define TSDB_INDEX_TYPE_SMA "SMA" +#define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" + #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 #define TSDB_TICK_PER_SECOND(precision) \ @@ -213,6 +216,9 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0' +#define TSDB_INDEX_TYPE_LEN 10 +#define TSDB_INDEX_EXTS_LEN 256 +#define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c3a17b6073..9938a2e1b9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -281,22 +281,35 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; - int32_t code = 0; SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return); - - if (pQuery->localCmd) { - CHECK_CODE_GOTO(execLocalCmd(pRequest, pQuery), _return); - } else if (pQuery->directRpc) { - CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); - } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + if (TSDB_CODE_SUCCESS == code) { + code = parseSql(pRequest, false, &pQuery); + } + + if (TSDB_CODE_SUCCESS == code) { + switch (pQuery->execMode) { + case QUERY_EXEC_MODE_LOCAL: + code = execLocalCmd(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_RPC: + code = execDdlQuery(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_SCHEDULE: + code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); + if (TSDB_CODE_SUCCESS == code) { + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); + } + break; + case QUERY_EXEC_MODE_EMPTY_RESULT: + pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + break; + default: + break; + } } -_return: taosArrayDestroy(pNodeList); qDestroyQuery(pQuery); if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8f55488e2c..3d8be91544 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -17,6 +17,7 @@ #include "tdatablock.h" #include "tcompare.h" #include "tglobal.h" +#include "tlog.h" int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; @@ -130,44 +131,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); pColumnInfoData->varmeta.length += varDataTLen(pData); } else { - char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; - switch (type) { - case TSDB_DATA_TYPE_BOOL: { - *(bool*)p = *(bool*)pData; - break; - } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: { - *(int8_t*)p = *(int8_t*)pData; - break; - } - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: { - *(int16_t*)p = *(int16_t*)pData; - break; - } - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: { - *(int32_t*)p = *(int32_t*)pData; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: { - *(int64_t*)p = *(int64_t*)pData; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float*)p = *(float*)pData; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double*)p = *(double*)pData; - break; - } - default: - assert(0); - } + memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); } return 0; @@ -175,6 +139,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, int32_t numOfRow2) { + if (numOfRow2 <= 0) return; + uint32_t total = numOfRow1 + numOfRow2; if (BitmapLen(numOfRow1) < BitmapLen(total)) { @@ -189,22 +155,32 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c if (remindBits == 0) { // no need to shift bits of bitmap memcpy(pColumnInfoData->nullbitmap + BitmapLen(numOfRow1), pSource->nullbitmap, BitmapLen(numOfRow2)); - } else { - int32_t len = BitmapLen(numOfRow2); - int32_t i = 0; + return; + } - uint8_t* p = (uint8_t*)pSource->nullbitmap; - pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); + uint8_t* p = (uint8_t*)pSource->nullbitmap; + pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits - uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; - while (i < len) { - start[i] |= (p[i] << shiftBits); - i += 1; + if (BitmapLen(numOfRow1) == BitmapLen(total)) { + return; + } - if (i > 1) { - start[i - 1] |= (p[i] >> remindBits); - } + int32_t len = BitmapLen(numOfRow2); + int32_t i = 0; + + uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; + int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + while (i < len) { // size limit of pSource->nullbitmap + if (i >= 1) { + start[i - 1] |= (p[i] >> remindBits); //copy remind bits } + + if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap + return; + } + + start[i] |= (p[i] << shiftBits); //copy shift bits + i += 1; } } @@ -216,6 +192,9 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co return numOfRow1; } + if (pSource->hasNull) { + pColumnInfoData->hasNull = pSource->hasNull; + } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); @@ -336,13 +315,18 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { return 0; } -int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); +// if pIndexMap = NULL, merger one column by on column +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) { + assert(pSrc != NULL && pDest != NULL); - int32_t numOfCols = pSrc->info.numOfCols; + int32_t numOfCols = pDest->info.numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { + int32_t mapIndex = i; + if(pIndexMap) { + mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); + } SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); @@ -399,49 +383,49 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // TODO speedup by checking if the whole page can fit in firstly. if (!hasVarCol) { size_t rowSize = blockDataGetRowSize(pBlock); - int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8; + int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0); + ASSERT(capacity > 0); - *stopIndex = startIndex + capacity; + *stopIndex = startIndex + capacity - 1; if (*stopIndex >= numOfRows) { *stopIndex = numOfRows - 1; } return TSDB_CODE_SUCCESS; - } else { - // iterate the rows that can be fit in this buffer page - int32_t size = (headerSize + colHeaderSize); - - for (int32_t j = startIndex; j < numOfRows; ++j) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - if (pColInfoData->varmeta.offset[j] != -1) { - char* p = colDataGetData(pColInfoData, j); - size += varDataTLen(p); - } - - size += sizeof(pColInfoData->varmeta.offset[0]); - } else { - size += pColInfoData->info.bytes; - - if (((j - startIndex) & 0x07) == 0) { - size += 1; // the space for null bitmap - } + } + // iterate the rows that can be fit in this buffer page + int32_t size = (headerSize + colHeaderSize); + for (int32_t j = startIndex; j < numOfRows; ++j) { + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->varmeta.offset[j] != -1) { + char* p = colDataGetData(pColInfoData, j); + size += varDataTLen(p); } - } - if (size > pageSize) { - *stopIndex = j - 1; - ASSERT(*stopIndex > startIndex); + size += sizeof(pColInfoData->varmeta.offset[0]); + } else { + size += pColInfoData->info.bytes; - return TSDB_CODE_SUCCESS; + if (((j - startIndex) & 0x07) == 0) { + size += 1; // the space for null bitmap + } } } - // all fit in - *stopIndex = numOfRows - 1; - return TSDB_CODE_SUCCESS; + if (size > pageSize) { // pageSize must be able to hold one row + *stopIndex = j - 1; + ASSERT(*stopIndex >= startIndex); + + return TSDB_CODE_SUCCESS; + } } + + // all fit in + *stopIndex = numOfRows - 1; + return TSDB_CODE_SUCCESS; + } SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) { @@ -546,6 +530,11 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { size_t metaSize = pBlock->info.rows * sizeof(int32_t); + char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pCol->varmeta.offset = (int32_t*)tmp; memcpy(pCol->varmeta.offset, pStart, metaSize); pStart += metaSize; } else { @@ -709,23 +698,13 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { void* left1 = colDataGetData(pColInfoData, left); void* right1 = colDataGetData(pColInfoData, right); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_INT: { - int32_t leftx = *(int32_t*)left1; - int32_t rightx = *(int32_t*)right1; + __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order); - if (leftx == rightx) { - break; - } else { - if (pOrder->order == TSDB_ORDER_ASC) { - return (leftx < rightx) ? -1 : 1; - } else { - return (leftx < rightx) ? 1 : -1; - } - } - } - default: - assert(0); + int ret = fn(left1, right1); + if (ret == 0) { + continue; + } else { + return ret; } } @@ -779,61 +758,12 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; } } else { - switch (pSrc->info.type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { - int32_t* p = (int32_t*)pDst->pData; - int32_t* srclist = (int32_t*)pSrc->pData; - - p[j] = srclist[index[j]]; - if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); - } - } - break; + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + continue; } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { - int32_t* p = (int32_t*)pDst->pData; - int32_t* srclist = (int32_t*)pSrc->pData; - - p[j] = srclist[index[j]]; - if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); - } - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { - int32_t* p = (int32_t*)pDst->pData; - int32_t* srclist = (int32_t*)pSrc->pData; - - p[j] = srclist[index[j]]; - if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); - } - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { - int32_t* p = (int32_t*)pDst->pData; - int32_t* srclist = (int32_t*)pSrc->pData; - - p[j] = srclist[index[j]]; - if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { - colDataSetNull_f(pDst->nullbitmap, j); - } - } - break; - } - default: - assert(0); + memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes); } } } @@ -906,33 +836,6 @@ static int32_t* createTupleIndex(size_t rows) { static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); } -static __compar_fn_t getComparFn(int32_t type, int32_t order) { - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc; - case TSDB_DATA_TYPE_SMALLINT: - return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc; - case TSDB_DATA_TYPE_INT: - return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; - case TSDB_DATA_TYPE_BIGINT: - return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc; - case TSDB_DATA_TYPE_FLOAT: - return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc; - case TSDB_DATA_TYPE_DOUBLE: - return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc; - case TSDB_DATA_TYPE_UTINYINT: - return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc; - case TSDB_DATA_TYPE_USMALLINT: - return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc; - case TSDB_DATA_TYPE_UINT: - return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc; - case TSDB_DATA_TYPE_UBIGINT: - return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc; - default: - return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; - } -} - int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { ASSERT(pDataBlock != NULL && pOrderInfo != NULL); if (pDataBlock->info.rows <= 1) { @@ -966,11 +869,11 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { int64_t p0 = taosGetTimestampUs(); - __compar_fn_t fn = getComparFn(pColInfoData->info.type, pOrder->order); + __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order); qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); + uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); return TSDB_CODE_SUCCESS; } else { // var data type @@ -999,24 +902,21 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); if (pCols == NULL) { + destroyTupleIndex(index); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } int64_t p2 = taosGetTimestampUs(); - int32_t code = blockDataAssign(pCols, pDataBlock, index); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return code; - } + blockDataAssign(pCols, pDataBlock, index); int64_t p3 = taosGetTimestampUs(); copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, + uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); destroyTupleIndex(index); @@ -1248,6 +1148,9 @@ void* blockDataDestroy(SSDataBlock* pBlock) { } SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { + if(pDataBlock == NULL){ + return NULL; + } int32_t numOfCols = pDataBlock->info.numOfCols; SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 57a8953253..ca59dcb218 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2033,6 +2033,64 @@ int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp) { return 0; } +int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->indexFName) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->indexFName) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tblFName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->colName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->indexType) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->indexExts) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tblFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->colName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->indexType) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->indexExts) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { SCoder encoder = {0}; @@ -2645,6 +2703,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; + return 0; } int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { diff --git a/source/dnode/mgmt/bm/bmHandle.c b/source/dnode/mgmt/bm/bmHandle.c index 2ae9b3817f..b73acd14c3 100644 --- a/source/dnode/mgmt/bm/bmHandle.c +++ b/source/dnode/mgmt/bm/bmHandle.c @@ -16,6 +16,34 @@ #define _DEFAULT_SOURCE #include "bmInt.h" +void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo) { +} + +int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonBmInfo bmInfo = {0}; + bmGetMonitorInfo(pWrapper, &bmInfo); + dmGetMonitorSysInfo(&bmInfo.sys); + monGetLogs(&bmInfo.log); + + int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonBmInfo(pRsp, rspLen, &bmInfo); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonBmInfo(&bmInfo); + return 0; +} + int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnode *pDnode = pWrapper->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; @@ -54,4 +82,6 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } } -void bmInitMsgHandle(SMgmtWrapper *pWrapper) {} +void bmInitMsgHandle(SMgmtWrapper *pWrapper) { + dndSetMsgHandle(pWrapper, TDMT_MON_BM_INFO, bmProcessMonitorMsg, DEFAULT_HANDLE); +} diff --git a/source/dnode/mgmt/bm/bmWorker.c b/source/dnode/mgmt/bm/bmWorker.c index a5a97f6af0..cf2d7ac939 100644 --- a/source/dnode/mgmt/bm/bmWorker.c +++ b/source/dnode/mgmt/bm/bmWorker.c @@ -33,7 +33,37 @@ static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t num } } -static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { +static inline void bmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, + .ahandle = pMsg->rpcMsg.ahandle, + .code = code, + .pCont = pMsg->pRsp, + .contLen = pMsg->rspLen}; + tmsgSendRsp(&rsp); +} + +static void bmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SBnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, get from bnode monitor queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType == TDMT_MON_BM_INFO) { + code = bmProcessGetMonBmInfoReq(pMgmt->pWrapper, pMsg); + } + + if (pRpc->msgType & 1U) { + if (code != 0 && terrno != 0) code = terrno; + bmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + +static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SBnodeMgmt *pMgmt = pInfo->ahandle; SMgmtWrapper *pWrapper = pMgmt->pWrapper; @@ -72,18 +102,37 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } +int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SBnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->monitorWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + int32_t bmStartWorker(SBnodeMgmt *pMgmt) { - SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt}; + SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessWriteQueue, .param = pMgmt}; if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { - dError("failed to start bnode write worker since %s", terrstr()); + dError("failed to start bnode-write worker since %s", terrstr()); return -1; } + if (tsMultiProcess) { + SSingleWorkerCfg mCfg = { + .min = 1, .max = 1, .name = "bnode-monitor", .fp = (FItem)bmProcessMonQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start bnode-monitor worker since %s", terrstr()); + return -1; + } + } + dDebug("bnode workers are initialized"); return 0; } void bmStopWorker(SBnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->monitorWorker); tMultiWorkerCleanup(&pMgmt->writeWorker); dDebug("bnode workers are closed"); } diff --git a/source/dnode/mgmt/dm/dmHandle.c b/source/dnode/mgmt/dm/dmHandle.c index cb712bfb48..46d94fb844 100644 --- a/source/dnode/mgmt/dm/dmHandle.c +++ b/source/dnode/mgmt/dm/dmHandle.c @@ -42,8 +42,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); if (pWrapper != NULL) { - req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); - vmMonitorVnodeLoads(pWrapper, req.pVloads); + SMonVloadInfo info = {0}; + dmGetVnodeLoads(pWrapper, &info); + req.pVloads = info.pVloads; dndReleaseWrapper(pWrapper); } @@ -96,6 +97,7 @@ int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { } pMgmt->statusSent = 0; + return TSDB_CODE_SUCCESS; } int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { @@ -117,7 +119,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { return TSDB_CODE_OPS_NOT_SUPPORT; } - static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, EDndType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { @@ -209,7 +210,7 @@ void dmInitMsgHandle(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessStatusMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMonitorMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/dm/dmMonitor.c b/source/dnode/mgmt/dm/dmMonitor.c index 22d9086971..fb5855070c 100644 --- a/source/dnode/mgmt/dm/dmMonitor.c +++ b/source/dnode/mgmt/dm/dmMonitor.c @@ -14,21 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "dndInt.h" - -static int32_t dmGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) { - tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); - pInfo->logdir.size = tsLogSpace.size; - tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); - pInfo->tempdir.size = tsTempSpace.size; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); - if (pWrapper != NULL) { - vmMonitorTfsInfo(pWrapper, pInfo); - dndReleaseWrapper(pWrapper); - } - return 0; -} +#include "dmInt.h" static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { pInfo->protocol = 1; @@ -39,6 +25,163 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); + if (pWrapper != NULL) { + pInfo->has_mnode = pWrapper->required; + dndReleaseWrapper(pWrapper); + } + tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); + pInfo->logdir.size = tsLogSpace.size; + tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); + pInfo->tempdir.size = tsTempSpace.size; +} + +static void dmGetMonitorInfo(SDnode *pDnode, SMonDmInfo *pInfo) { + dmGetMonitorBasicInfo(pDnode, &pInfo->basic); + dmGetMonitorSysInfo(&pInfo->sys); + dmGetMonitorDnodeInfo(pDnode, &pInfo->dnode); +} + +void dmSendMonitorReport(SDnode *pDnode) { + if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; + dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); + + SMonDmInfo dmInfo = {0}; + SMonMmInfo mmInfo = {0}; + SMonVmInfo vmInfo = {0}; + SMonQmInfo qmInfo = {0}; + SMonSmInfo smInfo = {0}; + SMonBmInfo bmInfo = {0}; + + SRpcMsg req = {0}; + SRpcMsg rsp; + SEpSet epset = {.inUse = 0, .numOfEps = 1}; + tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + epset.eps[0].port = tsServerPort; + + SMgmtWrapper *pWrapper = NULL; + dmGetMonitorInfo(pDnode, &dmInfo); + + bool getFromAPI = !tsMultiProcess; + pWrapper = &pDnode->wrappers[MNODE]; + if (getFromAPI) { + if (dndMarkWrapper(pWrapper) != 0) { + mmGetMonitorInfo(pWrapper, &mmInfo); + dndReleaseWrapper(pWrapper); + } + } else { + if (pWrapper->required) { + req.msgType = TDMT_MON_MM_INFO; + dndSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonMmInfo(rsp.pCont, rsp.contLen, &mmInfo); + } + rpcFreeCont(rsp.pCont); + } + } + + pWrapper = &pDnode->wrappers[VNODES]; + if (getFromAPI) { + if (dndMarkWrapper(pWrapper) != 0) { + vmGetMonitorInfo(pWrapper, &vmInfo); + dndReleaseWrapper(pWrapper); + } + } else { + if (pWrapper->required) { + req.msgType = TDMT_MON_VM_INFO; + dndSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonVmInfo(rsp.pCont, rsp.contLen, &vmInfo); + } + rpcFreeCont(rsp.pCont); + } + } + + pWrapper = &pDnode->wrappers[QNODE]; + if (getFromAPI) { + if (dndMarkWrapper(pWrapper) != 0) { + qmGetMonitorInfo(pWrapper, &qmInfo); + dndReleaseWrapper(pWrapper); + } + } else { + if (pWrapper->required) { + req.msgType = TDMT_MON_QM_INFO; + dndSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonQmInfo(rsp.pCont, rsp.contLen, &qmInfo); + } + rpcFreeCont(rsp.pCont); + } + } + + pWrapper = &pDnode->wrappers[SNODE]; + if (getFromAPI) { + if (dndMarkWrapper(pWrapper) != 0) { + smGetMonitorInfo(pWrapper, &smInfo); + dndReleaseWrapper(pWrapper); + } + } else { + if (pWrapper->required) { + req.msgType = TDMT_MON_SM_INFO; + dndSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonSmInfo(rsp.pCont, rsp.contLen, &smInfo); + } + rpcFreeCont(rsp.pCont); + } + } + + pWrapper = &pDnode->wrappers[BNODE]; + if (getFromAPI) { + if (dndMarkWrapper(pWrapper) != 0) { + bmGetMonitorInfo(pWrapper, &bmInfo); + dndReleaseWrapper(pWrapper); + } + } else { + if (pWrapper->required) { + req.msgType = TDMT_MON_BM_INFO; + dndSendRecv(pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonBmInfo(rsp.pCont, rsp.contLen, &bmInfo); + } + rpcFreeCont(rsp.pCont); + } + } + + monSetDmInfo(&dmInfo); + monSetMmInfo(&mmInfo); + monSetVmInfo(&vmInfo); + monSetQmInfo(&qmInfo); + monSetSmInfo(&smInfo); + monSetBmInfo(&bmInfo); + tFreeSMonMmInfo(&mmInfo); + tFreeSMonVmInfo(&vmInfo); + tFreeSMonQmInfo(&qmInfo); + tFreeSMonSmInfo(&smInfo); + tFreeSMonBmInfo(&bmInfo); + monSendReport(); +} + +void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { + bool getFromAPI = !tsMultiProcess; + if (getFromAPI) { + vmGetVnodeLoads(pWrapper, pInfo); + } else { + SRpcMsg req = {.msgType = TDMT_MON_VM_LOAD}; + SRpcMsg rsp = {0}; + SEpSet epset = {.inUse = 0, .numOfEps = 1}; + tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + epset.eps[0].port = tsServerPort; + + dndSendRecv(pWrapper->pDnode, &epset, &req, &rsp); + if (rsp.code == 0 && rsp.contLen > 0) { + tDeserializeSMonVloadInfo(rsp.pCont, rsp.contLen, pInfo); + } + rpcFreeCont(rsp.pCont); + } +} + +void dmGetMonitorSysInfo(SMonSysInfo *pInfo) { taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system); taosGetCpuCores(&pInfo->cpu_cores); taosGetProcMemory(&pInfo->mem_engine); @@ -47,61 +190,6 @@ static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->disk_engine = 0; pInfo->disk_used = tsDataSpace.size.used; pInfo->disk_total = tsDataSpace.size.total; - taosGetCardInfo(&pInfo->net_in, &pInfo->net_out); - taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES); - if (pWrapper != NULL) { - vmMonitorVnodeReqs(pWrapper, pInfo); - dndReleaseWrapper(pWrapper); - } - - pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - pInfo->has_mnode = pWrapper->required; - dndReleaseWrapper(pWrapper); - } + taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out); + taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk); } - -void dmSendMonitorReport(SDnode *pDnode) { - if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; - dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); - - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - SMonBasicInfo basicInfo = {0}; - dmGetMonitorBasicInfo(pDnode, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); - - SMonClusterInfo clusterInfo = {0}; - SMonVgroupInfo vgroupInfo = {0}; - SMonGrantInfo grantInfo = {0}; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) { - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); - } - dndReleaseWrapper(pWrapper); - } - - SMonDnodeInfo dnodeInfo = {0}; - dmGetMonitorDnodeInfo(pDnode, &dnodeInfo); - monSetDnodeInfo(pMonitor, &dnodeInfo); - - SMonDiskInfo diskInfo = {0}; - if (dmGetMonitorDiskInfo(pDnode, &diskInfo) == 0) { - monSetDiskInfo(pMonitor, &diskInfo); - } - - taosArrayDestroy(clusterInfo.dnodes); - taosArrayDestroy(clusterInfo.mnodes); - taosArrayDestroy(vgroupInfo.vgroups); - taosArrayDestroy(diskInfo.datadirs); - - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); -} \ No newline at end of file diff --git a/source/dnode/mgmt/dm/dmWorker.c b/source/dnode/mgmt/dm/dmWorker.c index ec7392d7e4..7009469c00 100644 --- a/source/dnode/mgmt/dm/dmWorker.c +++ b/source/dnode/mgmt/dm/dmWorker.c @@ -44,6 +44,7 @@ static void *dmThreadRoutine(void *param) { lastMonitorTime = curTime; } } + return TSDB_CODE_SUCCESS; } int32_t dmStartThread(SDnodeMgmt *pMgmt) { @@ -101,9 +102,9 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return -1; } - SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) { - dError("failed to start dnode status worker since %s", terrstr()); + SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-monitor", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &scfg) != 0) { + dError("failed to start dnode monitor worker since %s", terrstr()); return -1; } @@ -113,7 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { void dmStopWorker(SDnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->mgmtWorker); - tSingleWorkerCleanup(&pMgmt->statusWorker); + tSingleWorkerCleanup(&pMgmt->monitorWorker); if (pMgmt->threadId != NULL) { taosDestoryThread(pMgmt->threadId); @@ -131,9 +132,9 @@ int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } -int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { +int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; - SSingleWorker *pWorker = &pMgmt->statusWorker; + SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); diff --git a/source/dnode/mgmt/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h index 919b1d2c7c..84a6a53e99 100644 --- a/source/dnode/mgmt/inc/bmInt.h +++ b/source/dnode/mgmt/inc/bmInt.h @@ -17,6 +17,7 @@ #define _TD_DND_BNODE_INT_H_ #include "dndInt.h" + #include "bnode.h" #ifdef __cplusplus @@ -29,6 +30,7 @@ typedef struct SBnodeMgmt { SMgmtWrapper *pWrapper; const char *path; SMultiWorker writeWorker; + SSingleWorker monitorWorker; } SBnodeMgmt; // bmInt.c @@ -39,11 +41,13 @@ int32_t bmDrop(SMgmtWrapper *pWrapper); void bmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); // bmWorker.c int32_t bmStartWorker(SBnodeMgmt *pMgmt); void bmStopWorker(SBnodeMgmt *pMgmt); int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/dmInt.h b/source/dnode/mgmt/inc/dmInt.h index 5dcd1be47c..a671368f06 100644 --- a/source/dnode/mgmt/inc/dmInt.h +++ b/source/dnode/mgmt/inc/dmInt.h @@ -32,7 +32,7 @@ typedef struct SDnodeMgmt { TdThread *threadId; SRWLatch latch; SSingleWorker mgmtWorker; - SSingleWorker statusWorker; + SSingleWorker monitorWorker; SMsgCb msgCb; const char *path; SDnode *pDnode; @@ -54,6 +54,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg); // dmMonitor.c +void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); void dmSendMonitorReport(SDnode *pDnode); // dmWorker.c @@ -61,7 +62,7 @@ int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); -int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/dndInt.h b/source/dnode/mgmt/inc/dndInt.h index 5659becb20..7faf1e4276 100644 --- a/source/dnode/mgmt/inc/dndInt.h +++ b/source/dnode/mgmt/inc/dndInt.h @@ -169,6 +169,7 @@ void dndCleanupTrans(SDnode *pDnode); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); int32_t dndInitMsgHandle(SDnode *pDnode); +void dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); // mgmt void dmSetMgmtFp(SMgmtWrapper *pWrapper); @@ -182,22 +183,13 @@ void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); +void dmGetMonitorSysInfo(SMonSysInfo *pInfo); +void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo); +void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo); +void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo); +void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo); +void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo); +void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/mmInt.h b/source/dnode/mgmt/inc/mmInt.h index d09d15255d..df63e22059 100644 --- a/source/dnode/mgmt/inc/mmInt.h +++ b/source/dnode/mgmt/inc/mmInt.h @@ -32,6 +32,7 @@ typedef struct SMnodeMgmt { SSingleWorker readWorker; SSingleWorker writeWorker; SSingleWorker syncWorker; + SSingleWorker monitorWorker; SReplica replicas[TSDB_MAX_REPLICA]; int8_t replica; int8_t selfIndex; @@ -51,6 +52,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); @@ -59,6 +61,7 @@ int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); diff --git a/source/dnode/mgmt/inc/qmInt.h b/source/dnode/mgmt/inc/qmInt.h index 02905413ab..012869d637 100644 --- a/source/dnode/mgmt/inc/qmInt.h +++ b/source/dnode/mgmt/inc/qmInt.h @@ -30,6 +30,7 @@ typedef struct SQnodeMgmt { const char *path; SSingleWorker queryWorker; SSingleWorker fetchWorker; + SSingleWorker monitorWorker; } SQnodeMgmt; // qmInt.c @@ -40,6 +41,7 @@ int32_t qmDrop(SMgmtWrapper *pWrapper); void qmInitMsgHandle(SMgmtWrapper *pWrapper); int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); // qmWorker.c int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); @@ -50,6 +52,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt); void qmStopWorker(SQnodeMgmt *pMgmt); int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/smInt.h b/source/dnode/mgmt/inc/smInt.h index 285ec84942..039dea2491 100644 --- a/source/dnode/mgmt/inc/smInt.h +++ b/source/dnode/mgmt/inc/smInt.h @@ -32,6 +32,7 @@ typedef struct SSnodeMgmt { int8_t uniqueWorkerInUse; SArray *uniqueWorkers; // SArray SSingleWorker sharedWorker; + SSingleWorker monitorWorker; } SSnodeMgmt; // smInt.c @@ -42,6 +43,7 @@ int32_t smDrop(SMgmtWrapper *pWrapper); void smInitMsgHandle(SMgmtWrapper *pWrapper); int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); // smWorker.c int32_t smStartWorker(SSnodeMgmt *pMgmt); @@ -50,6 +52,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/inc/vmInt.h b/source/dnode/mgmt/inc/vmInt.h index 889ad7c164..f8466fe4f2 100644 --- a/source/dnode/mgmt/inc/vmInt.h +++ b/source/dnode/mgmt/inc/vmInt.h @@ -28,6 +28,7 @@ typedef struct SVnodesMgmt { SHashObj *hash; SRWLatch latch; SVnodesStat state; + SVnodesStat lastState; STfs *pTfs; SQWorkerPool queryPool; SQWorkerPool fetchPool; @@ -38,6 +39,7 @@ typedef struct SVnodesMgmt { SDnode *pDnode; SMgmtWrapper *pWrapper; SSingleWorker mgmtWorker; + SSingleWorker monitorWorker; } SVnodesMgmt; typedef struct { @@ -91,6 +93,8 @@ int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq); +int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); +int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq); // vmFile.c int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); @@ -114,6 +118,7 @@ int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrappert, SNodeMsg *pMsg); +int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/dndTransport.c b/source/dnode/mgmt/main/dndTransport.c index e76633bb1f..3b0aca4b39 100644 --- a/source/dnode/mgmt/main/dndTransport.c +++ b/source/dnode/mgmt/main/dndTransport.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - + #define _DEFAULT_SOURCE #include "dndInt.h" @@ -482,4 +482,8 @@ SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { .parent = pWrapper, .name = pWrapper->name}; return cfg; +} + +void dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) { + rpcSendRecv(pDnode->trans.clientRpc, pEpSet, pReq, pRsp); } \ No newline at end of file diff --git a/source/dnode/mgmt/mm/mmHandle.c b/source/dnode/mgmt/mm/mmHandle.c index e5495c7e1d..6a4a56de43 100644 --- a/source/dnode/mgmt/mm/mmHandle.c +++ b/source/dnode/mgmt/mm/mmHandle.c @@ -16,6 +16,36 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant); +} + +int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonMmInfo mmInfo = {0}; + mmGetMonitorInfo(pWrapper, &mmInfo); + dmGetMonitorSysInfo(&mmInfo.sys); + monGetLogs(&mmInfo.log); + + int32_t rspLen = tSerializeSMonMmInfo(NULL, 0, &mmInfo); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonMmInfo(pRsp, rspLen, &mmInfo); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonMmInfo(&mmInfo); + return 0; +} + int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnode *pDnode = pWrapper->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; @@ -74,6 +104,8 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { } void mmInitMsgHandle(SMgmtWrapper *pWrapper) { + dndSetMsgHandle(pWrapper, TDMT_MON_MM_INFO, mmProcessMonitorMsg, DEFAULT_HANDLE); + // Requests handled by DNODE dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); @@ -147,6 +179,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE); // Requests handled by VNODE dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); @@ -163,5 +196,4 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); - } diff --git a/source/dnode/mgmt/mm/mmInt.c b/source/dnode/mgmt/mm/mmInt.c index cd6ea1499f..8d4ac80e72 100644 --- a/source/dnode/mgmt/mm/mmInt.c +++ b/source/dnode/mgmt/mm/mmInt.c @@ -241,8 +241,3 @@ void mmSetMgmtFp(SMgmtWrapper *pWrapper) { pWrapper->fp = mgmtFp; } -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo); -} diff --git a/source/dnode/mgmt/mm/mmWorker.c b/source/dnode/mgmt/mm/mmWorker.c index 735ef53b37..44bac61906 100644 --- a/source/dnode/mgmt/mm/mmWorker.c +++ b/source/dnode/mgmt/mm/mmWorker.c @@ -23,19 +23,18 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; - if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) { + if (pMsg->rpcMsg.msgType == TDMT_DND_ALTER_MNODE) { + code = mmProcessAlterReq(pMgmt, pMsg); + } else if (pMsg->rpcMsg.msgType == TDMT_MON_MM_INFO) { + code = mmProcessGetMonMmInfoReq(pMgmt->pWrapper, pMsg); + } else { pMsg->pNode = pMgmt->pMnode; code = mndProcessMsg(pMsg); - } else { - code = mmProcessAlterReq(pMgmt, pMsg); } if (pRpc->msgType & 1U) { if (pRpc->handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - if (code != 0) { - code = terrno; - dError("msg:%p, failed to process since %s", pMsg, terrstr()); - } + if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; tmsgSendRsp(&rsp); } @@ -98,6 +97,15 @@ int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } +int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->monitorWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); if (pMsg == NULL) return -1; @@ -157,15 +165,24 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg sCfg = {.min = 1, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt}; if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { - dError("failed to start mnode sync-worker since %s", terrstr()); + dError("failed to start mnode mnode-sync worker since %s", terrstr()); return -1; } + if (tsMultiProcess) { + SSingleWorkerCfg mCfg = {.min = 1, .max = 1, .name = "mnode-monitor", .fp = (FItem)mmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start mnode mnode-monitor worker since %s", terrstr()); + return -1; + } + } + dDebug("mnode workers are initialized"); return 0; } void mmStopWorker(SMnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); diff --git a/source/dnode/mgmt/qm/qmHandle.c b/source/dnode/mgmt/qm/qmHandle.c index 77a9db1175..11b4ff0622 100644 --- a/source/dnode/mgmt/qm/qmHandle.c +++ b/source/dnode/mgmt/qm/qmHandle.c @@ -16,6 +16,34 @@ #define _DEFAULT_SOURCE #include "qmInt.h" +void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo) { +} + +int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonQmInfo qmInfo = {0}; + qmGetMonitorInfo(pWrapper, &qmInfo); + dmGetMonitorSysInfo(&qmInfo.sys); + monGetLogs(&qmInfo.log); + + int32_t rspLen = tSerializeSMonQmInfo(NULL, 0, &qmInfo); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonQmInfo(pRsp, rspLen, &qmInfo); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonQmInfo(&qmInfo); + return 0; +} + int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnode *pDnode = pWrapper->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; @@ -55,6 +83,8 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } void qmInitMsgHandle(SMgmtWrapper *pWrapper) { + dndSetMsgHandle(pWrapper, TDMT_MON_QM_INFO, qmProcessMonitorMsg, DEFAULT_HANDLE); + // Requests handled by VNODE dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); diff --git a/source/dnode/mgmt/qm/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c index db0752949d..6c8382aef9 100644 --- a/source/dnode/mgmt/qm/qmWorker.c +++ b/source/dnode/mgmt/qm/qmWorker.c @@ -16,11 +16,36 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { - SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; +static inline void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, + .ahandle = pMsg->rpcMsg.ahandle, + .code = code, + .pCont = pMsg->pRsp, + .contLen = pMsg->rspLen}; tmsgSendRsp(&rsp); } +static void qmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SQnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, get from qnode monitor queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) { + code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg); + } + + if (pRpc->msgType & 1U) { + if (code != 0 && terrno != 0) code = terrno; + qmSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SQnodeMgmt *pMgmt = pInfo->ahandle; @@ -66,6 +91,15 @@ int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } +int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->monitorWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); if (pMsg == NULL) { @@ -128,11 +162,21 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { return -1; } + if (tsMultiProcess) { + SSingleWorkerCfg mCfg = { + .min = 1, .max = 1, .name = "qnode-monitor", .fp = (FItem)qmProcessMonQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start qnode-monitor worker since %s", terrstr()); + return -1; + } + } + dDebug("qnode workers are initialized"); return 0; } void qmStopWorker(SQnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->queryWorker); tSingleWorkerCleanup(&pMgmt->fetchWorker); dDebug("qnode workers are closed"); diff --git a/source/dnode/mgmt/sm/smHandle.c b/source/dnode/mgmt/sm/smHandle.c index a1fa41a686..fed6b6528c 100644 --- a/source/dnode/mgmt/sm/smHandle.c +++ b/source/dnode/mgmt/sm/smHandle.c @@ -16,6 +16,33 @@ #define _DEFAULT_SOURCE #include "smInt.h" +void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo) {} + +int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonSmInfo smInfo = {0}; + smGetMonitorInfo(pWrapper, &smInfo); + dmGetMonitorSysInfo(&smInfo.sys); + monGetLogs(&smInfo.log); + + int32_t rspLen = tSerializeSMonSmInfo(NULL, 0, &smInfo); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonSmInfo(pRsp, rspLen, &smInfo); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonSmInfo(&smInfo); + return 0; +} + int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnode *pDnode = pWrapper->pDnode; SRpcMsg *pReq = &pMsg->rpcMsg; @@ -55,6 +82,8 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } void smInitMsgHandle(SMgmtWrapper *pWrapper) { + dndSetMsgHandle(pWrapper, TDMT_MON_SM_INFO, smProcessMonitorMsg, DEFAULT_HANDLE); + // Requests handled by SNODE dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/sm/smWorker.c b/source/dnode/mgmt/sm/smWorker.c index afa843953b..a29d5d1abc 100644 --- a/source/dnode/mgmt/sm/smWorker.c +++ b/source/dnode/mgmt/sm/smWorker.c @@ -16,6 +16,36 @@ #define _DEFAULT_SOURCE #include "smInt.h" +static inline void smSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, + .ahandle = pMsg->rpcMsg.ahandle, + .code = code, + .pCont = pMsg->pRsp, + .contLen = pMsg->rspLen}; + tmsgSendRsp(&rsp); +} + +static void smProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { + SSnodeMgmt *pMgmt = pInfo->ahandle; + + dTrace("msg:%p, get from snode monitor queue", pMsg); + SRpcMsg *pRpc = &pMsg->rpcMsg; + int32_t code = -1; + + if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) { + code = smProcessGetMonSmInfoReq(pMgmt->pWrapper, pMsg); + } + + if (pRpc->msgType & 1U) { + if (code != 0 && terrno != 0) code = terrno; + smSendRsp(pMgmt->pWrapper, pMsg, code); + } + + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pRpc->pCont); + taosFreeQitem(pMsg); +} + static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SSnodeMgmt *pMgmt = pInfo->ahandle; @@ -80,11 +110,21 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) { return -1; } + if (tsMultiProcess) { + SSingleWorkerCfg mCfg = { + .min = 1, .max = 1, .name = "snode-monitor", .fp = (FItem)smProcessMonitorQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start snode-monitor worker since %s", terrstr()); + return -1; + } + } + dDebug("snode workers are initialized"); return 0; } void smStopWorker(SSnodeMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->monitorWorker); for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) { SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i); tMultiWorkerCleanup(pWorker); @@ -120,6 +160,15 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } +int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SSnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->monitorWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SSnodeMgmt *pMgmt = pWrapper->pMgmt; int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); diff --git a/source/dnode/mgmt/vm/vmHandle.c b/source/dnode/mgmt/vm/vmHandle.c index 52af9cac60..ecad414fd6 100644 --- a/source/dnode/mgmt/vm/vmHandle.c +++ b/source/dnode/mgmt/vm/vmHandle.c @@ -16,6 +16,71 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + tfsGetMonitorInfo(pMgmt->pTfs, &vmInfo->tfs); + + taosWLockLatch(&pMgmt->latch); + vmInfo->vstat.totalVnodes = pMgmt->state.totalVnodes; + vmInfo->vstat.masterNum = pMgmt->state.masterNum; + vmInfo->vstat.numOfSelectReqs = pMgmt->state.numOfSelectReqs - pMgmt->lastState.numOfSelectReqs; + vmInfo->vstat.numOfInsertReqs = pMgmt->state.numOfInsertReqs - pMgmt->lastState.numOfInsertReqs; + vmInfo->vstat.numOfInsertSuccessReqs = pMgmt->state.numOfInsertSuccessReqs - pMgmt->lastState.numOfInsertSuccessReqs; + vmInfo->vstat.numOfBatchInsertReqs = pMgmt->state.numOfBatchInsertReqs - pMgmt->lastState.numOfBatchInsertReqs; + vmInfo->vstat.numOfBatchInsertSuccessReqs = + pMgmt->state.numOfBatchInsertSuccessReqs - pMgmt->lastState.numOfBatchInsertSuccessReqs; + pMgmt->lastState = pMgmt->state; + taosWUnLockLatch(&pMgmt->latch); +} + +int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonVmInfo vmInfo = {0}; + vmGetMonitorInfo(pWrapper, &vmInfo); + dmGetMonitorSysInfo(&vmInfo.sys); + monGetLogs(&vmInfo.log); + + int32_t rspLen = tSerializeSMonVmInfo(NULL, 0, &vmInfo); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonVmInfo(pRsp, rspLen, &vmInfo); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonVmInfo(&vmInfo); + return 0; +} + +int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { + SMonVloadInfo vloads = {0}; + vmGetVnodeLoads(pWrapper, &vloads); + + int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSMonVloadInfo(pRsp, rspLen, &vloads); + pReq->pRsp = pRsp; + pReq->rspLen = rspLen; + tFreeSMonVloadInfo(&vloads); + return 0; +} + static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; @@ -239,6 +304,9 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { } void vmInitMsgHandle(SMgmtWrapper *pWrapper) { + dndSetMsgHandle(pWrapper, TDMT_MON_VM_INFO, vmProcessMonitorMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MON_VM_LOAD, vmProcessMonitorMsg, DEFAULT_HANDLE); + // Requests handled by VNODE dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/vm/vmInt.c b/source/dnode/mgmt/vm/vmInt.c index b3390ba31c..6a1a5c3987 100644 --- a/source/dnode/mgmt/vm/vmInt.c +++ b/source/dnode/mgmt/vm/vmInt.c @@ -344,38 +344,21 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper) { pWrapper->fp = mgmtFp; } -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return -1; - - return tfsGetMonitorInfo(pMgmt->pTfs, pInfo); -} - -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; - if (pMgmt == NULL) return; - - SVnodesStat *pStat = &pMgmt->state; - pInfo->req_select = pStat->numOfSelectReqs; - pInfo->req_insert = pStat->numOfInsertReqs; - pInfo->req_insert_success = pStat->numOfInsertSuccessReqs; - pInfo->req_insert_batch = pStat->numOfBatchInsertReqs; - pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs; - pInfo->errors = tsNumOfErrorLogs; - pInfo->vnodes_num = pStat->totalVnodes; - pInfo->masters = pStat->masterNum; -} - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { +void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; SVnodesStat *pStat = &pMgmt->state; - int32_t totalVnodes = 0; - int32_t masterNum = 0; - int64_t numOfSelectReqs = 0; - int64_t numOfInsertReqs = 0; - int64_t numOfInsertSuccessReqs = 0; - int64_t numOfBatchInsertReqs = 0; - int64_t numOfBatchInsertSuccessReqs = 0; + SArray *pLoads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad)); + + int32_t totalVnodes = 0; + int32_t masterNum = 0; + int64_t numOfSelectReqs = 0; + int64_t numOfInsertReqs = 0; + int64_t numOfInsertSuccessReqs = 0; + int64_t numOfBatchInsertReqs = 0; + int64_t numOfBatchInsertSuccessReqs = 0; + + pInfo->pVloads = pLoads; + if (pLoads == NULL) return; taosRLockLatch(&pMgmt->latch); @@ -402,6 +385,7 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { taosRUnLockLatch(&pMgmt->latch); + taosWLockLatch(&pMgmt->latch); pStat->totalVnodes = totalVnodes; pStat->masterNum = masterNum; pStat->numOfSelectReqs = numOfSelectReqs; @@ -409,4 +393,5 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) { pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs; pStat->numOfBatchInsertReqs = numOfBatchInsertReqs; pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs; + taosWUnLockLatch(&pMgmt->latch); } \ No newline at end of file diff --git a/source/dnode/mgmt/vm/vmWorker.c b/source/dnode/mgmt/vm/vmWorker.c index ed1a4ca2f4..ff92cf880b 100644 --- a/source/dnode/mgmt/vm/vmWorker.c +++ b/source/dnode/mgmt/vm/vmWorker.c @@ -16,8 +16,12 @@ #define _DEFAULT_SOURCE #include "vmInt.h" -static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { - SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; +static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, + .ahandle = pMsg->rpcMsg.ahandle, + .code = code, + .pCont = pMsg->pRsp, + .contLen = pMsg->rspLen}; tmsgSendRsp(&rsp); } @@ -26,9 +30,15 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { int32_t code = -1; tmsg_t msgType = pMsg->rpcMsg.msgType; - dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg); + dTrace("msg:%p, will be processed in vnode-m queue", pMsg); switch (msgType) { + case TDMT_MON_VM_INFO: + code = vmProcessGetMonVmInfoReq(pMgmt->pWrapper, pMsg); + break; + case TDMT_MON_VM_LOAD: + code = vmProcessGetVnodeLoadsReq(pMgmt->pWrapper, pMsg); + break; case TDMT_DND_CREATE_VNODE: code = vmProcessCreateVnodeReq(pMgmt, pMsg); break; @@ -255,6 +265,15 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return 0; } +int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->monitorWorker; + + dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; SMsgHead *pHead = pRpc->pCont; @@ -412,11 +431,21 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { return -1; } + if (tsMultiProcess) { + SSingleWorkerCfg mCfg = { + .min = 1, .max = 1, .name = "vnode-monitor", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { + dError("failed to start mnode vnode-monitor worker since %s", terrstr()); + return -1; + } + } + dDebug("vnode workers are initialized"); return 0; } void vmStopWorker(SVnodesMgmt *pMgmt) { + tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker); tQWorkerCleanup(&pMgmt->fetchPool); tQWorkerCleanup(&pMgmt->queryPool); diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..91c6e24e28 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index ff2ea162fb..fd739dc657 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -22,6 +22,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndSma.h" #define TSDB_DB_VER_NUMBER 1 #define TSDB_DB_RESERVE_SIZE 64 @@ -40,7 +41,8 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); -static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); +static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); +static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_DB, @@ -58,6 +60,7 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); @@ -1661,3 +1664,50 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + //TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } + + return code; +} + diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 314e70db9b..22b1bd51e7 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * tsStatusInterval) { + if (interval > 30000 * tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -630,6 +630,7 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) { static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) { mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); + return TSDB_CODE_SUCCESS; } static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index aadd914439..71ee0baaae 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -18,6 +18,7 @@ #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) //!!!! Note: only APPEND columns in below tables, NO insert !!!! static const SInfosTableSchema dnodesSchema[] = { @@ -79,11 +80,11 @@ static const SInfosTableSchema userFuncSchema[] = { {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, }; static const SInfosTableSchema userIdxSchema[] = { - {.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, }; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index b402a27a41..965ed792e7 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -688,6 +688,39 @@ _OVER: return code; } +int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { + int32_t code = -1; + SSmaObj *pSma = NULL; + + pSma = mndAcquireSma(pMnode, indexReq->indexFName); + if (pSma == NULL) { + *exist = false; + return 0; + } + + memcpy(rsp->dbFName, pSma->db, sizeof(pSma->db)); + memcpy(rsp->tblFName, pSma->stb, sizeof(pSma->stb)); + strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA); + + SNodeList *pList = NULL; + int32_t extOffset = 0; + code = nodesStringToList(pSma->expr, &pList); + if (0 == code) { + SNode *node = NULL; + FOREACH(node, pList) { + SFunctionNode *pFunc = (SFunctionNode *)node; + extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? ",":""), pFunc->functionName); + } + + *exist = true; + } + + mndReleaseSma(pMnode, pSma); + + return code; +} + + static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5c3dd778e1..86bbc0257b 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -502,7 +502,11 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr SMonVgroupDesc desc = {0}; desc.vgroup_id = pVgroup->vgId; - strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name)); + + SName name = {0}; + tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tNameGetDbName(&name, desc.database_name); + desc.tables_num = pVgroup->numOfTables; pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries; tstrncpy(desc.status, "unsynced", sizeof(desc.status)); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 4ee85e61f1..902a2d8d35 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -346,6 +346,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { } taosHashCleanup(pSmaStat->smaStatItems); } + return TSDB_CODE_SUCCESS; } static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { @@ -450,8 +451,10 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t winSKey); return TSDB_CODE_FAILED; } + tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", REPO_ID(pTsdb), indexUid, winSKey); + return TSDB_CODE_SUCCESS; } /** @@ -1099,6 +1102,7 @@ static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { } // clear sma data files // TODO: + return TSDB_CODE_SUCCESS; } static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) { @@ -1221,6 +1225,7 @@ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interv pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit); pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel); + return TSDB_CODE_SUCCESS; } /** @@ -1240,6 +1245,7 @@ static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) pSmaH->dFile.path = strdup(tSmaFile); pSmaH->smaFsIter.iter = 0; pSmaH->smaFsIter.fid = fid; + return TSDB_CODE_SUCCESS; } /** @@ -1431,6 +1437,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { tdDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; } int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { @@ -1460,6 +1467,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { tsdbTSmaSub(pTsdb, 1); // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; } #if 0 diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6ed09ce7eb..2aa858fe06 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -606,6 +606,43 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } +int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *indexName, SIndexInfo *out) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get index from mnode, indexName:%s", indexName); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)]((void *)indexName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_INDEX, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)](out, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get index rsp failed, code:%x, indexName:%s", code, indexName); + CTG_ERR_RET(code); + } + + ctgDebug("Got index from mnode, indexName:%s", indexName); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -1776,7 +1813,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - + bool inCache = false; int32_t code = 0; uint64_t dbId = 0; @@ -2445,7 +2482,7 @@ _return: } int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { - + return 0; } int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { @@ -2512,7 +2549,7 @@ _return: } int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) { - + return 0; } int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { @@ -2764,6 +2801,17 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); } +int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); +} + + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 3faf208c0e..8d88b94d6c 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -120,9 +120,10 @@ static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - ASSERT(0); -// int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); -// return ((char *)page->data) + rowOffset + offset * numOfRows; + + // int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); + return ((char *)page->data); + } static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffset, int32_t offset) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 35a4b93a88..6a1ded6441 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -643,6 +643,7 @@ typedef struct SSortOperatorInfo { SSDataBlock *pDataBlock; SArray* pSortInfo; SSortHandle *pSortHandle; + SArray* inputSlotMap; // for index map from table scan output int32_t bufPageSize; int32_t numOfRowsInRes; @@ -679,8 +680,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); + SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo); -SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 8971ee33d3..c584df05dd 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -34,22 +34,22 @@ typedef struct SMultiMergeSource { SSDataBlock *pBlock; } SMultiMergeSource; -typedef struct SExternalMemSource { +typedef struct SSortSource { SMultiMergeSource src; - SArray* pageIdList; - int32_t pageIndex; -} SExternalMemSource; + union{ + struct{ + SArray* pageIdList; + int32_t pageIndex; + }; + void *param; + }; -typedef struct SGenericSource { - SMultiMergeSource src; - void *param; -} SGenericSource; +} SSortSource; typedef struct SMsortComparParam { void **pSources; int32_t numOfSources; SArray *orderInfo; // SArray - bool nullFirst; } SMsortComparParam; typedef struct SSortHandle SSortHandle; @@ -63,7 +63,7 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* * @param type * @return */ -SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr); +SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, SArray* pIndexMap, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr); /** * diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8a38b9ab48..bd96ab9e56 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -194,6 +194,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { } taosCloseQueue(pDispatcher->pDataBlocks); taosThreadMutexDestroy(&pDispatcher->mutex); + return TSDB_CODE_SUCCESS; } int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e85cfb76fd..9270b0da4e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3297,6 +3297,7 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { int8_t* rowRes = NULL; bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols); + filterFreeInfo(filter); SSDataBlock* px = createOneDataBlock(pBlock); blockDataEnsureCapacity(px, pBlock->info.rows); @@ -3306,19 +3307,25 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i); + if (keep) { + colDataAssign(pDst, pSrc, pBlock->info.rows); + numOfRow = pBlock->info.rows; + } else if (NULL != rowRes) { + numOfRow = 0; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } - numOfRow = 0; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; + if (colDataIsNull_s(pSrc, j)) { + colDataAppendNULL(pDst, numOfRow); + } else { + colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); + } + numOfRow += 1; } - - if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDst, numOfRow); - } else { - colDataAppend(pDst, numOfRow, colDataGetData(pSrc, j), false); - } - numOfRow += 1; + } else { + numOfRow = 0; } *pSrc = *pDst; @@ -4154,6 +4161,7 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) pSourceDataInfo->status = EX_SOURCE_DATA_READY; tsem_post(&pSourceDataInfo->pEx->ready); + return TSDB_CODE_SUCCESS; } static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { @@ -4753,6 +4761,8 @@ static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHan SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { blockDataCleanup(pDataBlock); + blockDataEnsureCapacity(pDataBlock, capacity); + while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { @@ -4954,13 +4964,13 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, "GET_TASKID(pTaskInfo)"); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; tsortAddSource(pInfo->pSortHandle, ps); } @@ -5101,16 +5111,17 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, pInfo->pDataBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - SGenericSource* ps = taosMemoryCalloc(1, sizeof(SGenericSource)); + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[0]; tsortAddSource(pInfo->pSortHandle, ps); int32_t code = tsortOpen(pInfo->pSortHandle); + taosMemoryFreeClear(ps); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, terrno); } @@ -5119,18 +5130,25 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); } -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; + int32_t rowSize = pResBlock->info.rowSize; + + if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } - pInfo->sortBufSize = 1024 * 16; // TODO dynamic set the available sort buffer - pInfo->bufPageSize = 1024; - pInfo->numOfRowsInRes = 1024; - pInfo->pDataBlock = pResBlock; - pInfo->pSortInfo = pSortInfo; + pInfo->bufPageSize = rowSize < 1024 ? 1024*2 : rowSize*2; // there are headers, so pageSize = rowSize + header + + pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + pInfo->numOfRowsInRes = 1024; + pInfo->pDataBlock = pResBlock; + pInfo->pSortInfo = pSortInfo; + pInfo->inputSlotMap = pIndexMap; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; @@ -6156,6 +6174,7 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf pBasicInfo->capacity = numOfRows; doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, pkey); + return TSDB_CODE_SUCCESS; } static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { @@ -6289,6 +6308,7 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->inputSlotMap); } void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { @@ -6841,6 +6861,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { return (pRes->info.rows == 0)? NULL:pInfo->pRes; #endif + return TSDB_CODE_SUCCESS; } SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { @@ -7090,7 +7111,9 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); -static SArray* createSortInfo(SNodeList* pNodeList); + +static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); +static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, @@ -7189,9 +7212,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - - SArray* info = createSortInfo(pSortPhyNode->pSortKeys); - return createSortOperatorInfo(op, pResBlock, info, pTaskInfo); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets); + SArray* slotMap = createIndexMap(pSortPhyNode->pTargets); + return createSortOperatorInfo(op, pResBlock, info, slotMap, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -7220,6 +7243,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createMultiTableAggOperatorInfo(op, pPhyNode->pTargets, pTaskInfo, pTableGroupInfo); } }*/ + return NULL; } static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STableGroupInfo* pGroupInfo, @@ -7331,7 +7355,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } -SArray* createSortInfo(SNodeList* pNodeList) { +SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { @@ -7340,19 +7364,53 @@ SArray* createSortInfo(SNodeList* pNodeList) { } for (int32_t i = 0; i < numOfCols; ++i) { - SOrderByExprNode* pSortKey = (SOrderByExprNode*)nodesListGetNode(pNodeList, i); + + SOrderByExprNode* pSortKey = (SOrderByExprNode*)nodesListGetNode(pNodeList, i); SBlockOrderInfo bi = {0}; bi.order = (pSortKey->order == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; - bi.slotId = pColNode->slotId; + + bool found = false; + for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) { + STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); + + SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; + if(pColNode->slotId == pColNodeT->slotId){ // to find slotId in PhysiSort OutputDataBlockDesc + bi.slotId = pTarget->slotId; + found = true; + break; + } + } + + if(!found){ + qError("sort slot id does not found"); + } taosArrayPush(pList, &bi); } return pList; } +SArray* createIndexMap(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(int32_t)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return pList; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeList, i); + + SColumnNode* pColNode = (SColumnNode*)pTarget->pExpr; + taosArrayPush(pList, &pColNode->slotId); + } + + return pList; +} + SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo)); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c67f833699..9eb37d7667 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -581,6 +581,7 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t } tsem_post(&pScanResInfo->ready); + return TSDB_CODE_SUCCESS; } static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { @@ -596,6 +597,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { int8_t* rowRes = NULL; bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); + filterFreeInfo(filter); SSDataBlock* px = createOneDataBlock(pInfo->pRes); blockDataEnsureCapacity(px, pInfo->pRes->info.rows); @@ -606,14 +608,21 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); - numOfRow = 0; - for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; + if (keep) { + colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); + numOfRow = pInfo->pRes->info.rows; + } else if (NULL != rowRes) { + numOfRow = 0; + for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { + if (rowRes[j] == 0) { + continue; + } + + colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); + numOfRow += 1; } - - colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); - numOfRow += 1; + } else { + numOfRow = 0; } } diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 1b55467c85..21fd54b620 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -409,6 +409,7 @@ char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) { int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen) { // todo + return TSDB_CODE_SUCCESS; } void tHashPrint(const SLHashObj* pHashObj, int32_t type) { diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 5c10216a96..fe370d93b2 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -258,6 +258,7 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) { // todo + return TSDB_CODE_SUCCESS; } void tSimpleHashClear(SSHashObj *pHashObj) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 5dfda92982..885112d440 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -22,6 +22,7 @@ #include "tpagedbuf.h" #include "tsort.h" #include "tutil.h" +#include "tcompare.h" struct STupleHandle { SSDataBlock* pBlock; @@ -36,6 +37,7 @@ struct SSortHandle { SDiskbasedBuf *pBuf; SArray *pSortInfo; + SArray *pIndexMap; SArray *pOrderedSource; _sort_fetch_block_fn_t fetchfp; @@ -89,13 +91,14 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { * @param type * @return */ -SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr) { +SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr) { SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle)); pSortHandle->type = type; pSortHandle->pageSize = pageSize; pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = pSortInfo; + pSortHandle->pIndexMap = pIndexMap; pSortHandle->pDataBlock = createOneDataBlock(pBlock); pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); @@ -110,6 +113,17 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page return pSortHandle; } +static int32_t sortComparClearup(SMsortComparParam* cmpParam) { + for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SSortSource* pSource = cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE + blockDataDestroy(pSource->src.pBlock); + taosMemoryFreeClear(pSource); + } + + cmpParam->numOfSources = 0; + return TSDB_CODE_SUCCESS; +} + void tsortDestroySortHandle(SSortHandle* pSortHandle) { tsortClose(pSortHandle); if (pSortHandle->pMergeTree != NULL) { @@ -118,15 +132,23 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { destroyDiskbasedBuf(pSortHandle->pBuf); taosMemoryFreeClear(pSortHandle->idStr); + blockDataDestroy(pSortHandle->pDataBlock); + for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++){ + SSortSource** pSource = taosArrayGet(pSortHandle->pOrderedSource, i); + blockDataDestroy((*pSource)->src.pBlock); + taosMemoryFreeClear(*pSource); + } + taosArrayDestroy(pSortHandle->pOrderedSource); taosMemoryFreeClear(pSortHandle); } int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { taosArrayPush(pSortHandle->pOrderedSource, &pSource); + return TSDB_CODE_SUCCESS; } static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { - SExternalMemSource* pSource = taosMemoryCalloc(1, sizeof(SExternalMemSource)); + SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); if (pSource == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -139,8 +161,8 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource (*sourceId) += 1; int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); - int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize; - + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize; // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation + ASSERT(numOfRows > 0); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -148,7 +170,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t start = 0; if (pHandle->pBuf == NULL) { - int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", "/tmp"); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -164,15 +186,16 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { } int32_t pageId = -1; - SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + void* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); if (pPage == NULL) { + blockDataDestroy(p); return terrno; } int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); - blockDataToBuf(pPage->data, p); + blockDataToBuf(pPage, p); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -184,10 +207,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { blockDataCleanup(pDataBlock); SSDataBlock* pBlock = createOneDataBlock(pDataBlock); - int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); } static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { @@ -198,11 +218,11 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int if (pHandle->type == SORT_SINGLESOURCE_SORT) { for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; + SSortSource* pSource = cmpParam->pSources[i]; SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); - SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); - code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -212,7 +232,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } else { // multi-pass internal merge sort is required if (pHandle->pBuf == NULL) { - code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", "/tmp"); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -220,7 +240,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SGenericSource* pSource = cmpParam->pSources[i]; + SSortSource* pSource = cmpParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); } } @@ -228,16 +248,6 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int return code; } -static int32_t sortComparClearup(SMsortComparParam* cmpParam) { - for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; - blockDataDestroy(pSource->src.pBlock); - taosMemoryFreeClear(pSource); - } - - cmpParam->numOfSources = 0; -} - static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -257,7 +267,7 @@ static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSou *rowIndex += 1; } -static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SSortHandle *pHandle, int32_t* numOfCompleted) { +static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeTreeInfo *pTree, SSortHandle *pHandle, int32_t* numOfCompleted) { /* * load a new SDataBlock into memory of a given intermediate data-set source, * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree @@ -266,6 +276,7 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa pSource->src.rowIndex = 0; if (pHandle->type == SORT_SINGLESOURCE_SORT) { + pSource->pageIndex ++; if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { (*numOfCompleted) += 1; pSource->src.rowIndex = -1; @@ -274,8 +285,8 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa } else { SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); - SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); - int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -283,7 +294,7 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa releaseBufPage(pHandle->pBuf, pPage); } } else { - pSource->src.pBlock = pHandle->fetchfp(((SGenericSource*)pSource)->param); + pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param); if (pSource->src.pBlock == NULL) { (*numOfCompleted) += 1; pSource->src.rowIndex = -1; @@ -308,9 +319,10 @@ static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwa printf("\nafter adjust:\t"); tMergeTreePrint(pTree); #endif + return TSDB_CODE_SUCCESS; } -static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { +static SSDataBlock* getSortedBlockDataInner(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { blockDataCleanup(pHandle->pDataBlock); while(1) { @@ -320,7 +332,7 @@ static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); - SExternalMemSource *pSource = (*cmpParam).pSources[index]; + SSortSource *pSource = (*cmpParam).pSources[index]; appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex); int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); @@ -345,8 +357,8 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { SArray *pInfo = pParam->orderInfo; - SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; - SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; + SSortSource* pLeftSource = pParam->pSources[pLeftIdx]; + SSortSource* pRightSource = pParam->pSources[pRightIdx]; // this input is exhausted, set the special value to denote this if (pLeftSource->src.rowIndex == -1) { @@ -381,35 +393,26 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { } if (rightNull) { - return pParam->nullFirst? 1:-1; + return pOrder->nullFirst? 1:-1; } if (leftNull) { - return pParam->nullFirst? -1:1; + return pOrder->nullFirst? -1:1; } void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); - switch(pLeftColInfoData->info.type) { - case TSDB_DATA_TYPE_INT: { - int32_t leftv = *(int32_t*)left1; - int32_t rightv = *(int32_t*)right1; + __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); - if (leftv == rightv) { - break; - } else { - if (pOrder->order == TSDB_ORDER_ASC) { - return leftv < rightv? -1 : 1; - } else { - return leftv < rightv? 1 : -1; - } - } - } - default: - assert(0); + int ret = fn(left1, right1); + if (ret == 0) { + continue; + } else { + return ret; } } + return 0; } static int32_t doInternalMergeSort(SSortHandle* pHandle) { @@ -422,7 +425,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages)); pHandle->totalElapsed = taosGetTimestampUs() - pHandle->startTs; - qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, + qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort elapsed:%"PRId64", total elapsed:%"PRId64, pHandle->idStr, (int32_t) (sortPass + 1), getTotalBufSize(pHandle->pBuf), pHandle->sortElapsed, pHandle->totalElapsed); int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize); @@ -459,13 +462,13 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { } while (1) { - SSDataBlock* pDataBlock = getSortedBlockData(pHandle, &pHandle->cmpParam, numOfRows); + SSDataBlock* pDataBlock = getSortedBlockDataInner(pHandle, &pHandle->cmpParam, numOfRows); if (pDataBlock == NULL) { break; } int32_t pageId = -1; - SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + void* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); if (pPage == NULL) { return terrno; } @@ -473,7 +476,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); - blockDataToBuf(pPage->data, pDataBlock); + blockDataToBuf(pPage, pDataBlock); setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); @@ -481,6 +484,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { blockDataCleanup(pDataBlock); } + sortComparClearup(&pHandle->cmpParam); tMergeTreeDestroy(pHandle->pMergeTree); pHandle->numOfCompletedSources = 0; @@ -491,8 +495,6 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { } } - sortComparClearup(&pHandle->cmpParam); - taosArrayClear(pHandle->pOrderedSource); taosArrayAddAll(pHandle->pOrderedSource, pResList); taosArrayDestroy(pResList); @@ -520,9 +522,8 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; if (pHandle->type == SORT_SINGLESOURCE_SORT) { - SGenericSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); + SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); taosArrayClear(pHandle->pOrderedSource); - while (1) { SSDataBlock* pBlock = pHandle->fetchfp(source->param); if (pBlock == NULL) { @@ -533,7 +534,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { pHandle->pDataBlock = createOneDataBlock(pBlock); } - int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); + int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap); if (code != 0) { return code; } @@ -555,8 +556,13 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { size_t size = blockDataGetSize(pHandle->pDataBlock); // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + // All sorted data can fit in memory, external memory sort is not needed. Return to directly if (size <= sortBufSize) { pHandle->cmpParam.numOfSources = 1; @@ -569,8 +575,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { doAddToBuf(pHandle->pDataBlock, pHandle); } } - - taosMemoryFreeClear(source); } return TSDB_CODE_SUCCESS; @@ -612,22 +616,22 @@ int32_t tsortOpen(SSortHandle* pHandle) { return code; } - code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + return tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); } int32_t tsortClose(SSortHandle* pHandle) { // do nothing + return TSDB_CODE_SUCCESS; } int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp) { pHandle->fetchfp = fp; + return TSDB_CODE_SUCCESS; } int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { pHandle->comparFn = fp; + return TSDB_CODE_SUCCESS; } STupleHandle* tsortNextTuple(SSortHandle* pHandle) { @@ -647,7 +651,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { } int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); - SExternalMemSource *pSource = pHandle->cmpParam.pSources[index]; + SSortSource *pSource = pHandle->cmpParam.pSources[index]; if (pHandle->needAdjust) { int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); @@ -677,7 +681,8 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { } bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { - return false; + SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); + return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL); } void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index ecea24135f..efa0581fd0 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -33,14 +33,29 @@ #include "tdef.h" #include "trpc.h" #include "tvariant.h" +#include "tcompare.h" namespace { typedef struct { int32_t startVal; int32_t count; int32_t pageRows; + int16_t type; } _info; +int16_t VARCOUNT = 16; + +float rand_f2() +{ + unsigned r = taosRand(); + r &= 0x007fffff; + r |= 0x40800000; + return *(float*)&r - 6.0; +} + +static const int32_t TEST_NUMBER = 1; +#define bigendian() ((*(char *)&TEST_NUMBER) == 0) + SSDataBlock* getSingleColDummyBlock(void* param) { _info* pInfo = (_info*) param; if (--pInfo->count < 0) { @@ -51,19 +66,65 @@ SSDataBlock* getSingleColDummyBlock(void* param) { pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); + colInfo.info.type = pInfo->type; + if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ + colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; + colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); + } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { + colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; + colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); + } else{ + colInfo.info.bytes = tDataTypes[pInfo->type].bytes; + colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes)); + colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); + } colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); taosArrayPush(pBlock->pDataBlock, &colInfo); for (int32_t i = 0; i < pInfo->pageRows; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); - int32_t v = ++pInfo->startVal; - colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ + int32_t size = taosRand() % VARCOUNT; + char str[128] = {0}; + char strOri[128] = {0}; + taosRandStr(strOri, size); + int32_t len = 0; + bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len); + if (!ret){ + printf("error\n"); + return NULL; + } + varDataSetLen(str, len); + colDataAppend(pColInfo, i, reinterpret_cast(str), false); + pBlock->info.hasVarCol = true; + printf("nchar: %s\n",strOri); + } else if(pInfo->type == TSDB_DATA_TYPE_BINARY){ + int32_t size = taosRand() % VARCOUNT; + char str[64] = {0}; + taosRandStr(varDataVal(str), size); + varDataSetLen(str, size); + colDataAppend(pColInfo, i, reinterpret_cast(str), false); + pBlock->info.hasVarCol = true; + printf("binary: %s\n", varDataVal(str)); + } else if(pInfo->type == TSDB_DATA_TYPE_DOUBLE || pInfo->type == TSDB_DATA_TYPE_FLOAT) { + double v = rand_f2(); + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + printf("float: %f\n", v); + } else{ + int64_t v = ++pInfo->startVal; + char *result = static_cast(taosMemoryCalloc(tDataTypes[pInfo->type].bytes, 1)); + if (!bigendian()){ + memcpy(result, &v, tDataTypes[pInfo->type].bytes); + }else{ + memcpy(result, (char*)(&v) + sizeof(int64_t) - tDataTypes[pInfo->type].bytes, tDataTypes[pInfo->type].bytes); + } + + colDataAppend(pColInfo, i, result, false); + printf("int: %ld\n", v); + taosMemoryFree(result); + } } pBlock->info.rows = pInfo->pageRows; @@ -71,17 +132,18 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return pBlock; } + int32_t docomp(const void* p1, const void* p2, void* param) { int32_t pLeftIdx = *(int32_t *)p1; int32_t pRightIdx = *(int32_t *)p2; SMsortComparParam *pParam = (SMsortComparParam *)param; - SGenericSource** px = reinterpret_cast(pParam->pSources); + SSortSource** px = reinterpret_cast(pParam->pSources); SArray *pInfo = pParam->orderInfo; - SGenericSource* pLeftSource = px[pLeftIdx]; - SGenericSource* pRightSource = px[pRightIdx]; + SSortSource* pLeftSource = px[pLeftIdx]; + SSortSource* pRightSource = px[pRightIdx]; // this input is exhausted, set the special value to denote this if (pLeftSource->src.rowIndex == -1) { @@ -116,33 +178,22 @@ int32_t docomp(const void* p1, const void* p2, void* param) { } if (rightNull) { - return pParam->nullFirst? 1:-1; + return pOrder->nullFirst? 1:-1; } if (leftNull) { - return pParam->nullFirst? -1:1; + return pOrder->nullFirst? -1:1; } void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); - switch(pLeftColInfoData->info.type) { - case TSDB_DATA_TYPE_INT: { - int32_t leftv = *(int32_t*)left1; - int32_t rightv = *(int32_t*)right1; - - if (leftv == rightv) { - break; - } else { - if (pOrder->order == TSDB_ORDER_ASC) { - return leftv < rightv? -1 : 1; - } else { - return leftv < rightv? 1 : -1; - } - } - } - default: - assert(0); + int ret = fn(left1, right1); + if (ret == 0) { + continue; + } else { + return ret; } } @@ -150,71 +201,30 @@ int32_t docomp(const void* p1, const void* p2, void* param) { } } // namespace -#if 0 +#if 1 TEST(testCase, inMem_sort_Test) { - SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); - SOrder o = {.order = TSDB_ORDER_ASC}; - o.col.info.colId = 1; - o.col.info.type = TSDB_DATA_TYPE_INT; - taosArrayPush(pOrderVal, &o); - - int32_t numOfRows = 1000; SBlockOrderInfo oi = {0}; oi.order = TSDB_ORDER_ASC; - oi.colIndex = 0; + oi.slotId = 0; SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; - SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); - tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); - tsortAddSource(phandle, &numOfRows); - - int32_t code = tsortOpen(phandle); - int32_t row = 1; - - while(1) { - STupleHandle* pTupleHandle = tsortNextTuple(phandle); - if (pTupleHandle == NULL) { - break; - } - - void* v = tsortGetValue(pTupleHandle, 0); - printf("%d: %d\n", row++, *(int32_t*) v); - - } - tsortDestroySortHandle(phandle); -} - -TEST(testCase, external_mem_sort_Test) { - SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); - SOrder o = {.order = TSDB_ORDER_ASC}; - o.col.info.colId = 1; - o.col.info.type = TSDB_DATA_TYPE_INT; - taosArrayPush(pOrderVal, &o); - - SBlockOrderInfo oi = {0}; - oi.order = TSDB_ORDER_ASC; - oi.colIndex = 0; - SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - taosArrayPush(orderInfo, &oi); - - SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; - SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_SINGLESOURCE_SORT, 1024, 5, NULL, "test_abc"); tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); _info* pInfo = (_info*) taosMemoryCalloc(1, sizeof(_info)); - pInfo->startVal = 100000; - pInfo->pageRows = 1000; - pInfo->count = 50; + pInfo->startVal = 0; + pInfo->pageRows = 100; + pInfo->count = 6; + pInfo->type = TSDB_DATA_TYPE_USMALLINT; - SGenericSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); + SSortSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SSortSource))); ps->param = pInfo; - tsortAddSource(phandle, ps); int32_t code = tsortOpen(phandle); int32_t row = 1; + taosMemoryFreeClear(ps); while(1) { STupleHandle* pTupleHandle = tsortNextTuple(phandle); @@ -223,40 +233,153 @@ TEST(testCase, external_mem_sort_Test) { } void* v = tsortGetValue(pTupleHandle, 0); - printf("%d: %d\n", row++, *(int32_t*) v); + printf("%d: %d\n", row, *(uint16_t*) v); + ASSERT_EQ(row++, *(uint16_t*) v); } + taosArrayDestroy(orderInfo); tsortDestroySortHandle(phandle); + taosMemoryFree(pInfo); +} + +TEST(testCase, external_mem_sort_Test) { + + _info* pInfo = (_info*) taosMemoryCalloc(8, sizeof(_info)); + pInfo[0].startVal = 0; + pInfo[0].pageRows = 10; + pInfo[0].count = 6; + pInfo[0].type = TSDB_DATA_TYPE_BOOL; + + pInfo[1].startVal = 0; + pInfo[1].pageRows = 10; + pInfo[1].count = 6; + pInfo[1].type = TSDB_DATA_TYPE_TINYINT; + + pInfo[2].startVal = 0; + pInfo[2].pageRows = 100; + pInfo[2].count = 6; + pInfo[2].type = TSDB_DATA_TYPE_USMALLINT; + + pInfo[3].startVal = 0; + pInfo[3].pageRows = 100; + pInfo[3].count = 6; + pInfo[3].type = TSDB_DATA_TYPE_INT; + + pInfo[4].startVal = 0; + pInfo[4].pageRows = 100; + pInfo[4].count = 6; + pInfo[4].type = TSDB_DATA_TYPE_UBIGINT; + + pInfo[5].startVal = 0; + pInfo[5].pageRows = 100; + pInfo[5].count = 6; + pInfo[5].type = TSDB_DATA_TYPE_DOUBLE; + + pInfo[6].startVal = 0; + pInfo[6].pageRows = 50; + pInfo[6].count = 6; + pInfo[6].type = TSDB_DATA_TYPE_NCHAR; + + pInfo[7].startVal = 0; + pInfo[7].pageRows = 100; + pInfo[7].count = 6; + pInfo[7].type = TSDB_DATA_TYPE_BINARY; + + for (int i = 0; i < 8; i++){ + SBlockOrderInfo oi = {0}; + + if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){ + oi.order = TSDB_ORDER_DESC; + }else{ + oi.order = TSDB_ORDER_ASC; + } + + oi.slotId = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_SINGLESOURCE_SORT, 128, 3, NULL, "test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + + SSortSource* ps = static_cast(taosMemoryCalloc(1, sizeof(SSortSource))); + ps->param = &pInfo[i]; + + tsortAddSource(phandle, ps); + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + taosMemoryFreeClear(ps); + + printf("--------start with %s-----------\n", tDataTypes[pInfo[i].type].name); + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + + if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){ + char buf[128] = {0}; + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf); + printf("%d: %s\n", row++, buf); + }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY){ + char buf[128] = {0}; + memcpy(buf, varDataVal(v), varDataLen(v)); + printf("%d: %s\n", row++, buf); + }else if(pInfo[i].type == TSDB_DATA_TYPE_DOUBLE) { + printf("double: %lf\n", *(double*)v); + }else if (pInfo[i].type == TSDB_DATA_TYPE_FLOAT) { + printf("float: %f\n", *(float*)v); + }else{ + int64_t result = 0; + if (!bigendian()){ + memcpy(&result, v, tDataTypes[pInfo[i].type].bytes); + }else{ + memcpy((char*)(&result) + sizeof(int64_t) - tDataTypes[pInfo[i].type].bytes, v, tDataTypes[pInfo[i].type].bytes); + } + printf("%d: %ld\n", row++, result); + } + } + taosArrayDestroy(orderInfo); + tsortDestroySortHandle(phandle); + } + taosMemoryFree(pInfo); } TEST(testCase, ordered_merge_sort_Test) { - SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); - SOrder o = {.order = TSDB_ORDER_ASC}; - o.col.info.colId = 1; - o.col.info.type = TSDB_DATA_TYPE_INT; - taosArrayPush(pOrderVal, &o); - - int32_t numOfRows = 1000; SBlockOrderInfo oi = {0}; oi.order = TSDB_ORDER_ASC; - oi.colIndex = 0; + oi.slotId = 0; SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4}; - SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_MULTISOURCE_MERGE, 1024, 5, &s, 1,"test_abc"); + SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, NULL, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); tsortSetComparFp(phandle, docomp); + SSortSource* p[10] = {0}; + _info c[10] = {0}; for(int32_t i = 0; i < 10; ++i) { - SGenericSource* p = static_cast(taosMemoryCalloc(1, sizeof(SGenericSource))); - _info* c = static_cast<_info*>(taosMemoryCalloc(1, sizeof(_info))); - c->count = 1; - c->pageRows = 1000; - c->startVal = 0; + p[i] = static_cast(taosMemoryCalloc(1, sizeof(SSortSource))); + c[i].count = 1; + c[i].pageRows = 1000; + c[i].startVal = i*1000; + c[i].type = TSDB_DATA_TYPE_INT; - p->param = c; - tsortAddSource(phandle, p); + p[i]->param = &c[i]; + tsortAddSource(phandle, p[i]); } int32_t code = tsortOpen(phandle); @@ -269,10 +392,14 @@ TEST(testCase, ordered_merge_sort_Test) { } void* v = tsortGetValue(pTupleHandle, 0); - printf("%d: %d\n", row++, *(int32_t*) v); + printf("%d: %d\n", row, *(int32_t*) v); + ASSERT_EQ(row++, *(int32_t*) v); } + + taosArrayDestroy(orderInfo); tsortDestroySortHandle(phandle); + blockDataDestroy(pBlock); } #endif diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eafe9e5a33..bad40422c8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -96,6 +96,7 @@ int32_t countFunction(SqlFunctionCtx *pCtx) { *((int64_t *)buf) += numOfElem; SET_VAL(pResInfo, numOfElem, 1); + return TSDB_CODE_SUCCESS; } #define LIST_ADD_N(_res, _col, _start, _rows, _t, numOfElem) \ @@ -166,6 +167,7 @@ int32_t sumFunction(SqlFunctionCtx *pCtx) { // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); + return TSDB_CODE_SUCCESS; } bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { @@ -434,11 +436,13 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int32_t minFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; } int32_t maxFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = doMinMaxHelper(pCtx, 0); SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; } typedef struct STopBotRes { @@ -588,6 +592,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); + return TSDB_CODE_SUCCESS; } void stddevFinalize(SqlFunctionCtx* pCtx) { @@ -714,6 +719,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { } SET_VAL(pResInfo, notNullElems, 1); + return TSDB_CODE_SUCCESS; } // TODO set the correct parameter. @@ -779,6 +785,7 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { } SET_VAL(pResInfo, numOfElems, 1); + return TSDB_CODE_SUCCESS; } int32_t lastFunction(SqlFunctionCtx *pCtx) { @@ -836,6 +843,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { } SET_VAL(pResInfo, numOfElems, 1); + return TSDB_CODE_SUCCESS; } typedef struct SDiffInfo { @@ -1064,6 +1072,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { * 2. current block may be null value */ assert(pCtx->hasNull); + return 0; } else { // for (int t = 0; t < pCtx->tagInfo.numOfTagCols; ++t) { // SqlFunctionCtx* tagCtx = pCtx->tagInfo.pTagCtxList[t]; diff --git a/source/libs/monitor/inc/monInt.h b/source/libs/monitor/inc/monInt.h index 452c38f66b..ae1af4ba62 100644 --- a/source/libs/monitor/inc/monInt.h +++ b/source/libs/monitor/inc/monInt.h @@ -18,43 +18,32 @@ #include "monitor.h" -#include "tarray.h" #include "tjson.h" typedef struct { - int64_t ts; - ELogLevel level; - char content[MON_LOG_LEN]; -} SMonLogItem; - -typedef struct { - int64_t time; - int64_t req_select; - int64_t req_insert; - int64_t req_insert_batch; - int64_t net_in; - int64_t net_out; - int64_t io_read; - int64_t io_write; - int64_t io_read_disk; - int64_t io_write_disk; -} SMonState; - -typedef struct SMonInfo { - int64_t curTime; - SMonState lastState; - SArray *logs; // array of SMonLogItem - SJson *pJson; + int64_t curTime; + int64_t lastTime; + SJson *pJson; + SMonLogs log; + SMonDmInfo dmInfo; + SMonMmInfo mmInfo; + SMonVmInfo vmInfo; + SMonSmInfo smInfo; + SMonQmInfo qmInfo; + SMonBmInfo bmInfo; } SMonInfo; typedef struct { TdThreadMutex lock; - SArray *logs; // array of SMonLogItem - int32_t maxLogs; - const char *server; - uint16_t port; - bool comp; - SMonState state; + SArray *logs; // array of SMonLogItem + SMonCfg cfg; + int64_t lastTime; + SMonDmInfo dmInfo; + SMonMmInfo mmInfo; + SMonVmInfo vmInfo; + SMonSmInfo smInfo; + SMonQmInfo qmInfo; + SMonBmInfo bmInfo; } SMonitor; #ifdef __cplusplus diff --git a/source/libs/monitor/src/monitor.c b/source/libs/monitor/src/monMain.c similarity index 52% rename from source/libs/monitor/src/monitor.c rename to source/libs/monitor/src/monMain.c index b92c08d51c..c90b1f58e8 100644 --- a/source/libs/monitor/src/monitor.c +++ b/source/libs/monitor/src/monMain.c @@ -17,7 +17,6 @@ #include "monInt.h" #include "taoserror.h" #include "thttp.h" -#include "tlog.h" #include "ttime.h" static SMonitor tsMonitor = {0}; @@ -25,7 +24,7 @@ static SMonitor tsMonitor = {0}; void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexLock(&tsMonitor.lock); int32_t size = taosArrayGetSize(tsMonitor.logs); - if (size < tsMonitor.maxLogs) { + if (size < tsMonitor.cfg.maxLogs) { SMonLogItem item = {.ts = ts, .level = level}; SMonLogItem *pItem = taosArrayPush(tsMonitor.logs, &item); if (pItem != NULL) { @@ -35,6 +34,68 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) { taosThreadMutexUnlock(&tsMonitor.lock); } +int32_t monGetLogs(SMonLogs *logs) { + taosThreadMutexLock(&tsMonitor.lock); + logs->logs = taosArrayDup(tsMonitor.logs); + logs->numOfInfoLogs = tsNumOfInfoLogs; + logs->numOfErrorLogs = tsNumOfErrorLogs; + logs->numOfDebugLogs = tsNumOfDebugLogs; + logs->numOfTraceLogs = tsNumOfTraceLogs; + tsNumOfInfoLogs = 0; + tsNumOfErrorLogs = 0; + tsNumOfDebugLogs = 0; + tsNumOfTraceLogs = 0; + taosArrayClear(tsMonitor.logs); + taosThreadMutexUnlock(&tsMonitor.lock); + if (logs->logs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return 0; +} + +void monSetDmInfo(SMonDmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.dmInfo, pInfo, sizeof(SMonDmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonDmInfo)); +} + +void monSetMmInfo(SMonMmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.mmInfo, pInfo, sizeof(SMonMmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonMmInfo)); +} + +void monSetVmInfo(SMonVmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.vmInfo, pInfo, sizeof(SMonVmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonVmInfo)); +} + +void monSetQmInfo(SMonQmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.qmInfo, pInfo, sizeof(SMonQmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonQmInfo)); +} + +void monSetSmInfo(SMonSmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.smInfo, pInfo, sizeof(SMonSmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonSmInfo)); +} + +void monSetBmInfo(SMonBmInfo *pInfo) { + taosThreadMutexLock(&tsMonitor.lock); + memcpy(&tsMonitor.bmInfo, pInfo, sizeof(SMonBmInfo)); + taosThreadMutexUnlock(&tsMonitor.lock); + memset(pInfo, 0, sizeof(SMonBmInfo)); +} + int32_t monInit(const SMonCfg *pCfg) { tsMonitor.logs = taosArrayInit(16, sizeof(SMonLogItem)); if (tsMonitor.logs == NULL) { @@ -42,12 +103,9 @@ int32_t monInit(const SMonCfg *pCfg) { return -1; } - tsMonitor.maxLogs = pCfg->maxLogs; - tsMonitor.server = pCfg->server; - tsMonitor.port = pCfg->port; - tsMonitor.comp = pCfg->comp; + tsMonitor.cfg = *pCfg; tsLogFp = monRecordLog; - tsMonitor.state.time = taosGetTimestampMs(); + tsMonitor.lastTime = taosGetTimestampMs(); taosThreadMutexInit(&tsMonitor.lock, NULL); return 0; } @@ -56,42 +114,65 @@ void monCleanup() { tsLogFp = NULL; taosArrayDestroy(tsMonitor.logs); tsMonitor.logs = NULL; + tFreeSMonMmInfo(&tsMonitor.mmInfo); + tFreeSMonVmInfo(&tsMonitor.vmInfo); + tFreeSMonSmInfo(&tsMonitor.smInfo); + tFreeSMonQmInfo(&tsMonitor.qmInfo); + tFreeSMonBmInfo(&tsMonitor.bmInfo); taosThreadMutexDestroy(&tsMonitor.lock); } -SMonInfo *monCreateMonitorInfo() { +static void monCleanupMonitorInfo(SMonInfo *pMonitor) { + tsMonitor.lastTime = pMonitor->curTime; + taosArrayDestroy(pMonitor->log.logs); + tFreeSMonMmInfo(&pMonitor->mmInfo); + tFreeSMonVmInfo(&pMonitor->vmInfo); + tFreeSMonSmInfo(&pMonitor->smInfo); + tFreeSMonQmInfo(&pMonitor->qmInfo); + tFreeSMonBmInfo(&pMonitor->bmInfo); + tjsonDelete(pMonitor->pJson); + taosMemoryFree(pMonitor); +} + +static SMonInfo *monCreateMonitorInfo() { SMonInfo *pMonitor = taosMemoryCalloc(1, sizeof(SMonInfo)); if (pMonitor == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + monGetLogs(&pMonitor->log); + taosThreadMutexLock(&tsMonitor.lock); - pMonitor->logs = taosArrayDup(tsMonitor.logs); - taosArrayClear(tsMonitor.logs); + memcpy(&pMonitor->dmInfo, &tsMonitor.dmInfo, sizeof(SMonDmInfo)); + memcpy(&pMonitor->mmInfo, &tsMonitor.mmInfo, sizeof(SMonMmInfo)); + memcpy(&pMonitor->vmInfo, &tsMonitor.vmInfo, sizeof(SMonVmInfo)); + memcpy(&pMonitor->smInfo, &tsMonitor.smInfo, sizeof(SMonSmInfo)); + memcpy(&pMonitor->qmInfo, &tsMonitor.qmInfo, sizeof(SMonQmInfo)); + memcpy(&pMonitor->bmInfo, &tsMonitor.bmInfo, sizeof(SMonBmInfo)); + memset(&tsMonitor.dmInfo, 0, sizeof(SMonDmInfo)); + memset(&tsMonitor.mmInfo, 0, sizeof(SMonMmInfo)); + memset(&tsMonitor.vmInfo, 0, sizeof(SMonVmInfo)); + memset(&tsMonitor.smInfo, 0, sizeof(SMonSmInfo)); + memset(&tsMonitor.qmInfo, 0, sizeof(SMonQmInfo)); + memset(&tsMonitor.bmInfo, 0, sizeof(SMonBmInfo)); taosThreadMutexUnlock(&tsMonitor.lock); pMonitor->pJson = tjsonCreateObject(); - if (pMonitor->pJson == NULL || pMonitor->logs == NULL) { + if (pMonitor->pJson == NULL || pMonitor->log.logs == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; monCleanupMonitorInfo(pMonitor); return NULL; } pMonitor->curTime = taosGetTimestampMs(); - pMonitor->lastState = tsMonitor.state; + pMonitor->lastTime = tsMonitor.lastTime; return pMonitor; } -void monCleanupMonitorInfo(SMonInfo *pMonitor) { - tsMonitor.state = pMonitor->lastState; - tsMonitor.state.time = pMonitor->curTime; - taosArrayDestroy(pMonitor->logs); - tjsonDelete(pMonitor->pJson); - taosMemoryFree(pMonitor); -} +static void monGenBasicJson(SMonInfo *pMonitor) { + SMonBasicInfo *pInfo = &pMonitor->dmInfo.basic; -void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { SJson *pJson = pMonitor->pJson; char buf[40] = {0}; taosFormatUtcTime(buf, sizeof(buf), pMonitor->curTime, TSDB_TIME_PRECISION_MILLI); @@ -104,7 +185,10 @@ void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { tjsonAddDoubleToObject(pJson, "protocol", pInfo->protocol); } -void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { +static void monGenClusterJson(SMonInfo *pMonitor) { + SMonClusterInfo *pInfo = &pMonitor->mmInfo.cluster; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; if (tjsonAddItemToObject(pMonitor->pJson, "cluster_info", pJson) != 0) { @@ -154,7 +238,10 @@ void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { } } -void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { +static void monGenVgroupJson(SMonInfo *pMonitor) { + SMonVgroupInfo *pInfo = &pMonitor->mmInfo.vgroup; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "vgroup_infos"); if (pJson == NULL) return; @@ -190,7 +277,10 @@ void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { } } -void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { +static void monGenGrantJson(SMonInfo *pMonitor) { + SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; if (tjsonAddItemToObject(pMonitor->pJson, "grant_info", pJson) != 0) { @@ -203,7 +293,11 @@ void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { tjsonAddDoubleToObject(pJson, "timeseries_total", pInfo->timeseries_total); } -void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { +static void monGenDnodeJson(SMonInfo *pMonitor) { + SMonDnodeInfo *pInfo = &pMonitor->dmInfo.dnode; + SMonSysInfo *pSys = &pMonitor->dmInfo.sys; + SVnodesStat *pStat = &pMonitor->vmInfo.vstat; + SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; if (tjsonAddItemToObject(pMonitor->pJson, "dnode_info", pJson) != 0) { @@ -211,58 +305,83 @@ void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { return; } - SMonState *pLast = &pMonitor->lastState; - double interval = (pMonitor->curTime - pLast->time) / 1000.0; - double req_select_rate = (pInfo->req_select - pLast->req_select) / interval; - double req_insert_rate = (pInfo->req_insert - pLast->req_insert) / interval; - double req_insert_batch_rate = (pInfo->req_insert_batch - pLast->req_insert_batch) / interval; - double net_in_rate = (pInfo->net_in - pLast->net_in) / interval; - double net_out_rate = (pInfo->net_out - pLast->net_out) / interval; - double io_read_rate = (pInfo->io_read - pLast->io_read) / interval; - double io_write_rate = (pInfo->io_write - pLast->io_write) / interval; - double io_read_disk_rate = (pInfo->io_read_disk - pLast->io_read_disk) / interval; - double io_write_disk_rate = (pInfo->io_write_disk - pLast->io_write_disk) / interval; - pLast->req_select = pInfo->req_select; - pLast->req_insert = pInfo->req_insert; - pLast->req_insert_batch = pInfo->req_insert_batch; - pLast->net_in = pInfo->net_in; - pLast->net_out = pInfo->net_out; - pLast->io_read = pInfo->io_read; - pLast->io_write = pInfo->io_write; - pLast->io_read_disk = pInfo->io_read_disk; - pLast->io_write_disk = pInfo->io_write_disk; + double interval = (pMonitor->curTime - pMonitor->lastTime) / 1000.0; + if (pMonitor->curTime - pMonitor->lastTime == 0) { + interval = 1; + } + + double cpu_engine = 0; + double mem_engine = 0; + double net_in = 0; + double net_out = 0; + double io_read = 0; + double io_write = 0; + double io_read_disk = 0; + double io_write_disk = 0; + + SMonSysInfo *sysArrays[6]; + sysArrays[0] = &pMonitor->dmInfo.sys; + sysArrays[1] = &pMonitor->mmInfo.sys; + sysArrays[2] = &pMonitor->vmInfo.sys; + sysArrays[3] = &pMonitor->qmInfo.sys; + sysArrays[4] = &pMonitor->smInfo.sys; + sysArrays[5] = &pMonitor->bmInfo.sys; + for (int32_t i = 0; i < 6; ++i) { + cpu_engine += sysArrays[i]->cpu_engine; + mem_engine += sysArrays[i]->mem_engine; + net_in += sysArrays[i]->net_in; + net_out += sysArrays[i]->net_out; + io_read += sysArrays[i]->io_read; + io_write += sysArrays[i]->io_write; + io_read_disk += sysArrays[i]->io_read_disk; + io_write_disk += sysArrays[i]->io_write_disk; + } + + double req_select_rate = pStat->numOfSelectReqs / interval; + double req_insert_rate = pStat->numOfInsertReqs / interval; + double req_insert_batch_rate = pStat->numOfBatchInsertReqs / interval; + double net_in_rate = net_in / interval; + double net_out_rate = net_out / interval; + double io_read_rate = io_read / interval; + double io_write_rate = io_write / interval; + double io_read_disk_rate = io_read_disk / interval; + double io_write_disk_rate = io_write_disk / interval; tjsonAddDoubleToObject(pJson, "uptime", pInfo->uptime); - tjsonAddDoubleToObject(pJson, "cpu_engine", pInfo->cpu_engine); - tjsonAddDoubleToObject(pJson, "cpu_system", pInfo->cpu_system); - tjsonAddDoubleToObject(pJson, "cpu_cores", pInfo->cpu_cores); - tjsonAddDoubleToObject(pJson, "mem_engine", pInfo->mem_engine); - tjsonAddDoubleToObject(pJson, "mem_system", pInfo->mem_system); - tjsonAddDoubleToObject(pJson, "mem_total", pInfo->mem_total); - tjsonAddDoubleToObject(pJson, "disk_engine", pInfo->disk_engine); - tjsonAddDoubleToObject(pJson, "disk_used", pInfo->disk_used); - tjsonAddDoubleToObject(pJson, "disk_total", pInfo->disk_total); + tjsonAddDoubleToObject(pJson, "cpu_engine", cpu_engine); + tjsonAddDoubleToObject(pJson, "cpu_system", pSys->cpu_system); + tjsonAddDoubleToObject(pJson, "cpu_cores", pSys->cpu_cores); + tjsonAddDoubleToObject(pJson, "mem_engine", mem_engine); + tjsonAddDoubleToObject(pJson, "mem_system", pSys->mem_system); + tjsonAddDoubleToObject(pJson, "mem_total", pSys->mem_total); + tjsonAddDoubleToObject(pJson, "disk_engine", pSys->disk_engine); + tjsonAddDoubleToObject(pJson, "disk_used", pSys->disk_used); + tjsonAddDoubleToObject(pJson, "disk_total", pSys->disk_total); tjsonAddDoubleToObject(pJson, "net_in", net_in_rate); tjsonAddDoubleToObject(pJson, "net_out", net_out_rate); tjsonAddDoubleToObject(pJson, "io_read", io_read_rate); tjsonAddDoubleToObject(pJson, "io_write", io_write_rate); tjsonAddDoubleToObject(pJson, "io_read_disk", io_read_disk_rate); tjsonAddDoubleToObject(pJson, "io_write_disk", io_write_disk_rate); - tjsonAddDoubleToObject(pJson, "req_select", pInfo->req_select); + tjsonAddDoubleToObject(pJson, "req_select", pStat->numOfSelectReqs); tjsonAddDoubleToObject(pJson, "req_select_rate", req_select_rate); - tjsonAddDoubleToObject(pJson, "req_insert", pInfo->req_insert); - tjsonAddDoubleToObject(pJson, "req_insert_success", pInfo->req_insert_success); + tjsonAddDoubleToObject(pJson, "req_insert", pStat->numOfInsertReqs); + tjsonAddDoubleToObject(pJson, "req_insert_success", pStat->numOfInsertSuccessReqs); tjsonAddDoubleToObject(pJson, "req_insert_rate", req_insert_rate); - tjsonAddDoubleToObject(pJson, "req_insert_batch", pInfo->req_insert_batch); - tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pInfo->req_insert_batch_success); + tjsonAddDoubleToObject(pJson, "req_insert_batch", pStat->numOfBatchInsertReqs); + tjsonAddDoubleToObject(pJson, "req_insert_batch_success", pStat->numOfBatchInsertSuccessReqs); tjsonAddDoubleToObject(pJson, "req_insert_batch_rate", req_insert_batch_rate); - tjsonAddDoubleToObject(pJson, "errors", pInfo->errors); - tjsonAddDoubleToObject(pJson, "vnodes_num", pInfo->vnodes_num); - tjsonAddDoubleToObject(pJson, "masters", pInfo->masters); + tjsonAddDoubleToObject(pJson, "errors", pStat->errors); + tjsonAddDoubleToObject(pJson, "vnodes_num", pStat->totalVnodes); + tjsonAddDoubleToObject(pJson, "masters", pStat->masterNum); tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); } -void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { +static void monGenDiskJson(SMonInfo *pMonitor) { + SMonDiskInfo *pInfo = &pMonitor->vmInfo.tfs; + SMonDiskDesc *pLogDesc = &pMonitor->dmInfo.dnode.logdir; + SMonDiskDesc *pTempDesc = &pMonitor->dmInfo.dnode.tempdir; + SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; if (tjsonAddItemToObject(pMonitor->pJson, "disk_infos", pJson) != 0) { @@ -290,18 +409,18 @@ void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { SJson *pLogdirJson = tjsonCreateObject(); if (pLogdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "logdir", pLogdirJson) != 0) return; - tjsonAddStringToObject(pLogdirJson, "name", pInfo->logdir.name); - tjsonAddDoubleToObject(pLogdirJson, "avail", pInfo->logdir.size.avail); - tjsonAddDoubleToObject(pLogdirJson, "used", pInfo->logdir.size.used); - tjsonAddDoubleToObject(pLogdirJson, "total", pInfo->logdir.size.total); + tjsonAddStringToObject(pLogdirJson, "name", pLogDesc->name); + tjsonAddDoubleToObject(pLogdirJson, "avail", pLogDesc->size.avail); + tjsonAddDoubleToObject(pLogdirJson, "used", pLogDesc->size.used); + tjsonAddDoubleToObject(pLogdirJson, "total", pLogDesc->size.total); SJson *pTempdirJson = tjsonCreateObject(); if (pTempdirJson == NULL) return; if (tjsonAddItemToObject(pJson, "tempdir", pTempdirJson) != 0) return; - tjsonAddStringToObject(pTempdirJson, "name", pInfo->tempdir.name); - tjsonAddDoubleToObject(pTempdirJson, "avail", pInfo->tempdir.size.avail); - tjsonAddDoubleToObject(pTempdirJson, "used", pInfo->tempdir.size.used); - tjsonAddDoubleToObject(pTempdirJson, "total", pInfo->tempdir.size.total); + tjsonAddStringToObject(pTempdirJson, "name", pTempDesc->name); + tjsonAddDoubleToObject(pTempdirJson, "avail", pTempDesc->size.avail); + tjsonAddDoubleToObject(pTempdirJson, "used", pTempDesc->size.used); + tjsonAddDoubleToObject(pTempdirJson, "total", pTempDesc->size.total); } static const char *monLogLevelStr(ELogLevel level) { @@ -319,7 +438,7 @@ static const char *monLogLevelStr(ELogLevel level) { } } -static void monSetLogInfo(SMonInfo *pMonitor) { +static void monGenLogJson(SMonInfo *pMonitor) { SJson *pJson = tjsonCreateObject(); if (pJson == NULL) return; if (tjsonAddItemToObject(pMonitor->pJson, "log_infos", pJson) != 0) { @@ -330,20 +449,41 @@ static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pLogsJson = tjsonAddArrayToObject(pJson, "logs"); if (pLogsJson == NULL) return; - for (int32_t i = 0; i < taosArrayGetSize(pMonitor->logs); ++i) { - SJson *pLogJson = tjsonCreateObject(); - if (pLogJson == NULL) continue; + SMonLogs *logs[6]; + logs[0] = &pMonitor->log; + logs[1] = &pMonitor->mmInfo.log; + logs[2] = &pMonitor->vmInfo.log; + logs[3] = &pMonitor->smInfo.log; + logs[4] = &pMonitor->qmInfo.log; + logs[5] = &pMonitor->bmInfo.log; - SMonLogItem *pLogItem = taosArrayGet(pMonitor->logs, i); + int32_t numOfErrorLogs = 0; + int32_t numOfInfoLogs = 0; + int32_t numOfDebugLogs = 0; + int32_t numOfTraceLogs = 0; - char buf[40] = {0}; - taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); + for (int32_t j = 0; j < 6; j++) { + SMonLogs *pLog = logs[j]; + numOfErrorLogs += pLog->numOfErrorLogs; + numOfInfoLogs += pLog->numOfInfoLogs; + numOfDebugLogs += pLog->numOfDebugLogs; + numOfTraceLogs += pLog->numOfTraceLogs; - tjsonAddStringToObject(pLogJson, "ts", buf); - tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); - tjsonAddStringToObject(pLogJson, "content", pLogItem->content); + for (int32_t i = 0; i < taosArrayGetSize(pLog->logs); ++i) { + SJson *pLogJson = tjsonCreateObject(); + if (pLogJson == NULL) continue; - if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); + SMonLogItem *pLogItem = taosArrayGet(pLog->logs, i); + + char buf[40] = {0}; + taosFormatUtcTime(buf, sizeof(buf), pLogItem->ts, TSDB_TIME_PRECISION_MILLI); + + tjsonAddStringToObject(pLogJson, "ts", buf); + tjsonAddStringToObject(pLogJson, "level", monLogLevelStr(pLogItem->level)); + tjsonAddStringToObject(pLogJson, "content", pLogItem->content); + + if (tjsonAddItemToArray(pLogsJson, pLogJson) != 0) tjsonDelete(pLogJson); + } } SJson *pSummaryJson = tjsonAddArrayToObject(pJson, "summary"); @@ -352,35 +492,48 @@ static void monSetLogInfo(SMonInfo *pMonitor) { SJson *pLogError = tjsonCreateObject(); if (pLogError == NULL) return; tjsonAddStringToObject(pLogError, "level", "error"); - tjsonAddDoubleToObject(pLogError, "total", tsNumOfErrorLogs); + tjsonAddDoubleToObject(pLogError, "total", numOfErrorLogs); if (tjsonAddItemToArray(pSummaryJson, pLogError) != 0) tjsonDelete(pLogError); SJson *pLogInfo = tjsonCreateObject(); if (pLogInfo == NULL) return; tjsonAddStringToObject(pLogInfo, "level", "info"); - tjsonAddDoubleToObject(pLogInfo, "total", tsNumOfInfoLogs); + tjsonAddDoubleToObject(pLogInfo, "total", numOfInfoLogs); if (tjsonAddItemToArray(pSummaryJson, pLogInfo) != 0) tjsonDelete(pLogInfo); SJson *pLogDebug = tjsonCreateObject(); if (pLogDebug == NULL) return; tjsonAddStringToObject(pLogDebug, "level", "debug"); - tjsonAddDoubleToObject(pLogDebug, "total", tsNumOfDebugLogs); + tjsonAddDoubleToObject(pLogDebug, "total", numOfDebugLogs); if (tjsonAddItemToArray(pSummaryJson, pLogDebug) != 0) tjsonDelete(pLogDebug); SJson *pLogTrace = tjsonCreateObject(); if (pLogTrace == NULL) return; tjsonAddStringToObject(pLogTrace, "level", "trace"); - tjsonAddDoubleToObject(pLogTrace, "total", tsNumOfTraceLogs); + tjsonAddDoubleToObject(pLogTrace, "total", numOfTraceLogs); if (tjsonAddItemToArray(pSummaryJson, pLogTrace) != 0) tjsonDelete(pLogTrace); } -void monSendReport(SMonInfo *pMonitor) { - monSetLogInfo(pMonitor); +void monSendReport() { + SMonInfo *pMonitor = monCreateMonitorInfo(); + if (pMonitor == NULL) return; + + monGenBasicJson(pMonitor); + monGenClusterJson(pMonitor); + monGenVgroupJson(pMonitor); + monGenGrantJson(pMonitor); + monGenDnodeJson(pMonitor); + monGenDiskJson(pMonitor); + monGenLogJson(pMonitor); char *pCont = tjsonToString(pMonitor->pJson); if (pCont != NULL) { - EHttpCompFlag flag = tsMonitor.comp ? HTTP_GZIP : HTTP_FLAT; - taosSendHttpReport(tsMonitor.server, tsMonitor.port, pCont, strlen(pCont), flag); + EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; + if (taosSendHttpReport(tsMonitor.cfg.server, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { + uError("failed to send monitor msg since %s", terrstr()); + } taosMemoryFree(pCont); } + + monCleanupMonitorInfo(pMonitor); } diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c new file mode 100644 index 0000000000..3aafcf071d --- /dev/null +++ b/source/libs/monitor/src/monMsg.c @@ -0,0 +1,532 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "monInt.h" +#include "tcoding.h" +#include "tencode.h" + +static int32_t tEncodeSMonSysInfo(SCoder *encoder, const SMonSysInfo *pInfo) { + if (tEncodeDouble(encoder, pInfo->cpu_engine) < 0) return -1; + if (tEncodeDouble(encoder, pInfo->cpu_system) < 0) return -1; + if (tEncodeFloat(encoder, pInfo->cpu_cores) < 0) return -1; + if (tEncodeI64(encoder, pInfo->mem_engine) < 0) return -1; + if (tEncodeI64(encoder, pInfo->mem_system) < 0) return -1; + if (tEncodeI64(encoder, pInfo->mem_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->disk_engine) < 0) return -1; + if (tEncodeI64(encoder, pInfo->disk_used) < 0) return -1; + if (tEncodeI64(encoder, pInfo->disk_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->net_in) < 0) return -1; + if (tEncodeI64(encoder, pInfo->net_out) < 0) return -1; + if (tEncodeI64(encoder, pInfo->io_read) < 0) return -1; + if (tEncodeI64(encoder, pInfo->io_write) < 0) return -1; + if (tEncodeI64(encoder, pInfo->io_read_disk) < 0) return -1; + if (tEncodeI64(encoder, pInfo->io_write_disk) < 0) return -1; + return 0; +} + +static int32_t tDecodeSMonSysInfo(SCoder *decoder, SMonSysInfo *pInfo) { + if (tDecodeDouble(decoder, &pInfo->cpu_engine) < 0) return -1; + if (tDecodeDouble(decoder, &pInfo->cpu_system) < 0) return -1; + if (tDecodeFloat(decoder, &pInfo->cpu_cores) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->mem_engine) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->mem_system) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->mem_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->disk_engine) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->disk_used) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->disk_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->net_in) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->net_out) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->io_read) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->io_write) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->io_read_disk) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->io_write_disk) < 0) return -1; + return 0; +} + +int32_t tEncodeSMonLogs(SCoder *encoder, const SMonLogs *pInfo) { + if (tEncodeI32(encoder, pInfo->numOfErrorLogs) < 0) return -1; + if (tEncodeI32(encoder, pInfo->numOfInfoLogs) < 0) return -1; + if (tEncodeI32(encoder, pInfo->numOfDebugLogs) < 0) return -1; + if (tEncodeI32(encoder, pInfo->numOfTraceLogs) < 0) return -1; + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->logs)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->logs); ++i) { + SMonLogItem *pLog = taosArrayGet(pInfo->logs, i); + if (tEncodeI64(encoder, pLog->ts) < 0) return -1; + if (tEncodeI8(encoder, pLog->level) < 0) return -1; + if (tEncodeCStr(encoder, pLog->content) < 0) return -1; + } + return 0; +} + +static int32_t tDecodeSMonLogs(SCoder *decoder, SMonLogs *pInfo) { + if (tDecodeI32(decoder, &pInfo->numOfErrorLogs) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->numOfInfoLogs) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->numOfDebugLogs) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->numOfTraceLogs) < 0) return -1; + + int32_t arraySize = 0; + if (tDecodeI32(decoder, &arraySize) < 0) return -1; + + pInfo->logs = taosArrayInit(arraySize, sizeof(SMonLogItem)); + if (pInfo->logs == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SMonLogItem desc = {0}; + if (tDecodeI64(decoder, &desc.ts) < 0) return -1; + int8_t level = 0; + if (tDecodeI8(decoder, &level) < 0) return -1; + desc.level = level; + if (tDecodeCStrTo(decoder, desc.content) < 0) return -1; + taosArrayPush(pInfo->logs, &desc); + } + + return 0; +} + +int32_t tEncodeSMonClusterInfo(SCoder *encoder, const SMonClusterInfo *pInfo) { + if (tEncodeCStr(encoder, pInfo->first_ep) < 0) return -1; + if (tEncodeI32(encoder, pInfo->first_ep_dnode_id) < 0) return -1; + if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; + if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; + if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; + if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; + if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; + if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; + if (tEncodeI32(encoder, pInfo->vnodes_alive) < 0) return -1; + if (tEncodeI32(encoder, pInfo->connections_total) < 0) return -1; + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->dnodes)) < 0) return -1; + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->mnodes)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dnodes); ++i) { + SMonDnodeDesc *pDesc = taosArrayGet(pInfo->dnodes, i); + if (tEncodeI32(encoder, pDesc->dnode_id) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->dnode_ep) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->status) < 0) return -1; + } + for (int32_t i = 0; i < taosArrayGetSize(pInfo->mnodes); ++i) { + SMonMnodeDesc *pDesc = taosArrayGet(pInfo->mnodes, i); + if (tEncodeI32(encoder, pDesc->mnode_id) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->mnode_ep) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->role) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSMonClusterInfo(SCoder *decoder, SMonClusterInfo *pInfo) { + if (tDecodeCStrTo(decoder, pInfo->first_ep) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->first_ep_dnode_id) < 0) return -1; + if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; + if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->vnodes_alive) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->connections_total) < 0) return -1; + + int32_t dnodesSize = 0; + int32_t mnodesSize = 0; + if (tDecodeI32(decoder, &dnodesSize) < 0) return -1; + if (tDecodeI32(decoder, &mnodesSize) < 0) return -1; + + pInfo->dnodes = taosArrayInit(dnodesSize, sizeof(SMonDnodeDesc)); + pInfo->mnodes = taosArrayInit(mnodesSize, sizeof(SMonMnodeDesc)); + if (pInfo->dnodes == NULL || pInfo->mnodes == NULL) return -1; + + for (int32_t i = 0; i < dnodesSize; ++i) { + SMonDnodeDesc desc = {0}; + if (tDecodeI32(decoder, &desc.dnode_id) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.dnode_ep) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.status) < 0) return -1; + taosArrayPush(pInfo->dnodes, &desc); + } + + for (int32_t i = 0; i < mnodesSize; ++i) { + SMonMnodeDesc desc = {0}; + if (tDecodeI32(decoder, &desc.mnode_id) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.mnode_ep) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.role) < 0) return -1; + taosArrayPush(pInfo->mnodes, &desc); + } + return 0; +} + +int32_t tEncodeSMonVgroupInfo(SCoder *encoder, const SMonVgroupInfo *pInfo) { + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->vgroups)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->vgroups); ++i) { + SMonVgroupDesc *pDesc = taosArrayGet(pInfo->vgroups, i); + if (tEncodeI32(encoder, pDesc->vgroup_id) < 0) return -1; + if (tEncodeI32(encoder, pDesc->tables_num) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->status) < 0) return -1; + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc *pVDesc = &pDesc->vnodes[j]; + if (tEncodeI32(encoder, pVDesc->dnode_id) < 0) return -1; + if (tEncodeCStr(encoder, pVDesc->vnode_role) < 0) return -1; + } + } + return 0; +} + +int32_t tDecodeSMonVgroupInfo(SCoder *decoder, SMonVgroupInfo *pInfo) { + int32_t arraySize = 0; + if (tDecodeI32(decoder, &arraySize) < 0) return -1; + + pInfo->vgroups = taosArrayInit(arraySize, sizeof(SMonVgroupDesc)); + if (pInfo->vgroups == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SMonVgroupDesc desc = {0}; + if (tDecodeI32(decoder, &desc.vgroup_id) < 0) return -1; + if (tDecodeI32(decoder, &desc.tables_num) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.status) < 0) return -1; + for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + SMonVnodeDesc vdesc = {0}; + if (tDecodeI32(decoder, &vdesc.dnode_id) < 0) return -1; + if (tDecodeCStrTo(decoder, vdesc.vnode_role) < 0) return -1; + } + taosArrayPush(pInfo->vgroups, &desc); + } + return 0; +} + +int32_t tEncodeSMonGrantInfo(SCoder *encoder, const SMonGrantInfo *pInfo) { + if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; + if (tEncodeI32(encoder, pInfo->timeseries_used) < 0) return -1; + if (tEncodeI32(encoder, pInfo->timeseries_total) < 0) return -1; + return 0; +} + +int32_t tDecodeSMonGrantInfo(SCoder *decoder, SMonGrantInfo *pInfo) { + if (tDecodeI32(decoder, &pInfo->expire_time) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->timeseries_used) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->timeseries_total) < 0) return -1; + return 0; +} + +int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1; + if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1; + if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1; + if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; + if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1; + if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1; + if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1; + if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; + if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonMmInfo(SMonMmInfo *pInfo) { + taosArrayDestroy(pInfo->log.logs); + taosArrayDestroy(pInfo->cluster.mnodes); + taosArrayDestroy(pInfo->cluster.dnodes); + taosArrayDestroy(pInfo->vgroup.vgroups); + pInfo->cluster.mnodes = NULL; + pInfo->cluster.dnodes = NULL; + pInfo->vgroup.vgroups = NULL; + pInfo->log.logs = NULL; +} + +int32_t tEncodeSMonDiskDesc(SCoder *encoder, const SMonDiskDesc *pDesc) { + if (tEncodeCStr(encoder, pDesc->name) < 0) return -1; + if (tEncodeI8(encoder, pDesc->level) < 0) return -1; + if (tEncodeI64(encoder, pDesc->size.total) < 0) return -1; + if (tEncodeI64(encoder, pDesc->size.used) < 0) return -1; + if (tEncodeI64(encoder, pDesc->size.avail) < 0) return -1; + return 0; +} + +static int32_t tDecodeSMonDiskDesc(SCoder *decoder, SMonDiskDesc *pDesc) { + if (tDecodeCStrTo(decoder, pDesc->name) < 0) return -1; + if (tDecodeI8(decoder, &pDesc->level) < 0) return -1; + if (tDecodeI64(decoder, &pDesc->size.total) < 0) return -1; + if (tDecodeI64(decoder, &pDesc->size.used) < 0) return -1; + if (tDecodeI64(decoder, &pDesc->size.avail) < 0) return -1; + return 0; +} + +int32_t tEncodeSMonDiskInfo(SCoder *encoder, const SMonDiskInfo *pInfo) { + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->datadirs)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->datadirs); ++i) { + SMonDiskDesc *pDesc = taosArrayGet(pInfo->datadirs, i); + if (tEncodeSMonDiskDesc(encoder, pDesc) < 0) return -1; + } + return 0; +} + +static int32_t tDecodeSMonDiskInfo(SCoder *decoder, SMonDiskInfo *pInfo) { + int32_t arraySize = 0; + if (tDecodeI32(decoder, &arraySize) < 0) return -1; + + pInfo->datadirs = taosArrayInit(arraySize, sizeof(SMonDiskDesc)); + if (pInfo->datadirs == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SMonDiskDesc desc = {0}; + if (tDecodeSMonDiskDesc(decoder, &desc) < 0) return -1; + taosArrayPush(pInfo->datadirs, &desc); + } + + return 0; +} + +int32_t tEncodeSVnodesStat(SCoder *encoder, const SVnodesStat *pStat) { + if (tEncodeI32(encoder, pStat->openVnodes) < 0) return -1; + if (tEncodeI32(encoder, pStat->totalVnodes) < 0) return -1; + if (tEncodeI32(encoder, pStat->masterNum) < 0) return -1; + if (tEncodeI64(encoder, pStat->numOfSelectReqs) < 0) return -1; + if (tEncodeI64(encoder, pStat->numOfInsertReqs) < 0) return -1; + if (tEncodeI64(encoder, pStat->numOfInsertSuccessReqs) < 0) return -1; + if (tEncodeI64(encoder, pStat->numOfBatchInsertReqs) < 0) return -1; + if (tEncodeI64(encoder, pStat->numOfBatchInsertSuccessReqs) < 0) return -1; + if (tEncodeI64(encoder, pStat->errors) < 0) return -1; + return 0; +} + +static int32_t tDecodeSVnodesStat(SCoder *decoder, SVnodesStat *pStat) { + if (tDecodeI32(decoder, &pStat->openVnodes) < 0) return -1; + if (tDecodeI32(decoder, &pStat->totalVnodes) < 0) return -1; + if (tDecodeI32(decoder, &pStat->masterNum) < 0) return -1; + if (tDecodeI64(decoder, &pStat->numOfSelectReqs) < 0) return -1; + if (tDecodeI64(decoder, &pStat->numOfInsertReqs) < 0) return -1; + if (tDecodeI64(decoder, &pStat->numOfInsertSuccessReqs) < 0) return -1; + if (tDecodeI64(decoder, &pStat->numOfBatchInsertReqs) < 0) return -1; + if (tDecodeI64(decoder, &pStat->numOfBatchInsertSuccessReqs) < 0) return -1; + if (tDecodeI64(decoder, &pStat->errors) < 0) return -1; + return 0; +} + +int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSMonDiskInfo(&encoder, &pInfo->tfs) < 0) return -1; + if (tEncodeSVnodesStat(&encoder, &pInfo->vstat) < 0) return -1; + if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; + if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSMonDiskInfo(&decoder, &pInfo->tfs) < 0) return -1; + if (tDecodeSVnodesStat(&decoder, &pInfo->vstat) < 0) return -1; + if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; + if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonVmInfo(SMonVmInfo *pInfo) { + taosArrayDestroy(pInfo->log.logs); + taosArrayDestroy(pInfo->tfs.datadirs); + pInfo->log.logs = NULL; + pInfo->tfs.datadirs = NULL; +} + +int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; + if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; + if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonQmInfo(SMonQmInfo *pInfo) { + taosArrayDestroy(pInfo->log.logs); + pInfo->log.logs = NULL; +} + +int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; + if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; + if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonSmInfo(SMonSmInfo *pInfo) { + taosArrayDestroy(pInfo->log.logs); + pInfo->log.logs = NULL; +} + +int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; + if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; + if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonBmInfo(SMonBmInfo *pInfo) { + taosArrayDestroy(pInfo->log.logs); + pInfo->log.logs = NULL; +} + +int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, taosArrayGetSize(pInfo->pVloads)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pVloads); ++i) { + SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); + if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; + if (tEncodeI8(&encoder, pLoad->role) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->compStorage) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->pointsWritten) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfSelectReqs) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfInsertReqs) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfInsertSuccessReqs) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfBatchInsertReqs) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->numOfBatchInsertSuccessReqs) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t arraySize = 0; + if (tDecodeI32(&decoder, &arraySize) < 0) return -1; + + pInfo->pVloads = taosArrayInit(arraySize, sizeof(SVnodeLoad)); + if (pInfo->pVloads == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SVnodeLoad load = {0}; + if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; + if (tDecodeI8(&decoder, &load.role) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; + if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; + if (tDecodeI64(&decoder, &load.compStorage) < 0) return -1; + if (tDecodeI64(&decoder, &load.pointsWritten) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfSelectReqs) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfInsertReqs) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfInsertSuccessReqs) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfBatchInsertReqs) < 0) return -1; + if (tDecodeI64(&decoder, &load.numOfBatchInsertSuccessReqs) < 0) return -1; + taosArrayPush(pInfo->pVloads, &load); + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSMonVloadInfo(SMonVloadInfo *pInfo) { + taosArrayDestroy(pInfo->pVloads); + pInfo->pVloads = NULL; +} diff --git a/source/libs/monitor/test/CMakeLists.txt b/source/libs/monitor/test/CMakeLists.txt index e3ab7e7337..cd1b94574d 100644 --- a/source/libs/monitor/test/CMakeLists.txt +++ b/source/libs/monitor/test/CMakeLists.txt @@ -1,14 +1,14 @@ enable_testing() aux_source_directory(. MONITOR_TEST_SRC) -add_executable(monitor_test ${MONITOR_TEST_SRC}) +add_executable(monitorTest ${MONITOR_TEST_SRC}) target_link_libraries( - monitor_test + monitorTest PUBLIC monitor PUBLIC gtest_main ) add_test( - NAME monitor_test - COMMAND monitor_test + NAME monitorTest + COMMAND monitorTest ) diff --git a/source/libs/monitor/test/monTest.cpp b/source/libs/monitor/test/monTest.cpp index 05c9fb8b38..ceb9805e3d 100644 --- a/source/libs/monitor/test/monTest.cpp +++ b/source/libs/monitor/test/monTest.cpp @@ -22,7 +22,7 @@ class MonitorTest : public ::testing::Test { cfg.maxLogs = 2; cfg.port = 80; cfg.server = "localhost"; - cfg.comp = 0; + cfg.comp = 1; monInit(&cfg); } @@ -32,24 +32,64 @@ class MonitorTest : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); - void GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo); - void GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo); - void GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo); - void GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo); - void GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo); + void GetBasicInfo(SMonBasicInfo *pInfo); + void GetDnodeInfo(SMonDnodeInfo *pInfo); + void GetSysInfo(SMonSysInfo *pInfo); + + void GetClusterInfo(SMonClusterInfo *pInfo); + void GetVgroupInfo(SMonVgroupInfo *pInfo); + void GetGrantInfo(SMonGrantInfo *pInfo); + + void GetVnodeStat(SVnodesStat *pStat); + void GetDiskInfo(SMonDiskInfo *pInfo); + + void GetLogInfo(SMonLogs *logs); + void AddLogInfo1(); void AddLogInfo2(); }; -void MonitorTest::GetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo) { +void MonitorTest::GetBasicInfo(SMonBasicInfo *pInfo) { pInfo->dnode_id = 1; strcpy(pInfo->dnode_ep, "localhost"); pInfo->cluster_id = 6980428120398645172; pInfo->protocol = 1; } -void MonitorTest::GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { +void MonitorTest::GetDnodeInfo(SMonDnodeInfo *pInfo) { + pInfo->uptime = 1.2; + pInfo->has_mnode = 1; + + strcpy(pInfo->logdir.name, "/log/dir/d"); + pInfo->logdir.size.avail = 41; + pInfo->logdir.size.total = 42; + pInfo->logdir.size.used = 43; + + strcpy(pInfo->tempdir.name, "/data/dir/d"); + pInfo->tempdir.size.avail = 51; + pInfo->tempdir.size.total = 52; + pInfo->tempdir.size.used = 53; +} + +void MonitorTest::GetSysInfo(SMonSysInfo *pInfo) { + pInfo->cpu_engine = 2.1; + pInfo->cpu_system = 2.1; + pInfo->cpu_cores = 2; + pInfo->mem_engine = 3.1; + pInfo->mem_system = 3.2; + pInfo->mem_total = 3.3; + pInfo->disk_engine = 4.1; + pInfo->disk_used = 4.2; + pInfo->disk_total = 4.3; + pInfo->net_in = 5.1; + pInfo->net_out = 5.2; + pInfo->io_read = 6.1; + pInfo->io_write = 6.2; + pInfo->io_read_disk = 7.1; + pInfo->io_write_disk = 7.2; +} + +void MonitorTest::GetClusterInfo(SMonClusterInfo *pInfo) { strcpy(pInfo->first_ep, "localhost:6030"); pInfo->first_ep_dnode_id = 1; strcpy(pInfo->version, "3.0.0.0"); @@ -86,7 +126,7 @@ void MonitorTest::GetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo) { taosArrayPush(pInfo->mnodes, &m2); } -void MonitorTest::GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { +void MonitorTest::GetVgroupInfo(SMonVgroupInfo *pInfo) { pInfo->vgroups = taosArrayInit(4, sizeof(SMonVgroupDesc)); SMonVgroupDesc vg1 = {0}; @@ -121,41 +161,24 @@ void MonitorTest::GetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo) { taosArrayPush(pInfo->vgroups, &vg3); } -void MonitorTest::GetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo) { +void MonitorTest::GetGrantInfo(SMonGrantInfo *pInfo) { pInfo->expire_time = 1234567; pInfo->timeseries_total = 234567; pInfo->timeseries_used = 34567; } -void MonitorTest::GetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo) { - pInfo->uptime = 1.2; - pInfo->cpu_engine = 2.1; - pInfo->cpu_system = 2.1; - pInfo->cpu_cores = 2; - pInfo->mem_engine = 3.1; - pInfo->mem_system = 3.2; - pInfo->mem_total = 3.3; - pInfo->disk_engine = 4.1; - pInfo->disk_used = 4.2; - pInfo->disk_total = 4.3; - pInfo->net_in = 5.1; - pInfo->net_out = 5.2; - pInfo->io_read = 6.1; - pInfo->io_write = 6.2; - pInfo->io_read_disk = 7.1; - pInfo->io_write_disk = 7.2; - pInfo->req_select = 8; - pInfo->req_insert = 9; - pInfo->req_insert_success = 10; - pInfo->req_insert_batch = 11; - pInfo->req_insert_batch_success = 12; +void MonitorTest::GetVnodeStat(SVnodesStat *pInfo) { + pInfo->numOfSelectReqs = 8; + pInfo->numOfInsertReqs = 9; + pInfo->numOfInsertSuccessReqs = 10; + pInfo->numOfBatchInsertReqs = 11; + pInfo->numOfBatchInsertSuccessReqs = 12; pInfo->errors = 4; - pInfo->vnodes_num = 5; - pInfo->masters = 6; - pInfo->has_mnode = 1; + pInfo->totalVnodes = 5; + pInfo->masterNum = 6; } -void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { +void MonitorTest::GetDiskInfo(SMonDiskInfo *pInfo) { pInfo->datadirs = taosArrayInit(2, sizeof(SMonDiskDesc)); SMonDiskDesc d1 = {0}; strcpy(d1.name, "/t1/d1/d"); @@ -180,16 +203,25 @@ void MonitorTest::GetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo) { d3.size.total = 32; d3.size.used = 33; taosArrayPush(pInfo->datadirs, &d3); +} - strcpy(pInfo->logdir.name, "/log/dir/d"); - pInfo->logdir.size.avail = 41; - pInfo->logdir.size.total = 42; - pInfo->logdir.size.used = 43; +void MonitorTest::GetLogInfo(SMonLogs *logs) { + logs->logs = taosArrayInit(4, sizeof(SMonLogItem)); - strcpy(pInfo->tempdir.name, "/data/dir/d"); - pInfo->tempdir.size.avail = 51; - pInfo->tempdir.size.total = 52; - pInfo->tempdir.size.used = 53; + SMonLogItem item1 = {.level = DEBUG_INFO}; + item1.ts = taosGetTimestampMs(); + strcpy(item1.content, "log test1"); + taosArrayPush(logs->logs, &item1); + + SMonLogItem item2 = {.level = DEBUG_ERROR}; + item2.ts = taosGetTimestampMs(); + strcpy(item2.content, "log test2"); + taosArrayPush(logs->logs, &item2); + + logs->numOfErrorLogs = 1; + logs->numOfInfoLogs = 2; + logs->numOfDebugLogs = 3; + logs->numOfTraceLogs = 4; } void MonitorTest::AddLogInfo1() { @@ -206,46 +238,52 @@ void MonitorTest::AddLogInfo2() { TEST_F(MonitorTest, 01_Full) { AddLogInfo1(); - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; + SMonDmInfo dmInfo = {0}; + GetBasicInfo(&dmInfo.basic); + GetDnodeInfo(&dmInfo.dnode); + GetSysInfo(&dmInfo.sys); - SMonBasicInfo basicInfo = {0}; - GetBasicInfo(pMonitor, &basicInfo); - monSetBasicInfo(pMonitor, &basicInfo); + SMonMmInfo mmInfo = {0}; + GetClusterInfo(&mmInfo.cluster); + GetVgroupInfo(&mmInfo.vgroup); + GetGrantInfo(&mmInfo.grant); + GetSysInfo(&mmInfo.sys); + GetLogInfo(&mmInfo.log); - SMonClusterInfo clusterInfo = {0}; - SMonVgroupInfo vgroupInfo = {0}; - SMonGrantInfo grantInfo = {0}; - GetClusterInfo(pMonitor, &clusterInfo); - GetVgroupInfo(pMonitor, &vgroupInfo); - GetGrantInfo(pMonitor, &grantInfo); - monSetClusterInfo(pMonitor, &clusterInfo); - monSetVgroupInfo(pMonitor, &vgroupInfo); - monSetGrantInfo(pMonitor, &grantInfo); + SMonVmInfo vmInfo = {0}; + GetDiskInfo(&vmInfo.tfs); + GetVnodeStat(&vmInfo.vstat); + GetSysInfo(&vmInfo.sys); + GetLogInfo(&vmInfo.log); - SMonDnodeInfo dnodeInfo = {0}; - GetDnodeInfo(pMonitor, &dnodeInfo); - monSetDnodeInfo(pMonitor, &dnodeInfo); + SMonQmInfo qmInfo = {0}; + GetSysInfo(&qmInfo.sys); + GetLogInfo(&qmInfo.log); - SMonDiskInfo diskInfo = {0}; - GetDiskInfo(pMonitor, &diskInfo); - monSetDiskInfo(pMonitor, &diskInfo); + SMonSmInfo smInfo = {0}; + GetSysInfo(&smInfo.sys); + GetLogInfo(&smInfo.log); - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); + SMonBmInfo bmInfo = {0}; + GetSysInfo(&bmInfo.sys); + GetLogInfo(&bmInfo.log); - taosArrayDestroy(clusterInfo.dnodes); - taosArrayDestroy(clusterInfo.mnodes); - taosArrayDestroy(vgroupInfo.vgroups); - taosArrayDestroy(diskInfo.datadirs); + monSetDmInfo(&dmInfo); + monSetMmInfo(&mmInfo); + monSetVmInfo(&vmInfo); + monSetQmInfo(&qmInfo); + monSetSmInfo(&smInfo); + monSetBmInfo(&bmInfo); + + tFreeSMonMmInfo(&mmInfo); + tFreeSMonVmInfo(&vmInfo); + tFreeSMonSmInfo(&smInfo); + tFreeSMonQmInfo(&qmInfo); + tFreeSMonBmInfo(&bmInfo); + monSendReport(); } TEST_F(MonitorTest, 02_Log) { AddLogInfo2(); - - SMonInfo *pMonitor = monCreateMonitorInfo(); - if (pMonitor == NULL) return; - - monSendReport(pMonitor); - monCleanupMonitorInfo(pMonitor); + monSendReport(); } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 0b4f79d3ff..048830b80b 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -149,22 +149,37 @@ static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTab return pCxt->code; } +static void rewriteConstCondition(SSelectStmt* pSelect, SNode** pCond) { + if (QUERY_NODE_VALUE != nodeType(*pCond)) { + return; + } + if (((SValueNode*)*pCond)->datum.b) { + nodesDestroyNode(*pCond); + *pCond = NULL; + } else { + pSelect->isEmptyResult = true; + } +} + static int32_t calcConstFromTable(SCalcConstContext* pCxt, SSelectStmt* pSelect) { - nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, pCxt); + pCxt->code = rewriteConditionForFromTable(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = rewriteConditionForFromTable(pCxt, pSelect->pFromTable); + nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, pCxt); } return pCxt->code; } -static int32_t calcConstCondition(SCalcConstContext* pCxt, SNode** pCond) { +static int32_t calcConstCondition(SCalcConstContext* pCxt, SSelectStmt* pSelect, SNode** pCond) { if (NULL == *pCond) { return TSDB_CODE_SUCCESS; } - nodesRewriteExprPostOrder(pCond, calcConst, pCxt); + pCxt->code = rewriteCondition(pCxt, pCond); if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = rewriteCondition(pCxt, pCond); + nodesRewriteExprPostOrder(pCond, calcConst, pCxt); + } + if (TSDB_CODE_SUCCESS == pCxt->code) { + rewriteConstCondition(pSelect, pCond); } return pCxt->code; } @@ -176,7 +191,7 @@ static int32_t calcConstSelect(SSelectStmt* pSelect) { cxt.code = calcConstFromTable(&cxt, pSelect); } if (TSDB_CODE_SUCCESS == cxt.code) { - cxt.code = calcConstCondition(&cxt, &pSelect->pWhere); + cxt.code = calcConstCondition(&cxt, pSelect, &pSelect->pWhere); } if (TSDB_CODE_SUCCESS == cxt.code) { nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); @@ -188,7 +203,7 @@ static int32_t calcConstSelect(SSelectStmt* pSelect) { nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); } if (TSDB_CODE_SUCCESS == cxt.code) { - cxt.code = calcConstCondition(&cxt, &pSelect->pHaving); + cxt.code = calcConstCondition(&cxt, pSelect, &pSelect->pHaving); } if (TSDB_CODE_SUCCESS == cxt.code) { nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); @@ -208,6 +223,22 @@ static int32_t calcConstQuery(SNode* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { - return calcConstQuery(pQuery->pRoot); +static bool isEmptyResultQuery(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return ((SSelectStmt*)pStmt)->isEmptyResult; + case QUERY_NODE_EXPLAIN_STMT: + return isEmptyResultQuery(((SExplainStmt*)pStmt)->pQuery); + default: + break; + } + return false; +} + +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { + int32_t code = calcConstQuery(pQuery->pRoot); + if (TSDB_CODE_SUCCESS == code) { + pQuery->execMode = isEmptyResultQuery(pQuery->pRoot) ? QUERY_EXEC_MODE_EMPTY_RESULT : pQuery->execMode; + } + return code; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d9cf0b39b5..9d945039b8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1084,7 +1084,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - (*pQuery)->directRpc = false; + (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; (*pQuery)->msgType = TDMT_VND_SUBMIT; (*pQuery)->pRoot = (SNode*)context.pOutput; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ebe5b639c4..a286918ce5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2923,21 +2923,23 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: case QUERY_NODE_EXPLAIN_STMT: + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = true; pQuery->msgType = TDMT_VND_QUERY; break; case QUERY_NODE_VNODE_MODIF_STMT: + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_CREATE_TABLE; break; case QUERY_NODE_DESCRIBE_STMT: - pQuery->localCmd = true; + pQuery->execMode = QUERY_EXEC_MODE_LOCAL; pQuery->haveResultSet = true; break; case QUERY_NODE_RESET_QUERY_CACHE_STMT: - pQuery->localCmd = true; + pQuery->execMode = QUERY_EXEC_MODE_LOCAL; break; default: - pQuery->directRpc = true; + pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*); pQuery->msgType = pQuery->pCmdMsg->msgType; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index d211e780b0..932107dfc1 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -139,6 +139,25 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SUserIndexReq indexReq = {0}; + strcpy(indexReq.indexFName, input); + + int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSUserIndexReq(pBuf, bufLen, &indexReq); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -343,6 +362,22 @@ int32_t queryProcessGetDbCfgRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetIndexRsp(void *output, char *msg, int32_t msgSize) { + SUserIndexRsp out = {0}; + + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSUserIndexRsp(msg, msgSize, &out) != 0) { + qError("tDeserializeSUserIndexRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + memcpy(output, &out, sizeof(out)); + + return TSDB_CODE_SUCCESS; +} void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -350,12 +385,14 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 16236fc05b..c780ed5725 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1797,6 +1797,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } else { SScalarParam out = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; out.columnData->info.type = type; + out.columnData->info.bytes = tDataTypes[type].bytes; // todo refactor the convert int32_t code = doConvertDataType(var, &out); @@ -1804,6 +1805,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) { qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; } + + memcpy(fi->data, out.columnData->pData, out.columnData->info.bytes); } } @@ -3638,6 +3641,10 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param) { + if (NULL == info) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + return fltSetColFieldDataImpl(info, param, fltGetDataFromSlotId, false); } @@ -3691,6 +3698,10 @@ _return: } bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + if (NULL == info) { + return false; + } + if (info->scalarMode) { SScalarParam output = {0}; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 07358f905f..f1f4e91a3c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -133,7 +133,6 @@ void sclFreeRes(SHashObj *res) { void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); - taosMemoryFreeClear(param->columnData); } if (param->pHashFilter != NULL) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7ce7b422ac..0956c2add5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -795,16 +795,19 @@ bool getTimePseudoFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); + return TSDB_CODE_SUCCESS; } int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 1)); + return TSDB_CODE_SUCCESS; } int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { ASSERT(inputNum == 1); colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 2)); + return TSDB_CODE_SUCCESS; } int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c5f18fb1a9..8ba8717854 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -476,7 +476,6 @@ static SColumnInfoData* doVectorConvert(SScalarParam* pInput, int32_t* doConvert static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { if (type == VECTOR_DO_CONVERT) { colDataDestroy(pCol); - taosMemoryFree(pCol); } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 5fb6069eff..11f8e880a6 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -343,6 +343,7 @@ _return: SCH_JOB_ELOG("invalid job status update, from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); SCH_ERR_RET(code); + return TSDB_CODE_SUCCESS; } int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { @@ -877,7 +878,7 @@ _return: SCH_RET(schProcessOnJobFailure(pJob, code)); } -int32_t schProcessOnDataFetched(SSchJob *job) { +void schProcessOnDataFetched(SSchJob *job) { atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); tsem_post(&job->rspSem); } @@ -1333,6 +1334,7 @@ int32_t schHandleExplainCallback(void *param, const SDataBuf *pMsg, int32_t code int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); + return TSDB_CODE_SUCCESS; } int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { @@ -2232,7 +2234,7 @@ void schDropJobAllTasks(SSchJob *pJob) { int32_t schCancelJob(SSchJob *pJob) { // TODO - + return TSDB_CODE_SUCCESS; // TODO MOVE ALL TASKS FROM EXEC LIST TO FAIL LIST } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b4e291f5e4..86272b3cad 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -298,4 +298,5 @@ static int tdbPCacheCloseImpl(SPCache *pCache) { } tdbPCacheDestroyLock(pCache); + return 0; } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 90f15dd7d0..cd1fbf8e0e 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -117,7 +117,7 @@ _OVER: static void clientConnCb(uv_connect_t* req, int32_t status) { if (status < 0) { terrno = TAOS_SYSTEM_ERROR(status); - uError("Connection error %s\n", uv_strerror(status)); + uError("connection error %s", uv_strerror(status)); uv_close((uv_handle_t*)req->handle, NULL); return; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index eb5712d2e5..26de26ab67 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -129,14 +129,12 @@ static void taosGetProcIOnfos() { static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - // printf("open file:%s failed", tsSysCpuFile); return -1; } char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - // printf("read file:%s failed", tsSysCpuFile); taosCloseFile(&pFile); return -1; } @@ -153,14 +151,12 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - // printf("open file:%s failed", tsProcCpuFile); return -1; } char *line = NULL; ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { - // printf("read file:%s failed", tsProcCpuFile); taosCloseFile(&pFile); return -1; } @@ -182,12 +178,12 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { #endif bool taosCheckSystemIsSmallEnd() { - union check{ - int16_t i; - char ch[2]; - }c; - c.i=1; - return c.ch[0]==1; + union check { + int16_t i; + char ch[2]; + } c; + c.i = 1; + return c.ch[0] == 1; } void taosGetSystemInfo() { @@ -617,6 +613,28 @@ int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int #endif } +void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { + static int64_t last_rchars = 0; + static int64_t last_wchars = 0; + static int64_t last_read_bytes = 0; + static int64_t last_write_bytes = 0; + + static int64_t cur_rchars = 0; + static int64_t cur_wchars = 0; + static int64_t cur_read_bytes = 0; + static int64_t cur_write_bytes = 0; + if (taosGetProcIO(&cur_rchars, &cur_wchars, &cur_read_bytes, &cur_write_bytes) == 0) { + *rchars = cur_rchars - last_rchars; + *wchars = cur_wchars - last_wchars; + *read_bytes = cur_read_bytes - last_read_bytes; + *write_bytes = cur_write_bytes - last_write_bytes; + last_rchars = cur_rchars; + last_wchars = cur_wchars; + last_read_bytes = cur_read_bytes; + last_write_bytes = cur_write_bytes; + } +} + int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) *receive_bytes = 0; @@ -672,6 +690,20 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { #endif } +void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { + static int64_t last_receive_bytes = 0; + static int64_t last_transmit_bytes = 0; + + static int64_t cur_receive_bytes = 0; + static int64_t cur_transmit_bytes = 0; + if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { + *receive_bytes = cur_receive_bytes - last_receive_bytes; + *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; + last_receive_bytes = cur_receive_bytes; + last_transmit_bytes = cur_transmit_bytes; + } +} + void taosKillSystem() { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) printf("function taosKillSystem, exit!"); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7f9a3ff593..93022de021 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -437,7 +437,6 @@ int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight) { return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; } - __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; @@ -569,53 +568,36 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { - __compar_fn_t comparFn = NULL; - switch (keyType) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: - comparFn = (order == TSDB_ORDER_ASC) ? compareInt8Val : compareInt8ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareInt8Val : compareInt8ValDesc; case TSDB_DATA_TYPE_SMALLINT: - comparFn = (order == TSDB_ORDER_ASC) ? compareInt16Val : compareInt16ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareInt16Val : compareInt16ValDesc; case TSDB_DATA_TYPE_INT: - comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = (order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; case TSDB_DATA_TYPE_FLOAT: - comparFn = (order == TSDB_ORDER_ASC) ? compareFloatVal : compareFloatValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareFloatVal : compareFloatValDesc; case TSDB_DATA_TYPE_DOUBLE: - comparFn = (order == TSDB_ORDER_ASC) ? compareDoubleVal : compareDoubleValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareDoubleVal : compareDoubleValDesc; case TSDB_DATA_TYPE_UTINYINT: - comparFn = (order == TSDB_ORDER_ASC) ? compareUint8Val : compareUint8ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareUint8Val : compareUint8ValDesc; case TSDB_DATA_TYPE_USMALLINT: - comparFn = (order == TSDB_ORDER_ASC) ? compareUint16Val : compareUint16ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareUint16Val : compareUint16ValDesc; case TSDB_DATA_TYPE_UINT: - comparFn = (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc; case TSDB_DATA_TYPE_UBIGINT: - comparFn = (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; case TSDB_DATA_TYPE_BINARY: - comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: - comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; default: - comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; - break; + return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; } - - return comparFn; } int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index e7e870e998..74d7c15e78 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -348,7 +348,7 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { } } - uError("name:%s, cfg not found", name); + // uError("name:%s, cfg not found", name); terrno = TSDB_CODE_CFG_NOT_FOUND; return NULL; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6d4de2f575..e02310aab3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -224,6 +224,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") // mnode-vgroup TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 84a2efb46c..48cf97a11f 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -136,7 +136,7 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pBuf->nextPos += size; int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret != 0) { + if (ret == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -167,7 +167,7 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // 3. write to disk. int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret != 0) { + if (ret == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -222,7 +222,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret != 0) { + if (ret == -1) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } @@ -348,8 +348,7 @@ static void lruListMoveToFront(SList* pList, SPageInfo* pi) { } static SPageInfo* getPageInfoFromPayload(void* page) { - int32_t offset = offsetof(SPageInfo, pData); - char* p = (char *)page - offset; + char* p = (char *)page - POINTER_BYTES; SPageInfo* ppi = ((SPageInfo**)p)[0]; return ppi; @@ -369,7 +368,6 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->totalBufSize = 0; pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. pPBuf->allocateId = -1; - pPBuf->comp = true; pPBuf->pFile = NULL; pPBuf->id = strdup(id); pPBuf->fileSize = 0; diff --git a/tests/script/tsim/query/stddev.sim b/tests/script/tsim/query/stddev.sim index 01cde31966..adab1c354d 100644 --- a/tests/script/tsim/query/stddev.sim +++ b/tests/script/tsim/query/stddev.sim @@ -121,4 +121,4 @@ if $rows != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT