Merge branch '3.0' into cpwu/3.0
This commit is contained in:
cpwu
commit
0af0b1a98e
|
|
@ -36,11 +36,13 @@ IF (TD_WINDOWS)
|
|||
ENDIF ()
|
||||
|
||||
ELSE ()
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -g3")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
|
||||
#SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
|
||||
#SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
|
||||
|
||||
MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64")
|
||||
IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
|
||||
ADD_DEFINITIONS("-D_TD_ARM_")
|
||||
ELSE ()
|
||||
ADD_DEFINITIONS("-msse4.2 -mfma")
|
||||
|
|
|
|||
|
|
@ -55,17 +55,6 @@ option(
|
|||
OFF
|
||||
)
|
||||
|
||||
IF(${TD_WINDOWS})
|
||||
MESSAGE("Not build BDB on Windows")
|
||||
ELSE ()
|
||||
option(
|
||||
BUILD_WITH_BDB
|
||||
"If build with BerkleyDB"
|
||||
ON
|
||||
)
|
||||
|
||||
ENDIF ()
|
||||
|
||||
option(
|
||||
BUILD_WITH_LUCENE
|
||||
"If build with lucene"
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
|
|||
|
||||
SET(TD_DARWIN TRUE)
|
||||
SET(OSTYPE "macOS")
|
||||
ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare -Wno-return-type")
|
||||
ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare")
|
||||
|
||||
MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
|
||||
IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
|
||||
|
|
|
|||
|
|
@ -63,9 +63,9 @@ if(${BUILD_WITH_UV})
|
|||
endif(${BUILD_WITH_UV})
|
||||
|
||||
# bdb
|
||||
if(${BUILD_WITH_BDB})
|
||||
cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
endif(${BUILD_WITH_BDB})
|
||||
#if(${BUILD_WITH_BDB})
|
||||
#cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
|
||||
#endif(${BUILD_WITH_BDB})
|
||||
|
||||
# sqlite
|
||||
if(${BUILD_WITH_SQLITE})
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
|
|||
add_subdirectory(lucene)
|
||||
endif(${BUILD_WITH_LUCENE})
|
||||
|
||||
if(${BUILD_WITH_BDB})
|
||||
add_subdirectory(bdb)
|
||||
endif(${BUILD_WITH_BDB})
|
||||
#if(${BUILD_WITH_BDB})
|
||||
#add_subdirectory(bdb)
|
||||
#endif(${BUILD_WITH_BDB})
|
||||
|
||||
if(${BUILD_WITH_SQLITE})
|
||||
add_subdirectory(sqlite)
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ typedef struct SDataBlockInfo {
|
|||
int64_t uid;
|
||||
int64_t blockId;
|
||||
};
|
||||
int64_t groupId; // no need to serialize
|
||||
uint64_t groupId; // no need to serialize
|
||||
} SDataBlockInfo;
|
||||
|
||||
typedef struct SSDataBlock {
|
||||
|
|
|
|||
|
|
@ -127,39 +127,39 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
|
|||
pColumnInfoData->hasNull = true;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
|
||||
static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
|
||||
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT ||
|
||||
pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(int8_t*)p = *(int8_t*)v;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
|
||||
static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) {
|
||||
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(int16_t*)p = *(int16_t*)v;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
|
||||
static FORCE_INLINE void colDataAppendInt32(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int32_t* v) {
|
||||
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_INT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UINT);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(int32_t*)p = *(int32_t*)v;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
|
||||
static FORCE_INLINE void colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
|
||||
int32_t type = pColumnInfoData->info.type;
|
||||
ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(int64_t*)p = *(int64_t*)v;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
|
||||
static FORCE_INLINE void colDataAppendFloat(SColumnInfoData* pColumnInfoData, uint32_t currentRow, float* v) {
|
||||
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_FLOAT);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(float*)p = *(float*)v;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
|
||||
static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, uint32_t currentRow, double* v) {
|
||||
ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_DOUBLE);
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
*(double*)p = *(double*)v;
|
||||
|
|
@ -177,11 +177,12 @@ void colDataTrim(SColumnInfoData* pColumnInfoData);
|
|||
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
|
||||
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
|
||||
|
||||
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
|
||||
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap);
|
||||
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
|
||||
int32_t pageSize);
|
||||
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
|
||||
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
|
||||
int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity);
|
||||
|
||||
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
|
||||
|
||||
|
|
|
|||
|
|
@ -41,9 +41,6 @@ extern int32_t tsMaxConnections;
|
|||
extern int32_t tsMaxShellConns;
|
||||
extern int32_t tsShellActivityTimer;
|
||||
extern int32_t tsMaxTmrCtrl;
|
||||
extern float tsNumOfThreadsPerCore;
|
||||
extern int32_t tsNumOfCommitThreads;
|
||||
extern float tsRatioOfQueryCores;
|
||||
extern int32_t tsCompressMsgSize;
|
||||
extern int32_t tsCompressColData;
|
||||
extern int32_t tsMaxNumOfDistinctResults;
|
||||
|
|
@ -60,6 +57,22 @@ extern int32_t tsQnodeShmSize;
|
|||
extern int32_t tsSnodeShmSize;
|
||||
extern int32_t tsBnodeShmSize;
|
||||
|
||||
// queue & threads
|
||||
extern int32_t tsNumOfRpcThreads;
|
||||
extern int32_t tsNumOfCommitThreads;
|
||||
extern int32_t tsNumOfTaskQueueThreads;
|
||||
extern int32_t tsNumOfMnodeQueryThreads;
|
||||
extern int32_t tsNumOfMnodeReadThreads;
|
||||
extern int32_t tsNumOfVnodeQueryThreads;
|
||||
extern int32_t tsNumOfVnodeFetchThreads;
|
||||
extern int32_t tsNumOfVnodeWriteThreads;
|
||||
extern int32_t tsNumOfVnodeSyncThreads;
|
||||
extern int32_t tsNumOfVnodeMergeThreads;
|
||||
extern int32_t tsNumOfQnodeQueryThreads;
|
||||
extern int32_t tsNumOfQnodeFetchThreads;
|
||||
extern int32_t tsNumOfSnodeSharedThreads;
|
||||
extern int32_t tsNumOfSnodeUniqueThreads;
|
||||
|
||||
// monitor
|
||||
extern bool tsEnableMonitor;
|
||||
extern int32_t tsMonitorInterval;
|
||||
|
|
|
|||
|
|
@ -483,7 +483,8 @@ typedef struct {
|
|||
int32_t tz; // query client timezone
|
||||
char intervalUnit;
|
||||
char slidingUnit;
|
||||
char offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
|
||||
char
|
||||
offsetUnit; // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
|
||||
int8_t precision;
|
||||
int64_t interval;
|
||||
int64_t sliding;
|
||||
|
|
@ -511,6 +512,7 @@ typedef struct {
|
|||
int32_t maxRows;
|
||||
int32_t commitTime;
|
||||
int32_t fsyncPeriod;
|
||||
int32_t ttl;
|
||||
int8_t walLevel;
|
||||
int8_t precision; // time resolution
|
||||
int8_t compression;
|
||||
|
|
@ -520,6 +522,7 @@ typedef struct {
|
|||
int8_t cacheLastRow;
|
||||
int8_t ignoreExist;
|
||||
int8_t streamMode;
|
||||
int8_t singleSTable;
|
||||
int32_t numOfRetensions;
|
||||
SArray* pRetensions; // SRetention
|
||||
} SCreateDbReq;
|
||||
|
|
@ -585,6 +588,41 @@ int32_t tSerializeSUseDbRspImp(SCoder* pEncoder, const SUseDbRsp* pRsp);
|
|||
int32_t tDeserializeSUseDbRspImp(SCoder* pDecoder, SUseDbRsp* pRsp);
|
||||
void tFreeSUsedbRsp(SUseDbRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
char db[TSDB_DB_FNAME_LEN];
|
||||
} SDbCfgReq;
|
||||
|
||||
int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
|
||||
int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int32_t numOfVgroups;
|
||||
int32_t cacheBlockSize;
|
||||
int32_t totalBlocks;
|
||||
int32_t daysPerFile;
|
||||
int32_t daysToKeep0;
|
||||
int32_t daysToKeep1;
|
||||
int32_t daysToKeep2;
|
||||
int32_t minRows;
|
||||
int32_t maxRows;
|
||||
int32_t commitTime;
|
||||
int32_t fsyncPeriod;
|
||||
int32_t ttl;
|
||||
int8_t walLevel;
|
||||
int8_t precision;
|
||||
int8_t compression;
|
||||
int8_t replications;
|
||||
int8_t quorum;
|
||||
int8_t update;
|
||||
int8_t cacheLastRow;
|
||||
int8_t streamMode;
|
||||
int8_t singleSTable;
|
||||
} SDbCfgRsp;
|
||||
|
||||
int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp);
|
||||
int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp);
|
||||
|
||||
|
||||
typedef struct {
|
||||
int32_t rowNum;
|
||||
} SQnodeListReq;
|
||||
|
|
@ -679,6 +717,18 @@ typedef struct {
|
|||
char charset[TD_LOCALE_LEN]; // tsCharset
|
||||
} SClusterCfg;
|
||||
|
||||
typedef struct {
|
||||
int32_t openVnodes;
|
||||
int32_t totalVnodes;
|
||||
int32_t masterNum;
|
||||
int64_t numOfSelectReqs;
|
||||
int64_t numOfInsertReqs;
|
||||
int64_t numOfInsertSuccessReqs;
|
||||
int64_t numOfBatchInsertReqs;
|
||||
int64_t numOfBatchInsertSuccessReqs;
|
||||
int64_t errors;
|
||||
} SVnodesStat;
|
||||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int8_t role;
|
||||
|
|
@ -934,12 +984,12 @@ typedef struct SExplainExecInfo {
|
|||
uint64_t startupCost;
|
||||
uint64_t totalCost;
|
||||
uint64_t numOfRows;
|
||||
void *verboseInfo;
|
||||
void* verboseInfo;
|
||||
} SExplainExecInfo;
|
||||
|
||||
typedef struct {
|
||||
int32_t numOfPlans;
|
||||
SExplainExecInfo *subplanInfo;
|
||||
SExplainExecInfo* subplanInfo;
|
||||
} SExplainRsp;
|
||||
|
||||
int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
|
||||
|
|
@ -1866,6 +1916,37 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
|
|||
return buf;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int64_t leftForVer;
|
||||
int32_t vgId;
|
||||
int32_t epoch;
|
||||
int64_t consumerId;
|
||||
char topicName[TSDB_TOPIC_FNAME_LEN];
|
||||
} SMqCancelConnReq;
|
||||
|
||||
static FORCE_INLINE int32_t tEncodeSMqCancelConnReq(void** buf, const SMqCancelConnReq* pReq) {
|
||||
int32_t tlen = 0;
|
||||
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
|
||||
tlen += taosEncodeFixedI32(buf, pReq->vgId);
|
||||
tlen += taosEncodeFixedI32(buf, pReq->epoch);
|
||||
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
|
||||
tlen += taosEncodeString(buf, pReq->topicName);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void* tDecodeSMqCancelConnReq(void* buf, SMqCancelConnReq* pReq) {
|
||||
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
|
||||
buf = taosDecodeFixedI32(buf, &pReq->vgId);
|
||||
buf = taosDecodeFixedI32(buf, &pReq->epoch);
|
||||
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
|
||||
buf = taosDecodeStringTo(buf, pReq->topicName);
|
||||
return buf;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int8_t reserved;
|
||||
} SMqCancelConnRsp;
|
||||
|
||||
typedef struct {
|
||||
int64_t leftForVer;
|
||||
int32_t vgId;
|
||||
|
|
@ -2224,6 +2305,24 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
|
|||
return buf;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
char indexFName[TSDB_INDEX_FNAME_LEN];
|
||||
} SUserIndexReq;
|
||||
|
||||
int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq);
|
||||
int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
char dbFName[TSDB_DB_FNAME_LEN];
|
||||
char tblFName[TSDB_TABLE_FNAME_LEN];
|
||||
char colName[TSDB_COL_NAME_LEN];
|
||||
char indexType[TSDB_INDEX_TYPE_LEN];
|
||||
char indexExts[TSDB_INDEX_EXTS_LEN];
|
||||
} SUserIndexRsp;
|
||||
|
||||
int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp);
|
||||
int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
int8_t mqMsgType;
|
||||
int32_t code;
|
||||
|
|
@ -2241,6 +2340,7 @@ typedef struct {
|
|||
char cgroup[TSDB_CGROUP_LEN];
|
||||
|
||||
int64_t currentOffset;
|
||||
uint64_t reqId;
|
||||
char topic[TSDB_TOPIC_FNAME_LEN];
|
||||
} SMqPollReq;
|
||||
|
||||
|
|
|
|||
|
|
@ -155,6 +155,8 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "mnode-get-index", NULL, NULL)
|
||||
|
||||
// Requests handled by VNODE
|
||||
TD_NEW_MSG_SEG(TDMT_VND_MSG)
|
||||
|
|
@ -176,6 +178,7 @@ enum {
|
|||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
|
||||
|
|
@ -216,6 +219,15 @@ enum {
|
|||
TD_NEW_MSG_SEG(TDMT_SCH_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL)
|
||||
|
||||
// Monitor info exchange between processes
|
||||
TD_NEW_MSG_SEG(TDMT_MON_MSG)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_QM_INFO, "monitor-qinfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL)
|
||||
|
||||
#if defined(TD_MSG_NUMBER_)
|
||||
TDMT_MAX
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -69,153 +69,154 @@
|
|||
#define TK_CACHELAST 51
|
||||
#define TK_COMP 52
|
||||
#define TK_DAYS 53
|
||||
#define TK_FSYNC 54
|
||||
#define TK_MAXROWS 55
|
||||
#define TK_MINROWS 56
|
||||
#define TK_KEEP 57
|
||||
#define TK_PRECISION 58
|
||||
#define TK_QUORUM 59
|
||||
#define TK_REPLICA 60
|
||||
#define TK_TTL 61
|
||||
#define TK_WAL 62
|
||||
#define TK_VGROUPS 63
|
||||
#define TK_SINGLE_STABLE 64
|
||||
#define TK_STREAM_MODE 65
|
||||
#define TK_RETENTIONS 66
|
||||
#define TK_NK_COMMA 67
|
||||
#define TK_TABLE 68
|
||||
#define TK_NK_LP 69
|
||||
#define TK_NK_RP 70
|
||||
#define TK_STABLE 71
|
||||
#define TK_ADD 72
|
||||
#define TK_COLUMN 73
|
||||
#define TK_MODIFY 74
|
||||
#define TK_RENAME 75
|
||||
#define TK_TAG 76
|
||||
#define TK_SET 77
|
||||
#define TK_NK_EQ 78
|
||||
#define TK_USING 79
|
||||
#define TK_TAGS 80
|
||||
#define TK_NK_DOT 81
|
||||
#define TK_COMMENT 82
|
||||
#define TK_BOOL 83
|
||||
#define TK_TINYINT 84
|
||||
#define TK_SMALLINT 85
|
||||
#define TK_INT 86
|
||||
#define TK_INTEGER 87
|
||||
#define TK_BIGINT 88
|
||||
#define TK_FLOAT 89
|
||||
#define TK_DOUBLE 90
|
||||
#define TK_BINARY 91
|
||||
#define TK_TIMESTAMP 92
|
||||
#define TK_NCHAR 93
|
||||
#define TK_UNSIGNED 94
|
||||
#define TK_JSON 95
|
||||
#define TK_VARCHAR 96
|
||||
#define TK_MEDIUMBLOB 97
|
||||
#define TK_BLOB 98
|
||||
#define TK_VARBINARY 99
|
||||
#define TK_DECIMAL 100
|
||||
#define TK_SMA 101
|
||||
#define TK_ROLLUP 102
|
||||
#define TK_FILE_FACTOR 103
|
||||
#define TK_NK_FLOAT 104
|
||||
#define TK_DELAY 105
|
||||
#define TK_SHOW 106
|
||||
#define TK_DATABASES 107
|
||||
#define TK_TABLES 108
|
||||
#define TK_STABLES 109
|
||||
#define TK_MNODES 110
|
||||
#define TK_MODULES 111
|
||||
#define TK_QNODES 112
|
||||
#define TK_FUNCTIONS 113
|
||||
#define TK_INDEXES 114
|
||||
#define TK_FROM 115
|
||||
#define TK_ACCOUNTS 116
|
||||
#define TK_APPS 117
|
||||
#define TK_CONNECTIONS 118
|
||||
#define TK_LICENCE 119
|
||||
#define TK_QUERIES 120
|
||||
#define TK_SCORES 121
|
||||
#define TK_TOPICS 122
|
||||
#define TK_VARIABLES 123
|
||||
#define TK_LIKE 124
|
||||
#define TK_INDEX 125
|
||||
#define TK_FULLTEXT 126
|
||||
#define TK_FUNCTION 127
|
||||
#define TK_INTERVAL 128
|
||||
#define TK_TOPIC 129
|
||||
#define TK_AS 130
|
||||
#define TK_DESC 131
|
||||
#define TK_DESCRIBE 132
|
||||
#define TK_RESET 133
|
||||
#define TK_QUERY 134
|
||||
#define TK_EXPLAIN 135
|
||||
#define TK_ANALYZE 136
|
||||
#define TK_VERBOSE 137
|
||||
#define TK_NK_BOOL 138
|
||||
#define TK_RATIO 139
|
||||
#define TK_COMPACT 140
|
||||
#define TK_VNODES 141
|
||||
#define TK_IN 142
|
||||
#define TK_OUTPUTTYPE 143
|
||||
#define TK_AGGREGATE 144
|
||||
#define TK_BUFSIZE 145
|
||||
#define TK_STREAM 146
|
||||
#define TK_INTO 147
|
||||
#define TK_KILL 148
|
||||
#define TK_CONNECTION 149
|
||||
#define TK_MERGE 150
|
||||
#define TK_VGROUP 151
|
||||
#define TK_REDISTRIBUTE 152
|
||||
#define TK_SPLIT 153
|
||||
#define TK_SYNCDB 154
|
||||
#define TK_NULL 155
|
||||
#define TK_NK_VARIABLE 156
|
||||
#define TK_NOW 157
|
||||
#define TK_ROWTS 158
|
||||
#define TK_TBNAME 159
|
||||
#define TK_QSTARTTS 160
|
||||
#define TK_QENDTS 161
|
||||
#define TK_WSTARTTS 162
|
||||
#define TK_WENDTS 163
|
||||
#define TK_WDURATION 164
|
||||
#define TK_BETWEEN 165
|
||||
#define TK_IS 166
|
||||
#define TK_NK_LT 167
|
||||
#define TK_NK_GT 168
|
||||
#define TK_NK_LE 169
|
||||
#define TK_NK_GE 170
|
||||
#define TK_NK_NE 171
|
||||
#define TK_MATCH 172
|
||||
#define TK_NMATCH 173
|
||||
#define TK_JOIN 174
|
||||
#define TK_INNER 175
|
||||
#define TK_SELECT 176
|
||||
#define TK_DISTINCT 177
|
||||
#define TK_WHERE 178
|
||||
#define TK_PARTITION 179
|
||||
#define TK_BY 180
|
||||
#define TK_SESSION 181
|
||||
#define TK_STATE_WINDOW 182
|
||||
#define TK_SLIDING 183
|
||||
#define TK_FILL 184
|
||||
#define TK_VALUE 185
|
||||
#define TK_NONE 186
|
||||
#define TK_PREV 187
|
||||
#define TK_LINEAR 188
|
||||
#define TK_NEXT 189
|
||||
#define TK_GROUP 190
|
||||
#define TK_HAVING 191
|
||||
#define TK_ORDER 192
|
||||
#define TK_SLIMIT 193
|
||||
#define TK_SOFFSET 194
|
||||
#define TK_LIMIT 195
|
||||
#define TK_OFFSET 196
|
||||
#define TK_ASC 197
|
||||
#define TK_NULLS 198
|
||||
#define TK_FIRST 199
|
||||
#define TK_LAST 200
|
||||
#define TK_NK_VARIABLE 54
|
||||
#define TK_FSYNC 55
|
||||
#define TK_MAXROWS 56
|
||||
#define TK_MINROWS 57
|
||||
#define TK_KEEP 58
|
||||
#define TK_PRECISION 59
|
||||
#define TK_QUORUM 60
|
||||
#define TK_REPLICA 61
|
||||
#define TK_TTL 62
|
||||
#define TK_WAL 63
|
||||
#define TK_VGROUPS 64
|
||||
#define TK_SINGLE_STABLE 65
|
||||
#define TK_STREAM_MODE 66
|
||||
#define TK_RETENTIONS 67
|
||||
#define TK_NK_COMMA 68
|
||||
#define TK_NK_COLON 69
|
||||
#define TK_TABLE 70
|
||||
#define TK_NK_LP 71
|
||||
#define TK_NK_RP 72
|
||||
#define TK_STABLE 73
|
||||
#define TK_ADD 74
|
||||
#define TK_COLUMN 75
|
||||
#define TK_MODIFY 76
|
||||
#define TK_RENAME 77
|
||||
#define TK_TAG 78
|
||||
#define TK_SET 79
|
||||
#define TK_NK_EQ 80
|
||||
#define TK_USING 81
|
||||
#define TK_TAGS 82
|
||||
#define TK_NK_DOT 83
|
||||
#define TK_COMMENT 84
|
||||
#define TK_BOOL 85
|
||||
#define TK_TINYINT 86
|
||||
#define TK_SMALLINT 87
|
||||
#define TK_INT 88
|
||||
#define TK_INTEGER 89
|
||||
#define TK_BIGINT 90
|
||||
#define TK_FLOAT 91
|
||||
#define TK_DOUBLE 92
|
||||
#define TK_BINARY 93
|
||||
#define TK_TIMESTAMP 94
|
||||
#define TK_NCHAR 95
|
||||
#define TK_UNSIGNED 96
|
||||
#define TK_JSON 97
|
||||
#define TK_VARCHAR 98
|
||||
#define TK_MEDIUMBLOB 99
|
||||
#define TK_BLOB 100
|
||||
#define TK_VARBINARY 101
|
||||
#define TK_DECIMAL 102
|
||||
#define TK_SMA 103
|
||||
#define TK_ROLLUP 104
|
||||
#define TK_FILE_FACTOR 105
|
||||
#define TK_NK_FLOAT 106
|
||||
#define TK_DELAY 107
|
||||
#define TK_SHOW 108
|
||||
#define TK_DATABASES 109
|
||||
#define TK_TABLES 110
|
||||
#define TK_STABLES 111
|
||||
#define TK_MNODES 112
|
||||
#define TK_MODULES 113
|
||||
#define TK_QNODES 114
|
||||
#define TK_FUNCTIONS 115
|
||||
#define TK_INDEXES 116
|
||||
#define TK_FROM 117
|
||||
#define TK_ACCOUNTS 118
|
||||
#define TK_APPS 119
|
||||
#define TK_CONNECTIONS 120
|
||||
#define TK_LICENCE 121
|
||||
#define TK_QUERIES 122
|
||||
#define TK_SCORES 123
|
||||
#define TK_TOPICS 124
|
||||
#define TK_VARIABLES 125
|
||||
#define TK_LIKE 126
|
||||
#define TK_INDEX 127
|
||||
#define TK_FULLTEXT 128
|
||||
#define TK_FUNCTION 129
|
||||
#define TK_INTERVAL 130
|
||||
#define TK_TOPIC 131
|
||||
#define TK_AS 132
|
||||
#define TK_DESC 133
|
||||
#define TK_DESCRIBE 134
|
||||
#define TK_RESET 135
|
||||
#define TK_QUERY 136
|
||||
#define TK_EXPLAIN 137
|
||||
#define TK_ANALYZE 138
|
||||
#define TK_VERBOSE 139
|
||||
#define TK_NK_BOOL 140
|
||||
#define TK_RATIO 141
|
||||
#define TK_COMPACT 142
|
||||
#define TK_VNODES 143
|
||||
#define TK_IN 144
|
||||
#define TK_OUTPUTTYPE 145
|
||||
#define TK_AGGREGATE 146
|
||||
#define TK_BUFSIZE 147
|
||||
#define TK_STREAM 148
|
||||
#define TK_INTO 149
|
||||
#define TK_KILL 150
|
||||
#define TK_CONNECTION 151
|
||||
#define TK_MERGE 152
|
||||
#define TK_VGROUP 153
|
||||
#define TK_REDISTRIBUTE 154
|
||||
#define TK_SPLIT 155
|
||||
#define TK_SYNCDB 156
|
||||
#define TK_NULL 157
|
||||
#define TK_FIRST 158
|
||||
#define TK_LAST 159
|
||||
#define TK_NOW 160
|
||||
#define TK_ROWTS 161
|
||||
#define TK_TBNAME 162
|
||||
#define TK_QSTARTTS 163
|
||||
#define TK_QENDTS 164
|
||||
#define TK_WSTARTTS 165
|
||||
#define TK_WENDTS 166
|
||||
#define TK_WDURATION 167
|
||||
#define TK_BETWEEN 168
|
||||
#define TK_IS 169
|
||||
#define TK_NK_LT 170
|
||||
#define TK_NK_GT 171
|
||||
#define TK_NK_LE 172
|
||||
#define TK_NK_GE 173
|
||||
#define TK_NK_NE 174
|
||||
#define TK_MATCH 175
|
||||
#define TK_NMATCH 176
|
||||
#define TK_JOIN 177
|
||||
#define TK_INNER 178
|
||||
#define TK_SELECT 179
|
||||
#define TK_DISTINCT 180
|
||||
#define TK_WHERE 181
|
||||
#define TK_PARTITION 182
|
||||
#define TK_BY 183
|
||||
#define TK_SESSION 184
|
||||
#define TK_STATE_WINDOW 185
|
||||
#define TK_SLIDING 186
|
||||
#define TK_FILL 187
|
||||
#define TK_VALUE 188
|
||||
#define TK_NONE 189
|
||||
#define TK_PREV 190
|
||||
#define TK_LINEAR 191
|
||||
#define TK_NEXT 192
|
||||
#define TK_GROUP 193
|
||||
#define TK_HAVING 194
|
||||
#define TK_ORDER 195
|
||||
#define TK_SLIMIT 196
|
||||
#define TK_SOFFSET 197
|
||||
#define TK_LIMIT 198
|
||||
#define TK_OFFSET 199
|
||||
#define TK_ASC 200
|
||||
#define TK_NULLS 201
|
||||
|
||||
#define TK_NK_SPACE 300
|
||||
#define TK_NK_COMMENT 301
|
||||
|
|
@ -226,7 +227,6 @@
|
|||
#define TK_NK_FILE 306
|
||||
#define TK_NK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
|
||||
|
||||
#define TK_NK_COLON 500
|
||||
#define TK_NK_BITNOT 501
|
||||
#define TK_INSERT 502
|
||||
#define TK_VALUES 507
|
||||
|
|
|
|||
|
|
@ -77,6 +77,9 @@ typedef struct SDbVgVersion {
|
|||
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
|
||||
} SDbVgVersion;
|
||||
|
||||
typedef SDbCfgRsp SDbCfgInfo;
|
||||
typedef SUserIndexRsp SIndexInfo;
|
||||
|
||||
int32_t catalogInit(SCatalogCfg *cfg);
|
||||
|
||||
/**
|
||||
|
|
@ -217,6 +220,10 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable
|
|||
|
||||
int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num);
|
||||
|
||||
int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg);
|
||||
|
||||
int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
|
||||
|
||||
|
||||
/**
|
||||
* Destroy catalog and relase all resources
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "tarray.h"
|
||||
#include "tdef.h"
|
||||
#include "tlog.h"
|
||||
#include "tmsg.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -29,6 +30,44 @@ extern "C" {
|
|||
#define MON_VER_LEN 12
|
||||
#define MON_LOG_LEN 1024
|
||||
|
||||
typedef struct {
|
||||
int64_t ts;
|
||||
ELogLevel level;
|
||||
char content[MON_LOG_LEN];
|
||||
} SMonLogItem;
|
||||
|
||||
typedef struct {
|
||||
SArray *logs; // array of SMonLogItem
|
||||
int32_t numOfErrorLogs;
|
||||
int32_t numOfInfoLogs;
|
||||
int32_t numOfDebugLogs;
|
||||
int32_t numOfTraceLogs;
|
||||
} SMonLogs;
|
||||
|
||||
typedef struct {
|
||||
char name[TSDB_FILENAME_LEN];
|
||||
int8_t level;
|
||||
SDiskSize size;
|
||||
} SMonDiskDesc;
|
||||
|
||||
typedef struct {
|
||||
double cpu_engine;
|
||||
double cpu_system;
|
||||
float cpu_cores;
|
||||
int64_t mem_engine; // KB
|
||||
int64_t mem_system; // KB
|
||||
int64_t mem_total; // KB
|
||||
int64_t disk_engine; // Byte
|
||||
int64_t disk_used; // Byte
|
||||
int64_t disk_total; // Byte
|
||||
int64_t net_in; // bytes
|
||||
int64_t net_out; // bytes
|
||||
int64_t io_read; // bytes
|
||||
int64_t io_write; // bytes
|
||||
int64_t io_read_disk; // bytes
|
||||
int64_t io_write_disk; // bytes
|
||||
} SMonSysInfo;
|
||||
|
||||
typedef struct {
|
||||
int32_t dnode_id;
|
||||
char dnode_ep[TSDB_EP_LEN];
|
||||
|
|
@ -36,6 +75,19 @@ typedef struct {
|
|||
int32_t protocol;
|
||||
} SMonBasicInfo;
|
||||
|
||||
typedef struct {
|
||||
float uptime; // day
|
||||
int8_t has_mnode;
|
||||
SMonDiskDesc logdir;
|
||||
SMonDiskDesc tempdir;
|
||||
} SMonDnodeInfo;
|
||||
|
||||
typedef struct {
|
||||
SMonBasicInfo basic;
|
||||
SMonDnodeInfo dnode;
|
||||
SMonSysInfo sys;
|
||||
} SMonDmInfo;
|
||||
|
||||
typedef struct {
|
||||
int32_t dnode_id;
|
||||
char dnode_ep[TSDB_EP_LEN];
|
||||
|
|
@ -87,46 +139,65 @@ typedef struct {
|
|||
} SMonGrantInfo;
|
||||
|
||||
typedef struct {
|
||||
float uptime; // day
|
||||
double cpu_engine;
|
||||
double cpu_system;
|
||||
float cpu_cores;
|
||||
int64_t mem_engine; // KB
|
||||
int64_t mem_system; // KB
|
||||
int64_t mem_total; // KB
|
||||
int64_t disk_engine; // Byte
|
||||
int64_t disk_used; // Byte
|
||||
int64_t disk_total; // Byte
|
||||
int64_t net_in; // bytes
|
||||
int64_t net_out; // bytes
|
||||
int64_t io_read; // bytes
|
||||
int64_t io_write; // bytes
|
||||
int64_t io_read_disk; // bytes
|
||||
int64_t io_write_disk; // bytes
|
||||
int64_t req_select;
|
||||
int64_t req_insert;
|
||||
int64_t req_insert_success;
|
||||
int64_t req_insert_batch;
|
||||
int64_t req_insert_batch_success;
|
||||
int32_t errors;
|
||||
int32_t vnodes_num;
|
||||
int32_t masters;
|
||||
int8_t has_mnode;
|
||||
} SMonDnodeInfo;
|
||||
SMonClusterInfo cluster;
|
||||
SMonVgroupInfo vgroup;
|
||||
SMonGrantInfo grant;
|
||||
SMonSysInfo sys;
|
||||
SMonLogs log;
|
||||
} SMonMmInfo;
|
||||
|
||||
typedef struct {
|
||||
char name[TSDB_FILENAME_LEN];
|
||||
int8_t level;
|
||||
SDiskSize size;
|
||||
} SMonDiskDesc;
|
||||
int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo);
|
||||
int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo);
|
||||
void tFreeSMonMmInfo(SMonMmInfo *pInfo);
|
||||
|
||||
typedef struct {
|
||||
SArray *datadirs; // array of SMonDiskDesc
|
||||
SMonDiskDesc logdir;
|
||||
SMonDiskDesc tempdir;
|
||||
} SMonDiskInfo;
|
||||
|
||||
typedef struct SMonInfo SMonInfo;
|
||||
typedef struct {
|
||||
SMonDiskInfo tfs;
|
||||
SVnodesStat vstat;
|
||||
SMonSysInfo sys;
|
||||
SMonLogs log;
|
||||
} SMonVmInfo;
|
||||
|
||||
int32_t tSerializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo);
|
||||
int32_t tDeserializeSMonVmInfo(void *buf, int32_t bufLen, SMonVmInfo *pInfo);
|
||||
void tFreeSMonVmInfo(SMonVmInfo *pInfo);
|
||||
|
||||
typedef struct {
|
||||
SMonSysInfo sys;
|
||||
SMonLogs log;
|
||||
} SMonQmInfo;
|
||||
|
||||
int32_t tSerializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo);
|
||||
int32_t tDeserializeSMonQmInfo(void *buf, int32_t bufLen, SMonQmInfo *pInfo);
|
||||
void tFreeSMonQmInfo(SMonQmInfo *pInfo);
|
||||
|
||||
typedef struct {
|
||||
SMonSysInfo sys;
|
||||
SMonLogs log;
|
||||
} SMonSmInfo;
|
||||
|
||||
int32_t tSerializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo);
|
||||
int32_t tDeserializeSMonSmInfo(void *buf, int32_t bufLen, SMonSmInfo *pInfo);
|
||||
void tFreeSMonSmInfo(SMonSmInfo *pInfo);
|
||||
typedef struct {
|
||||
SMonSysInfo sys;
|
||||
SMonLogs log;
|
||||
} SMonBmInfo;
|
||||
|
||||
int32_t tSerializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo);
|
||||
int32_t tDeserializeSMonBmInfo(void *buf, int32_t bufLen, SMonBmInfo *pInfo);
|
||||
void tFreeSMonBmInfo(SMonBmInfo *pInfo);
|
||||
|
||||
typedef struct {
|
||||
SArray *pVloads; // SVnodeLoad
|
||||
} SMonVloadInfo;
|
||||
|
||||
int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo);
|
||||
int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo);
|
||||
void tFreeSMonVloadInfo(SMonVloadInfo *pInfo);
|
||||
|
||||
typedef struct {
|
||||
const char *server;
|
||||
|
|
@ -138,16 +209,14 @@ typedef struct {
|
|||
int32_t monInit(const SMonCfg *pCfg);
|
||||
void monCleanup();
|
||||
void monRecordLog(int64_t ts, ELogLevel level, const char *content);
|
||||
|
||||
SMonInfo *monCreateMonitorInfo();
|
||||
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);
|
||||
void monSetClusterInfo(SMonInfo *pMonitor, SMonClusterInfo *pInfo);
|
||||
void monSetVgroupInfo(SMonInfo *pMonitor, SMonVgroupInfo *pInfo);
|
||||
void monSetGrantInfo(SMonInfo *pMonitor, SMonGrantInfo *pInfo);
|
||||
void monSetDnodeInfo(SMonInfo *pMonitor, SMonDnodeInfo *pInfo);
|
||||
void monSetDiskInfo(SMonInfo *pMonitor, SMonDiskInfo *pInfo);
|
||||
void monSendReport(SMonInfo *pMonitor);
|
||||
void monCleanupMonitorInfo(SMonInfo *pMonitor);
|
||||
int32_t monGetLogs(SMonLogs *logs);
|
||||
void monSetDmInfo(SMonDmInfo *pInfo);
|
||||
void monSetMmInfo(SMonMmInfo *pInfo);
|
||||
void monSetVmInfo(SMonVmInfo *pInfo);
|
||||
void monSetQmInfo(SMonQmInfo *pInfo);
|
||||
void monSetSmInfo(SMonSmInfo *pInfo);
|
||||
void monSetBmInfo(SMonBmInfo *pInfo);
|
||||
void monSendReport();
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,25 +30,23 @@ extern "C" {
|
|||
|
||||
typedef struct SDatabaseOptions {
|
||||
ENodeType type;
|
||||
int32_t numOfBlocks;
|
||||
int32_t cacheBlockSize;
|
||||
int8_t cachelast;
|
||||
int32_t compressionLevel;
|
||||
int32_t daysPerFile;
|
||||
int32_t fsyncPeriod;
|
||||
int32_t maxRowsPerBlock;
|
||||
int32_t minRowsPerBlock;
|
||||
int32_t keep0;
|
||||
int32_t keep1;
|
||||
int32_t keep2;
|
||||
int32_t precision;
|
||||
int32_t quorum;
|
||||
int32_t replica;
|
||||
int32_t ttl;
|
||||
int32_t walLevel;
|
||||
int32_t numOfVgroups;
|
||||
int8_t singleStable;
|
||||
int8_t streamMode;
|
||||
SValueNode* pNumOfBlocks;
|
||||
SValueNode* pCacheBlockSize;
|
||||
SValueNode* pCachelast;
|
||||
SValueNode* pCompressionLevel;
|
||||
SValueNode* pDaysPerFile;
|
||||
SValueNode* pFsyncPeriod;
|
||||
SValueNode* pMaxRowsPerBlock;
|
||||
SValueNode* pMinRowsPerBlock;
|
||||
SNodeList* pKeep;
|
||||
SValueNode* pPrecision;
|
||||
SValueNode* pQuorum;
|
||||
SValueNode* pReplica;
|
||||
SValueNode* pTtl;
|
||||
SValueNode* pWalLevel;
|
||||
SValueNode* pNumOfVgroups;
|
||||
SValueNode* pSingleStable;
|
||||
SValueNode* pStreamMode;
|
||||
SNodeList* pRetentions;
|
||||
} SDatabaseOptions;
|
||||
|
||||
|
|
@ -78,15 +76,13 @@ typedef struct SAlterDatabaseStmt {
|
|||
|
||||
typedef struct STableOptions {
|
||||
ENodeType type;
|
||||
int32_t keep0;
|
||||
int32_t keep1;
|
||||
int32_t keep2;
|
||||
int32_t ttl;
|
||||
char comments[TSDB_STB_COMMENT_LEN];
|
||||
SNodeList* pKeep;
|
||||
SValueNode* pTtl;
|
||||
SValueNode* pComments;
|
||||
SNodeList* pSma;
|
||||
SNodeList* pFuncs;
|
||||
float filesFactor;
|
||||
int32_t delay;
|
||||
SValueNode* pFilesFactor;
|
||||
SValueNode* pDelay;
|
||||
} STableOptions;
|
||||
|
||||
typedef struct SColumnDefNode {
|
||||
|
|
|
|||
|
|
@ -30,11 +30,19 @@ extern "C" {
|
|||
#define FOREACH(node, list) \
|
||||
for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext)
|
||||
|
||||
// only be use in FOREACH
|
||||
#define ERASE_NODE(list) cell = nodesListErase(list, cell);
|
||||
|
||||
#define REPLACE_NODE(newNode) cell->pNode = (SNode*)(newNode)
|
||||
|
||||
#define INSERT_LIST(target, src) nodesListInsertList((target), cell, src)
|
||||
|
||||
#define WHERE_EACH(node, list) \
|
||||
SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \
|
||||
while (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false))
|
||||
|
||||
#define WHERE_NEXT cell = cell->pNext
|
||||
|
||||
// only be use in WHERE_EACH
|
||||
#define ERASE_NODE(list) cell = nodesListErase((list), cell)
|
||||
|
||||
#define FORBOTH(node1, list1, node2, list2) \
|
||||
for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \
|
||||
(NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
|
||||
|
|
@ -202,7 +210,9 @@ int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
|
|||
int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
|
||||
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
|
||||
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
|
||||
int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode);
|
||||
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
|
||||
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
|
||||
SNodeptr nodesListGetNode(SNodeList* pList, int32_t index);
|
||||
void nodesDestroyList(SNodeList* pList);
|
||||
// Only clear the linked list structure, without releasing the elements inside
|
||||
|
|
|
|||
|
|
@ -286,12 +286,14 @@ typedef struct SSortPhysiNode {
|
|||
SPhysiNode node;
|
||||
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
|
||||
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
|
||||
SNodeList* pTargets;
|
||||
} SSortPhysiNode;
|
||||
|
||||
typedef struct SPartitionPhysiNode {
|
||||
SPhysiNode node;
|
||||
SNodeList* pExprs; // these are expression list of partition_by_clause
|
||||
SNodeList* pPartitionKeys;
|
||||
SNodeList* pTargets;
|
||||
} SPartitionPhysiNode;
|
||||
|
||||
typedef struct SDataSinkNode {
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ typedef struct STableNode {
|
|||
char dbName[TSDB_DB_NAME_LEN];
|
||||
char tableName[TSDB_TABLE_NAME_LEN];
|
||||
char tableAlias[TSDB_TABLE_NAME_LEN];
|
||||
uint8_t precision;
|
||||
} STableNode;
|
||||
|
||||
struct STableMeta;
|
||||
|
|
@ -236,6 +237,8 @@ typedef struct SSelectStmt {
|
|||
SNode* pLimit;
|
||||
SNode* pSlimit;
|
||||
char stmtName[TSDB_TABLE_NAME_LEN];
|
||||
uint8_t precision;
|
||||
bool isEmptyResult;
|
||||
} SSelectStmt;
|
||||
|
||||
typedef enum ESetOperatorType {
|
||||
|
|
@ -258,6 +261,7 @@ typedef enum ESqlClause {
|
|||
SQL_CLAUSE_WINDOW,
|
||||
SQL_CLAUSE_GROUP_BY,
|
||||
SQL_CLAUSE_HAVING,
|
||||
SQL_CLAUSE_DISTINCT,
|
||||
SQL_CLAUSE_SELECT,
|
||||
SQL_CLAUSE_ORDER_BY
|
||||
} ESqlClause;
|
||||
|
|
|
|||
|
|
@ -44,8 +44,15 @@ typedef struct SCmdMsgInfo {
|
|||
void* pExtension; // todo remove it soon
|
||||
} SCmdMsgInfo;
|
||||
|
||||
typedef enum EQueryExecMode {
|
||||
QUERY_EXEC_MODE_LOCAL = 1,
|
||||
QUERY_EXEC_MODE_RPC,
|
||||
QUERY_EXEC_MODE_SCHEDULE,
|
||||
QUERY_EXEC_MODE_EMPTY_RESULT
|
||||
} EQueryExecMode;
|
||||
|
||||
typedef struct SQuery {
|
||||
bool directRpc;
|
||||
EQueryExecMode execMode;
|
||||
bool haveResultSet;
|
||||
SNode* pRoot;
|
||||
int32_t numOfResCols;
|
||||
|
|
@ -55,7 +62,6 @@ typedef struct SQuery {
|
|||
SArray* pDbList;
|
||||
SArray* pTableList;
|
||||
bool showRewrite;
|
||||
bool localCmd;
|
||||
} SQuery;
|
||||
|
||||
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
|
||||
|
|
|
|||
|
|
@ -56,6 +56,12 @@ extern "C" {
|
|||
__typeof(b) __b = (b); \
|
||||
(__a < __b) ? __a : __b; \
|
||||
})
|
||||
|
||||
#define TRANGE(a, b, c) \
|
||||
({ \
|
||||
a = TMAX(a, b); \
|
||||
a = TMIN(a, c); \
|
||||
})
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -44,8 +44,8 @@ int32_t taosGetTotalMemory(int64_t *totalKB);
|
|||
int32_t taosGetProcMemory(int64_t *usedKB);
|
||||
int32_t taosGetSysMemory(int64_t *usedKB);
|
||||
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
|
||||
int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
|
||||
int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
|
||||
void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
|
||||
void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes);
|
||||
|
||||
void taosKillSystem();
|
||||
int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
|
||||
|
|
|
|||
|
|
@ -217,6 +217,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385)
|
||||
#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0386)
|
||||
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0387)
|
||||
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0388)
|
||||
|
||||
// mnode-vgroup
|
||||
#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390)
|
||||
|
|
@ -482,6 +483,18 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_PAR_DB_NOT_SPECIFIED TAOS_DEF_ERROR_CODE(0, 0x2616)
|
||||
#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME TAOS_DEF_ERROR_CODE(0, 0x2617)
|
||||
#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR TAOS_DEF_ERROR_CODE(0, 0x2618)
|
||||
#define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619)
|
||||
#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A)
|
||||
#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B)
|
||||
#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x261C)
|
||||
#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D)
|
||||
#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E)
|
||||
#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F)
|
||||
#define TSDB_CODE_PAR_INVALID_COMMENT_OPTION TAOS_DEF_ERROR_CODE(0, 0x2620)
|
||||
#define TSDB_CODE_PAR_INVALID_F_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2621)
|
||||
#define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION TAOS_DEF_ERROR_CODE(0, 0x2622)
|
||||
#define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2623)
|
||||
#define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624)
|
||||
|
||||
//planner
|
||||
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
|
||||
|
|
|
|||
|
|
@ -109,6 +109,9 @@ extern const int32_t TYPE_BYTES[15];
|
|||
#define TSDB_INS_TABLE_USER_USERS "user_users"
|
||||
#define TSDB_INS_TABLE_VGROUPS "vgroups"
|
||||
|
||||
#define TSDB_INDEX_TYPE_SMA "SMA"
|
||||
#define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
|
||||
|
||||
#define TSDB_INS_USER_STABLES_DBNAME_COLID 2
|
||||
|
||||
#define TSDB_TICK_PER_SECOND(precision) \
|
||||
|
|
@ -213,6 +216,9 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_FUNC_MAX_RETRIEVE 1024
|
||||
|
||||
#define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0'
|
||||
#define TSDB_INDEX_TYPE_LEN 10
|
||||
#define TSDB_INDEX_EXTS_LEN 256
|
||||
#define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
|
||||
#define TSDB_TYPE_STR_MAX_LEN 32
|
||||
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
|
||||
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
|
||||
|
|
@ -472,9 +478,6 @@ typedef struct {
|
|||
|
||||
#define TMQ_SEPARATOR ':'
|
||||
|
||||
#define SND_UNIQUE_THREAD_NUM 2
|
||||
#define SND_SHARED_THREAD_NUM 2
|
||||
|
||||
enum {
|
||||
SND_WORKER_TYPE__SHARED = 1,
|
||||
SND_WORKER_TYPE__UNIQUE,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,536 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This file is used to install database on linux systems. The operating system
|
||||
# is required to use systemd to manage services at boot
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
# -----------------------Variables definition---------------------
|
||||
script_dir=$(dirname $(readlink -f "$0"))
|
||||
# Dynamic directory
|
||||
data_dir="/var/lib/taos"
|
||||
log_dir="/var/log/taos"
|
||||
|
||||
data_link_dir="/usr/local/taos/data"
|
||||
log_link_dir="/usr/local/taos/log"
|
||||
|
||||
cfg_install_dir="/etc/taos"
|
||||
|
||||
bin_link_dir="/usr/bin"
|
||||
lib_link_dir="/usr/lib"
|
||||
lib64_link_dir="/usr/lib64"
|
||||
inc_link_dir="/usr/include"
|
||||
|
||||
#install main path
|
||||
install_main_dir="/usr/local/taos"
|
||||
|
||||
# old bin dir
|
||||
bin_dir="/usr/local/taos/bin"
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
|
||||
# Color setting
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[1;32m'
|
||||
GREEN_DARK='\033[0;32m'
|
||||
GREEN_UNDERLINE='\033[4;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo"
|
||||
fi
|
||||
|
||||
update_flag=0
|
||||
prompt_force=0
|
||||
|
||||
initd_mod=0
|
||||
service_mod=2
|
||||
if pidof systemd &> /dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &> /dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &> /dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &> /dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &> /dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
|
||||
|
||||
# get the operating system type for using the corresponding init file
|
||||
# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification
|
||||
#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
|
||||
if [[ -e /etc/os-release ]]; then
|
||||
osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||:
|
||||
else
|
||||
osinfo=""
|
||||
fi
|
||||
#echo "osinfo: ${osinfo}"
|
||||
os_type=0
|
||||
if echo $osinfo | grep -qwi "ubuntu" ; then
|
||||
# echo "This is ubuntu system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "debian" ; then
|
||||
# echo "This is debian system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "Kylin" ; then
|
||||
# echo "This is Kylin system"
|
||||
os_type=1
|
||||
elif echo $osinfo | grep -qwi "centos" ; then
|
||||
# echo "This is centos system"
|
||||
os_type=2
|
||||
elif echo $osinfo | grep -qwi "fedora" ; then
|
||||
# echo "This is fedora system"
|
||||
os_type=2
|
||||
elif echo $osinfo | grep -qwi "Linx" ; then
|
||||
# echo "This is Linx system"
|
||||
os_type=1
|
||||
service_mod=0
|
||||
initd_mod=0
|
||||
service_config_dir="/etc/systemd/system"
|
||||
else
|
||||
echo " osinfo: ${osinfo}"
|
||||
echo " This is an officially unverified linux system,"
|
||||
echo " if there are any problems with the installation and operation, "
|
||||
echo " please feel free to contact taosdata.com for support."
|
||||
os_type=1
|
||||
fi
|
||||
|
||||
|
||||
# ============================= get input parameters =================================================
|
||||
|
||||
# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...]
|
||||
|
||||
# set parameters by default value
|
||||
interactiveFqdn=yes # [yes | no]
|
||||
verType=server # [server | client]
|
||||
initType=systemd # [systemd | service | ...]
|
||||
|
||||
while getopts "hv:e:i:" arg
|
||||
do
|
||||
case $arg in
|
||||
e)
|
||||
#echo "interactiveFqdn=$OPTARG"
|
||||
interactiveFqdn=$( echo $OPTARG )
|
||||
;;
|
||||
v)
|
||||
#echo "verType=$OPTARG"
|
||||
verType=$(echo $OPTARG)
|
||||
;;
|
||||
i)
|
||||
#echo "initType=$OPTARG"
|
||||
initType=$(echo $OPTARG)
|
||||
;;
|
||||
h)
|
||||
echo "Usage: `basename $0` -v [server | client] -e [yes | no]"
|
||||
exit 0
|
||||
;;
|
||||
?) #unknow option
|
||||
echo "unkonw argument"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
|
||||
|
||||
function kill_process() {
|
||||
pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo} kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
function install_main_path() {
|
||||
#create install main dir and all sub dir
|
||||
${csudo} rm -rf ${install_main_dir} || :
|
||||
${csudo} mkdir -p ${install_main_dir}
|
||||
${csudo} mkdir -p ${install_main_dir}/cfg
|
||||
${csudo} mkdir -p ${install_main_dir}/bin
|
||||
${csudo} mkdir -p ${install_main_dir}/connector
|
||||
${csudo} mkdir -p ${install_main_dir}/driver
|
||||
${csudo} mkdir -p ${install_main_dir}/examples
|
||||
${csudo} mkdir -p ${install_main_dir}/include
|
||||
${csudo} mkdir -p ${install_main_dir}/init.d
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
${csudo} mkdir -p ${nginx_dir}
|
||||
fi
|
||||
|
||||
if [[ -e ${script_dir}/email ]]; then
|
||||
${csudo} cp ${script_dir}/email ${install_main_dir}/ ||:
|
||||
fi
|
||||
}
|
||||
|
||||
function install_bin() {
|
||||
# Remove links
|
||||
${csudo} rm -f ${bin_link_dir}/taos || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosd || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosadapter || :
|
||||
${csudo} rm -f ${bin_link_dir}/create_table || :
|
||||
${csudo} rm -f ${bin_link_dir}/tmq_sim || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo} rm -f ${bin_link_dir}/rmtaos || :
|
||||
#${csudo} rm -f ${bin_link_dir}/set_core || :
|
||||
|
||||
${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/*
|
||||
|
||||
#Make link
|
||||
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
|
||||
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || :
|
||||
[ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table || :
|
||||
[ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim || :
|
||||
# [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
|
||||
# [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
|
||||
# [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
|
||||
}
|
||||
|
||||
function install_lib() {
|
||||
# Remove links
|
||||
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
|
||||
${csudo} rm -f ${lib64_link_dir}/libtaos.* || :
|
||||
|
||||
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
|
||||
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
|
||||
|
||||
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib_link_dir}/libtdb.so.1
|
||||
${csudo} ln -s ${lib_link_dir}/libtdb.so.1 ${lib_link_dir}/libtdb.so
|
||||
|
||||
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
|
||||
${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
|
||||
${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
|
||||
|
||||
${csudo} ln -s ${install_main_dir}/lib/libtdb.* ${lib64_link_dir}/libtdb.so.1 || :
|
||||
${csudo} ln -s ${lib64_link_dir}/libtdb.so.1 ${lib64_link_dir}/libtdb.so || :
|
||||
fi
|
||||
|
||||
${csudo} ldconfig
|
||||
}
|
||||
|
||||
function install_header() {
|
||||
${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
|
||||
${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/*
|
||||
${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
|
||||
# ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
|
||||
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
|
||||
}
|
||||
|
||||
function add_newHostname_to_hosts() {
|
||||
localIp="127.0.0.1"
|
||||
OLD_IFS="$IFS"
|
||||
IFS=" "
|
||||
iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
|
||||
arr=($iphost)
|
||||
IFS="$OLD_IFS"
|
||||
for s in "${arr[@]}"
|
||||
do
|
||||
if [[ "$s" == "$localIp" ]]; then
|
||||
return
|
||||
fi
|
||||
done
|
||||
${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||:
|
||||
}
|
||||
|
||||
function set_hostname() {
|
||||
echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:"
|
||||
read newHostname
|
||||
while true; do
|
||||
if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then
|
||||
break
|
||||
else
|
||||
read -p "Please enter one hostname(must not be 'localhost'):" newHostname
|
||||
fi
|
||||
done
|
||||
|
||||
${csudo} hostname $newHostname ||:
|
||||
retval=`echo $?`
|
||||
if [[ $retval != 0 ]]; then
|
||||
echo
|
||||
echo "set hostname fail!"
|
||||
return
|
||||
fi
|
||||
#echo -e -n "$(hostnamectl status --static)"
|
||||
#echo -e -n "$(hostnamectl status --transient)"
|
||||
#echo -e -n "$(hostnamectl status --pretty)"
|
||||
|
||||
#ubuntu/centos /etc/hostname
|
||||
if [[ -e /etc/hostname ]]; then
|
||||
${csudo} echo $newHostname > /etc/hostname ||:
|
||||
fi
|
||||
|
||||
#debian: #HOSTNAME=yourname
|
||||
if [[ -e /etc/sysconfig/network ]]; then
|
||||
${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||:
|
||||
fi
|
||||
|
||||
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
|
||||
serverFqdn=$newHostname
|
||||
|
||||
if [[ -e /etc/hosts ]]; then
|
||||
add_newHostname_to_hosts $newHostname
|
||||
fi
|
||||
}
|
||||
|
||||
function is_correct_ipaddr() {
|
||||
newIp=$1
|
||||
OLD_IFS="$IFS"
|
||||
IFS=" "
|
||||
arr=($iplist)
|
||||
IFS="$OLD_IFS"
|
||||
for s in "${arr[@]}"
|
||||
do
|
||||
if [[ "$s" == "$newIp" ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
function set_ipAsFqdn() {
|
||||
iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||:
|
||||
if [ -z "$iplist" ]; then
|
||||
iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||:
|
||||
fi
|
||||
|
||||
if [ -z "$iplist" ]; then
|
||||
echo
|
||||
echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
|
||||
localFqdn="127.0.0.1"
|
||||
# Write the local FQDN to configuration file
|
||||
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
|
||||
serverFqdn=$localFqdn
|
||||
echo
|
||||
return
|
||||
fi
|
||||
|
||||
echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:"
|
||||
echo
|
||||
echo -e -n "${GREEN}$iplist${NC}"
|
||||
echo
|
||||
echo
|
||||
echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:"
|
||||
read localFqdn
|
||||
while true; do
|
||||
if [ ! -z "$localFqdn" ]; then
|
||||
# Check if correct ip address
|
||||
is_correct_ipaddr $localFqdn
|
||||
retval=`echo $?`
|
||||
if [[ $retval != 0 ]]; then
|
||||
read -p "Please choose an IP from local IP list:" localFqdn
|
||||
else
|
||||
# Write the local FQDN to configuration file
|
||||
${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
|
||||
serverFqdn=$localFqdn
|
||||
break
|
||||
fi
|
||||
else
|
||||
read -p "Please choose an IP from local IP list:" localFqdn
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function local_fqdn_check() {
|
||||
#serverFqdn=$(hostname)
|
||||
echo
|
||||
echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}"
|
||||
echo
|
||||
if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then
|
||||
echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
|
||||
echo
|
||||
|
||||
while true
|
||||
do
|
||||
read -r -p "Set hostname now? [Y/n] " input
|
||||
if [ ! -n "$input" ]; then
|
||||
set_hostname
|
||||
break
|
||||
else
|
||||
case $input in
|
||||
[yY][eE][sS]|[yY])
|
||||
set_hostname
|
||||
break
|
||||
;;
|
||||
|
||||
[nN][oO]|[nN])
|
||||
set_ipAsFqdn
|
||||
break
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Invalid input..."
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
function install_log() {
|
||||
${csudo} rm -rf ${log_dir} || :
|
||||
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
|
||||
|
||||
${csudo} ln -s ${log_dir} ${install_main_dir}/log
|
||||
}
|
||||
|
||||
function install_data() {
|
||||
${csudo} mkdir -p ${data_dir}
|
||||
|
||||
${csudo} ln -s ${data_dir} ${install_main_dir}/data
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
taosd_service_config="${service_config_dir}/taosd.service"
|
||||
if systemctl is-active --quiet taosd; then
|
||||
echo "TDengine is running, stopping it..."
|
||||
${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null
|
||||
fi
|
||||
${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null
|
||||
${csudo} rm -f ${taosd_service_config}
|
||||
|
||||
tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
|
||||
if systemctl is-active --quiet tarbitratord; then
|
||||
echo "tarbitrator is running, stopping it..."
|
||||
${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null
|
||||
fi
|
||||
${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null
|
||||
${csudo} rm -f ${tarbitratord_service_config}
|
||||
|
||||
if [ "$verMode" == "cluster" ]; then
|
||||
nginx_service_config="${service_config_dir}/nginxd.service"
|
||||
if systemctl is-active --quiet nginxd; then
|
||||
echo "Nginx for TDengine is running, stopping it..."
|
||||
${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null
|
||||
fi
|
||||
${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null
|
||||
${csudo} rm -f ${nginx_service_config}
|
||||
fi
|
||||
}
|
||||
|
||||
# taos:2345:respawn:/etc/init.d/taosd start
|
||||
|
||||
function install_service_on_systemd() {
|
||||
clean_service_on_systemd
|
||||
|
||||
taosd_service_config="${service_config_dir}/taosd.service"
|
||||
${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'After=network-online.target taosadapter.service' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
|
||||
#${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}"
|
||||
${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
|
||||
${csudo} systemctl enable taosd
|
||||
|
||||
${csudo} systemctl daemon-reload
|
||||
}
|
||||
|
||||
function install_service() {
|
||||
if ((${service_mod}==0)); then
|
||||
install_service_on_systemd
|
||||
elif ((${service_mod}==1)); then
|
||||
install_service_on_sysvinit
|
||||
else
|
||||
# must manual stop taosd
|
||||
kill_process taosd
|
||||
fi
|
||||
}
|
||||
|
||||
function install_TDengine() {
|
||||
# Start to install
|
||||
echo -e "${GREEN}Start to install TDengine...${NC}"
|
||||
|
||||
install_main_path
|
||||
install_data
|
||||
install_log
|
||||
install_header
|
||||
install_lib
|
||||
|
||||
if [ -z $1 ]; then # install service and client
|
||||
# For installing new
|
||||
install_bin
|
||||
install_service
|
||||
#install_config
|
||||
|
||||
# Ask if to start the service
|
||||
#echo
|
||||
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
|
||||
echo
|
||||
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg"
|
||||
if ((${service_mod}==0)); then
|
||||
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}"
|
||||
elif ((${service_mod}==1)); then
|
||||
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}"
|
||||
fi
|
||||
|
||||
if [ ! -z "$firstEp" ]; then
|
||||
tmpFqdn=${firstEp%%:*}
|
||||
substr=":"
|
||||
if [[ $firstEp =~ $substr ]];then
|
||||
tmpPort=${firstEp#*:}
|
||||
else
|
||||
tmpPort=""
|
||||
fi
|
||||
if [[ "$tmpPort" != "" ]];then
|
||||
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
|
||||
else
|
||||
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
|
||||
fi
|
||||
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
|
||||
echo
|
||||
elif [ ! -z "$serverFqdn" ]; then
|
||||
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
|
||||
echo
|
||||
fi
|
||||
|
||||
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
|
||||
echo
|
||||
else # Only install client
|
||||
install_bin
|
||||
#install_config
|
||||
echo
|
||||
echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}"
|
||||
fi
|
||||
|
||||
touch ~/.taos_history
|
||||
}
|
||||
|
||||
|
||||
## ==============================Main program starts from here============================
|
||||
serverFqdn=$(hostname)
|
||||
if [ "$verType" == "server" ]; then
|
||||
# Install server and client
|
||||
install_TDengine
|
||||
elif [ "$verType" == "client" ]; then
|
||||
interactiveFqdn=no
|
||||
# Only install client
|
||||
install_TDengine client
|
||||
else
|
||||
echo "please input correct verType"
|
||||
fi
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
|
||||
# Generate the tar.gz package for linux os
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
|
@ -17,9 +17,12 @@ echo "=======================new version number: ${verNumber}===================
|
|||
|
||||
build_time=$(date +"%F %R")
|
||||
|
||||
echo "script_dir: ${script_dir}"
|
||||
echo "top_dir: ${top_dir}"
|
||||
|
||||
cd ${top_dir}
|
||||
git checkout -- .
|
||||
git checkout 3.0
|
||||
git pull || :
|
||||
|
||||
echo "curr_dir: ${curr_dir}"
|
||||
|
|
@ -49,28 +52,38 @@ cd ${release_dir}
|
|||
|
||||
install_dir="${release_dir}/TDengine-server-${version}"
|
||||
mkdir -p ${install_dir}
|
||||
mkdir -p ${install_dir}/bin
|
||||
mkdir -p ${install_dir}/lib
|
||||
mkdir -p ${install_dir}/inc
|
||||
|
||||
bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table"
|
||||
cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || :
|
||||
install_files="${script_dir}/install.sh"
|
||||
chmod a+x ${script_dir}/install.sh || :
|
||||
cp ${install_files} ${install_dir}
|
||||
|
||||
header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h"
|
||||
cp ${header_files} ${install_dir}/inc
|
||||
|
||||
bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh"
|
||||
cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
|
||||
|
||||
cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/
|
||||
cp ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/
|
||||
|
||||
#cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/
|
||||
#cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/
|
||||
|
||||
pkg_name=${install_dir}-Linux-x64
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,148 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Script to stop the service and uninstall TDengine, but retain the config, data and log files.
|
||||
|
||||
set -e
|
||||
#set -x
|
||||
|
||||
verMode=edge
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[1;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
#install main path
|
||||
install_main_dir="/usr/local/taos"
|
||||
data_link_dir="/usr/local/taos/data"
|
||||
log_link_dir="/usr/local/taos/log"
|
||||
cfg_link_dir="/usr/local/taos/cfg"
|
||||
bin_link_dir="/usr/bin"
|
||||
lib_link_dir="/usr/lib"
|
||||
lib64_link_dir="/usr/lib64"
|
||||
inc_link_dir="/usr/include"
|
||||
|
||||
service_config_dir="/etc/systemd/system"
|
||||
taos_service_name="taosd"
|
||||
|
||||
csudo=""
|
||||
if command -v sudo > /dev/null; then
|
||||
csudo="sudo"
|
||||
fi
|
||||
|
||||
initd_mod=0
|
||||
service_mod=2
|
||||
if pidof systemd &> /dev/null; then
|
||||
service_mod=0
|
||||
elif $(which service &> /dev/null); then
|
||||
service_mod=1
|
||||
service_config_dir="/etc/init.d"
|
||||
if $(which chkconfig &> /dev/null); then
|
||||
initd_mod=1
|
||||
elif $(which insserv &> /dev/null); then
|
||||
initd_mod=2
|
||||
elif $(which update-rc.d &> /dev/null); then
|
||||
initd_mod=3
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
else
|
||||
service_mod=2
|
||||
fi
|
||||
|
||||
|
||||
function kill_taosd() {
|
||||
pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
|
||||
if [ -n "$pid" ]; then
|
||||
${csudo} kill -9 $pid || :
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function clean_bin() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${bin_link_dir}/taos || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosd || :
|
||||
${csudo} rm -f ${bin_link_dir}/create_table || :
|
||||
${csudo} rm -f ${bin_link_dir}/tmq_sim || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosdemo || :
|
||||
${csudo} rm -f ${bin_link_dir}/taosdump || :
|
||||
${csudo} rm -f ${bin_link_dir}/rmtaos || :
|
||||
#${csudo} rm -f ${bin_link_dir}/set_core || :
|
||||
}
|
||||
|
||||
function clean_lib() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
|
||||
${csudo} rm -f ${lib64_link_dir}/libtaos.* || :
|
||||
|
||||
${csudo} rm -f ${lib_link_dir}/libtdb.* || :
|
||||
${csudo} rm -f ${lib64_link_dir}/libtdb.* || :
|
||||
}
|
||||
|
||||
function clean_header() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${inc_link_dir}/taos.h || :
|
||||
${csudo} rm -f ${inc_link_dir}/taosdef.h || :
|
||||
${csudo} rm -f ${inc_link_dir}/taoserror.h || :
|
||||
}
|
||||
|
||||
function clean_config() {
|
||||
# Remove link
|
||||
${csudo} rm -f ${cfg_link_dir}/* || :
|
||||
}
|
||||
|
||||
function clean_log() {
|
||||
# Remove link
|
||||
${csudo} rm -rf ${log_link_dir} || :
|
||||
}
|
||||
|
||||
function clean_service_on_systemd() {
|
||||
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
|
||||
if systemctl is-active --quiet ${taos_service_name}; then
|
||||
echo "TDengine taosd is running, stopping it..."
|
||||
${csudo} systemctl stop ${taos_service_name} &> /dev/null || echo &> /dev/null
|
||||
fi
|
||||
${csudo} systemctl disable ${taos_service_name} &> /dev/null || echo &> /dev/null
|
||||
${csudo} rm -f ${taosd_service_config}
|
||||
}
|
||||
|
||||
function clean_service_on_sysvinit() {
|
||||
echo " "
|
||||
}
|
||||
|
||||
function clean_service() {
|
||||
if ((${service_mod}==0)); then
|
||||
clean_service_on_systemd
|
||||
elif ((${service_mod}==1)); then
|
||||
clean_service_on_sysvinit
|
||||
else
|
||||
# must manual stop taosd
|
||||
kill_taosd
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop service and disable booting start.
|
||||
clean_service
|
||||
# Remove binary file and links
|
||||
clean_bin
|
||||
# Remove header file.
|
||||
clean_header
|
||||
# Remove lib file
|
||||
clean_lib
|
||||
# Remove link log directory
|
||||
clean_log
|
||||
# Remove link configuration file
|
||||
clean_config
|
||||
# Remove data link directory
|
||||
${csudo} rm -rf ${data_link_dir} || :
|
||||
|
||||
${csudo} rm -rf ${install_main_dir}
|
||||
|
||||
if [[ -e /etc/os-release ]]; then
|
||||
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
|
||||
else
|
||||
osinfo=""
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}TDengine is removed successfully!${NC}"
|
||||
echo
|
||||
|
|
@ -281,22 +281,35 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
|
|||
SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
|
||||
SRequestObj* pRequest = NULL;
|
||||
SQuery* pQuery = NULL;
|
||||
int32_t code = 0;
|
||||
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
|
||||
|
||||
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
|
||||
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
|
||||
|
||||
if (pQuery->localCmd) {
|
||||
CHECK_CODE_GOTO(execLocalCmd(pRequest, pQuery), _return);
|
||||
} else if (pQuery->directRpc) {
|
||||
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
|
||||
} else {
|
||||
CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
|
||||
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
|
||||
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = parseSql(pRequest, false, &pQuery);
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
switch (pQuery->execMode) {
|
||||
case QUERY_EXEC_MODE_LOCAL:
|
||||
code = execLocalCmd(pRequest, pQuery);
|
||||
break;
|
||||
case QUERY_EXEC_MODE_RPC:
|
||||
code = execDdlQuery(pRequest, pQuery);
|
||||
break;
|
||||
case QUERY_EXEC_MODE_SCHEDULE:
|
||||
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
|
||||
}
|
||||
break;
|
||||
case QUERY_EXEC_MODE_EMPTY_RESULT:
|
||||
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_return:
|
||||
taosArrayDestroy(pNodeList);
|
||||
qDestroyQuery(pQuery);
|
||||
if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
|
||||
|
|
|
|||
|
|
@ -1044,7 +1044,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
|
|||
int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
|
||||
if (epStatus == 1) {
|
||||
int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
|
||||
tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
|
||||
tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
|
||||
if (epSkipCnt < 5000) return 0;
|
||||
}
|
||||
atomic_store_32(&tmq->epSkipCnt, 0);
|
||||
|
|
@ -1149,6 +1149,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo
|
|||
pReq->consumerId = tmq->consumerId;
|
||||
pReq->epoch = tmq->epoch;
|
||||
pReq->currentOffset = reqOffset;
|
||||
pReq->reqId = generateRequestId();
|
||||
|
||||
pReq->head.vgId = htonl(pVg->vgId);
|
||||
pReq->head.contLen = htonl(sizeof(SMqPollReq));
|
||||
|
|
@ -1234,7 +1235,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
|
|||
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
|
||||
if (vgStatus != TMQ_VG_STATUS__IDLE) {
|
||||
int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
|
||||
tscDebug("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
|
||||
tscTrace("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt);
|
||||
continue;
|
||||
/*if (vgSkipCnt < 10000) continue;*/
|
||||
#if 0
|
||||
|
|
@ -1279,7 +1280,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
|
|||
.len = sizeof(SMqPollReq),
|
||||
.handle = NULL,
|
||||
};
|
||||
sendInfo->requestId = generateRequestId();
|
||||
sendInfo->requestId = pReq->reqId;
|
||||
sendInfo->requestObjRefId = 0;
|
||||
sendInfo->param = pParam;
|
||||
sendInfo->fp = tmqPollCb;
|
||||
|
|
@ -1288,7 +1289,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
|
|||
int64_t transporterId = 0;
|
||||
/*printf("send poll\n");*/
|
||||
atomic_add_fetch_32(&tmq->waitingRequest, 1);
|
||||
tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset);
|
||||
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
|
||||
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
|
||||
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
|
||||
pVg->pollCnt++;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "tdatablock.h"
|
||||
#include "tcompare.h"
|
||||
#include "tglobal.h"
|
||||
#include "tlog.h"
|
||||
|
||||
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
|
||||
pEp->port = 0;
|
||||
|
|
@ -130,44 +131,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
|
|||
memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
|
||||
pColumnInfoData->varmeta.length += varDataTLen(pData);
|
||||
} else {
|
||||
char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_BOOL: {
|
||||
*(bool*)p = *(bool*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_TINYINT:
|
||||
case TSDB_DATA_TYPE_UTINYINT: {
|
||||
*(int8_t*)p = *(int8_t*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_SMALLINT:
|
||||
case TSDB_DATA_TYPE_USMALLINT: {
|
||||
*(int16_t*)p = *(int16_t*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_INT:
|
||||
case TSDB_DATA_TYPE_UINT: {
|
||||
*(int32_t*)p = *(int32_t*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_TIMESTAMP:
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
case TSDB_DATA_TYPE_UBIGINT: {
|
||||
*(int64_t*)p = *(int64_t*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_FLOAT: {
|
||||
*(float*)p = *(float*)pData;
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_DOUBLE: {
|
||||
*(double*)p = *(double*)pData;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
assert(0);
|
||||
}
|
||||
memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -175,6 +139,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
|
|||
|
||||
static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource,
|
||||
int32_t numOfRow2) {
|
||||
if (numOfRow2 <= 0) return;
|
||||
|
||||
uint32_t total = numOfRow1 + numOfRow2;
|
||||
|
||||
if (BitmapLen(numOfRow1) < BitmapLen(total)) {
|
||||
|
|
@ -189,22 +155,32 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
|
|||
|
||||
if (remindBits == 0) { // no need to shift bits of bitmap
|
||||
memcpy(pColumnInfoData->nullbitmap + BitmapLen(numOfRow1), pSource->nullbitmap, BitmapLen(numOfRow2));
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t* p = (uint8_t*)pSource->nullbitmap;
|
||||
pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits
|
||||
|
||||
if (BitmapLen(numOfRow1) == BitmapLen(total)) {
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t len = BitmapLen(numOfRow2);
|
||||
int32_t i = 0;
|
||||
|
||||
uint8_t* p = (uint8_t*)pSource->nullbitmap;
|
||||
pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits);
|
||||
|
||||
uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
|
||||
while (i < len) {
|
||||
start[i] |= (p[i] << shiftBits);
|
||||
i += 1;
|
||||
int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1);
|
||||
while (i < len) { // size limit of pSource->nullbitmap
|
||||
if (i >= 1) {
|
||||
start[i - 1] |= (p[i] >> remindBits); //copy remind bits
|
||||
}
|
||||
|
||||
if (i > 1) {
|
||||
start[i - 1] |= (p[i] >> remindBits);
|
||||
}
|
||||
if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap
|
||||
return;
|
||||
}
|
||||
|
||||
start[i] |= (p[i] << shiftBits); //copy shift bits
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -216,6 +192,9 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
|
|||
return numOfRow1;
|
||||
}
|
||||
|
||||
if (pSource->hasNull) {
|
||||
pColumnInfoData->hasNull = pSource->hasNull;
|
||||
}
|
||||
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
// Handle the bitmap
|
||||
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
|
||||
|
|
@ -336,13 +315,18 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
|
||||
assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols);
|
||||
// if pIndexMap = NULL, merger one column by on column
|
||||
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pIndexMap) {
|
||||
assert(pSrc != NULL && pDest != NULL);
|
||||
|
||||
int32_t numOfCols = pSrc->info.numOfCols;
|
||||
int32_t numOfCols = pDest->info.numOfCols;
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
int32_t mapIndex = i;
|
||||
if(pIndexMap) {
|
||||
mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i);
|
||||
}
|
||||
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
|
||||
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
|
||||
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex);
|
||||
|
||||
uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows);
|
||||
uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows);
|
||||
|
|
@ -399,26 +383,23 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
|
|||
// TODO speedup by checking if the whole page can fit in firstly.
|
||||
if (!hasVarCol) {
|
||||
size_t rowSize = blockDataGetRowSize(pBlock);
|
||||
int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8;
|
||||
int32_t capacity = payloadSize / (rowSize + numOfCols * bitmapChar / 8.0);
|
||||
ASSERT(capacity > 0);
|
||||
|
||||
*stopIndex = startIndex + capacity;
|
||||
*stopIndex = startIndex + capacity - 1;
|
||||
if (*stopIndex >= numOfRows) {
|
||||
*stopIndex = numOfRows - 1;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
}
|
||||
// iterate the rows that can be fit in this buffer page
|
||||
int32_t size = (headerSize + colHeaderSize);
|
||||
|
||||
for (int32_t j = startIndex; j < numOfRows; ++j) {
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i);
|
||||
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
|
||||
bool isNull = colDataIsNull(pColInfoData, numOfRows, j, NULL);
|
||||
if (isNull) {
|
||||
// do nothing
|
||||
} else {
|
||||
if (pColInfoData->varmeta.offset[j] != -1) {
|
||||
char* p = colDataGetData(pColInfoData, j);
|
||||
size += varDataTLen(p);
|
||||
}
|
||||
|
|
@ -433,9 +414,9 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
|
|||
}
|
||||
}
|
||||
|
||||
if (size > pageSize) {
|
||||
if (size > pageSize) { // pageSize must be able to hold one row
|
||||
*stopIndex = j - 1;
|
||||
ASSERT(*stopIndex > startIndex);
|
||||
ASSERT(*stopIndex >= startIndex);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
|
@ -444,7 +425,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
|
|||
// all fit in
|
||||
*stopIndex = numOfRows - 1;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) {
|
||||
|
|
@ -547,8 +528,13 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
|
|||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
|
||||
|
||||
size_t metaSize = pBlock->info.rows * sizeof(int32_t);
|
||||
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
|
||||
size_t metaSize = pBlock->info.rows * sizeof(int32_t);
|
||||
char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize); // preview calloc is too small
|
||||
if (tmp == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
pCol->varmeta.offset = (int32_t*)tmp;
|
||||
memcpy(pCol->varmeta.offset, pStart, metaSize);
|
||||
pStart += metaSize;
|
||||
} else {
|
||||
|
|
@ -581,6 +567,49 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) {
|
||||
pBlock->info.rows = *(int32_t*)buf;
|
||||
|
||||
int32_t numOfCols = pBlock->info.numOfCols;
|
||||
const char* pStart = buf + sizeof(uint32_t);
|
||||
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
|
||||
size_t metaSize = capacity * sizeof(int32_t);
|
||||
memcpy(pCol->varmeta.offset, pStart, metaSize);
|
||||
pStart += metaSize;
|
||||
} else {
|
||||
memcpy(pCol->nullbitmap, pStart, BitmapLen(capacity));
|
||||
pStart += BitmapLen(capacity);
|
||||
}
|
||||
|
||||
int32_t colLength = *(int32_t*)pStart;
|
||||
pStart += sizeof(int32_t);
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
|
||||
if (pCol->varmeta.allocLen < colLength) {
|
||||
char* tmp = taosMemoryRealloc(pCol->pData, colLength);
|
||||
if (tmp == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pCol->pData = tmp;
|
||||
pCol->varmeta.allocLen = colLength;
|
||||
}
|
||||
|
||||
pCol->varmeta.length = colLength;
|
||||
ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen);
|
||||
}
|
||||
|
||||
memcpy(pCol->pData, pStart, colLength);
|
||||
pStart += pCol->info.bytes * capacity;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
size_t blockDataGetRowSize(SSDataBlock* pBlock) {
|
||||
ASSERT(pBlock != NULL);
|
||||
if (pBlock->info.rowSize == 0) {
|
||||
|
|
@ -627,6 +656,10 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
|
|||
return rowSize;
|
||||
}
|
||||
|
||||
int32_t getAllowedRowsForPage(const SSDataBlock* pBlock, size_t pgSize) {
|
||||
return (int32_t) ((pgSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock));
|
||||
}
|
||||
|
||||
typedef struct SSDataBlockSortHelper {
|
||||
SArray* orderInfo; // SArray<SBlockOrderInfo>
|
||||
SSDataBlock* pDataBlock;
|
||||
|
|
@ -665,23 +698,13 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
|
|||
void* left1 = colDataGetData(pColInfoData, left);
|
||||
void* right1 = colDataGetData(pColInfoData, right);
|
||||
|
||||
switch (pColInfoData->info.type) {
|
||||
case TSDB_DATA_TYPE_INT: {
|
||||
int32_t leftx = *(int32_t*)left1;
|
||||
int32_t rightx = *(int32_t*)right1;
|
||||
__compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
|
||||
if (leftx == rightx) {
|
||||
break;
|
||||
int ret = fn(left1, right1);
|
||||
if (ret == 0) {
|
||||
continue;
|
||||
} else {
|
||||
if (pOrder->order == TSDB_ORDER_ASC) {
|
||||
return (leftx < rightx) ? -1 : 1;
|
||||
} else {
|
||||
return (leftx < rightx) ? 1 : -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
assert(0);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -735,61 +758,12 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
|
|||
pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]];
|
||||
}
|
||||
} else {
|
||||
switch (pSrc->info.type) {
|
||||
case TSDB_DATA_TYPE_UINT:
|
||||
case TSDB_DATA_TYPE_INT: {
|
||||
for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
|
||||
int32_t* p = (int32_t*)pDst->pData;
|
||||
int32_t* srclist = (int32_t*)pSrc->pData;
|
||||
|
||||
p[j] = srclist[index[j]];
|
||||
if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
|
||||
colDataSetNull_f(pDst->nullbitmap, j);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_UTINYINT:
|
||||
case TSDB_DATA_TYPE_TINYINT: {
|
||||
for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
|
||||
int32_t* p = (int32_t*)pDst->pData;
|
||||
int32_t* srclist = (int32_t*)pSrc->pData;
|
||||
|
||||
p[j] = srclist[index[j]];
|
||||
if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
|
||||
colDataSetNull_f(pDst->nullbitmap, j);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_USMALLINT:
|
||||
case TSDB_DATA_TYPE_SMALLINT: {
|
||||
for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
|
||||
int32_t* p = (int32_t*)pDst->pData;
|
||||
int32_t* srclist = (int32_t*)pSrc->pData;
|
||||
|
||||
p[j] = srclist[index[j]];
|
||||
if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
|
||||
colDataSetNull_f(pDst->nullbitmap, j);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case TSDB_DATA_TYPE_UBIGINT:
|
||||
case TSDB_DATA_TYPE_BIGINT: {
|
||||
for (int32_t j = 0; j < pDataBlock->info.rows; ++j) {
|
||||
int32_t* p = (int32_t*)pDst->pData;
|
||||
int32_t* srclist = (int32_t*)pSrc->pData;
|
||||
|
||||
p[j] = srclist[index[j]];
|
||||
if (colDataIsNull_f(pSrc->nullbitmap, index[j])) {
|
||||
colDataSetNull_f(pDst->nullbitmap, j);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
assert(0);
|
||||
memcpy(pDst->pData + j * pDst->info.bytes, pSrc->pData + index[j] * pDst->info.bytes, pDst->info.bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -862,33 +836,6 @@ static int32_t* createTupleIndex(size_t rows) {
|
|||
|
||||
static void destroyTupleIndex(int32_t* index) { taosMemoryFreeClear(index); }
|
||||
|
||||
static __compar_fn_t getComparFn(int32_t type, int32_t order) {
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_TINYINT:
|
||||
return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc;
|
||||
case TSDB_DATA_TYPE_SMALLINT:
|
||||
return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc;
|
||||
case TSDB_DATA_TYPE_INT:
|
||||
return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
|
||||
case TSDB_DATA_TYPE_BIGINT:
|
||||
return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc;
|
||||
case TSDB_DATA_TYPE_FLOAT:
|
||||
return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc;
|
||||
case TSDB_DATA_TYPE_DOUBLE:
|
||||
return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc;
|
||||
case TSDB_DATA_TYPE_UTINYINT:
|
||||
return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc;
|
||||
case TSDB_DATA_TYPE_USMALLINT:
|
||||
return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc;
|
||||
case TSDB_DATA_TYPE_UINT:
|
||||
return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc;
|
||||
case TSDB_DATA_TYPE_UBIGINT:
|
||||
return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc;
|
||||
default:
|
||||
return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
||||
ASSERT(pDataBlock != NULL && pOrderInfo != NULL);
|
||||
if (pDataBlock->info.rows <= 1) {
|
||||
|
|
@ -922,11 +869,11 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
|
||||
int64_t p0 = taosGetTimestampUs();
|
||||
|
||||
__compar_fn_t fn = getComparFn(pColInfoData->info.type, pOrder->order);
|
||||
__compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
|
||||
|
||||
int64_t p1 = taosGetTimestampUs();
|
||||
printf("sort:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
|
||||
uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else { // var data type
|
||||
|
|
@ -955,24 +902,21 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
|
||||
SColumnInfoData* pCols = createHelpColInfoData(pDataBlock);
|
||||
if (pCols == NULL) {
|
||||
destroyTupleIndex(index);
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
int64_t p2 = taosGetTimestampUs();
|
||||
|
||||
int32_t code = blockDataAssign(pCols, pDataBlock, index);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
terrno = code;
|
||||
return code;
|
||||
}
|
||||
blockDataAssign(pCols, pDataBlock, index);
|
||||
|
||||
int64_t p3 = taosGetTimestampUs();
|
||||
|
||||
copyBackToBlock(pDataBlock, pCols);
|
||||
int64_t p4 = taosGetTimestampUs();
|
||||
|
||||
printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
|
||||
uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
|
||||
p3 - p2, p4 - p3, rows);
|
||||
destroyTupleIndex(index);
|
||||
|
||||
|
|
@ -1178,6 +1122,9 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows)
|
|||
|
||||
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
|
||||
int32_t code = 0;
|
||||
if (numOfRows == 0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
|
||||
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
|
||||
|
|
@ -1201,6 +1148,9 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
|
|||
}
|
||||
|
||||
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
|
||||
if(pDataBlock == NULL){
|
||||
return NULL;
|
||||
}
|
||||
int32_t numOfCols = pDataBlock->info.numOfCols;
|
||||
|
||||
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
|
||||
|
|
@ -1220,7 +1170,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) {
|
|||
}
|
||||
|
||||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
|
||||
return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock));
|
||||
return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock));
|
||||
}
|
||||
|
||||
void colDataDestroy(SColumnInfoData* pColData) {
|
||||
|
|
|
|||
|
|
@ -30,21 +30,18 @@ char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
|
|||
uint16_t tsServerPort = 6030;
|
||||
int32_t tsVersion = 30000000;
|
||||
int32_t tsStatusInterval = 1; // second
|
||||
bool tsEnableTelemetryReporting = 0;
|
||||
bool tsEnableTelemetryReporting = false;
|
||||
|
||||
// common
|
||||
int32_t tsRpcTimer = 300;
|
||||
int32_t tsRpcMaxTime = 600; // seconds;
|
||||
bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
|
||||
bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default
|
||||
int32_t tsMaxShellConns = 50000;
|
||||
int32_t tsMaxConnections = 50000;
|
||||
int32_t tsShellActivityTimer = 3; // second
|
||||
float tsNumOfThreadsPerCore = 1.0f;
|
||||
int32_t tsNumOfCommitThreads = 4;
|
||||
float tsRatioOfQueryCores = 1.0f;
|
||||
int32_t tsMaxBinaryDisplayWidth = 30;
|
||||
bool tsEnableSlaveQuery = 1;
|
||||
bool tsPrintAuth = 0;
|
||||
bool tsEnableSlaveQuery = true;
|
||||
bool tsPrintAuth = false;
|
||||
|
||||
// multi process
|
||||
bool tsMultiProcess = false;
|
||||
|
|
@ -54,8 +51,24 @@ int32_t tsQnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
|
|||
int32_t tsSnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
|
||||
int32_t tsBnodeShmSize = TSDB_MAX_WAL_SIZE * 4;
|
||||
|
||||
// queue & threads
|
||||
int32_t tsNumOfRpcThreads = 1;
|
||||
int32_t tsNumOfCommitThreads = 2;
|
||||
int32_t tsNumOfTaskQueueThreads = 1;
|
||||
int32_t tsNumOfMnodeQueryThreads = 1;
|
||||
int32_t tsNumOfMnodeReadThreads = 1;
|
||||
int32_t tsNumOfVnodeQueryThreads = 2;
|
||||
int32_t tsNumOfVnodeFetchThreads = 2;
|
||||
int32_t tsNumOfVnodeWriteThreads = 2;
|
||||
int32_t tsNumOfVnodeSyncThreads = 2;
|
||||
int32_t tsNumOfVnodeMergeThreads = 2;
|
||||
int32_t tsNumOfQnodeQueryThreads = 2;
|
||||
int32_t tsNumOfQnodeFetchThreads = 2;
|
||||
int32_t tsNumOfSnodeSharedThreads = 2;
|
||||
int32_t tsNumOfSnodeUniqueThreads = 2;
|
||||
|
||||
// monitor
|
||||
bool tsEnableMonitor = 1;
|
||||
bool tsEnableMonitor = true;
|
||||
int32_t tsMonitorInterval = 30;
|
||||
char tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
|
||||
uint16_t tsMonitorPort = 6043;
|
||||
|
|
@ -124,13 +137,13 @@ int32_t tsQueryBufferSize = -1;
|
|||
int64_t tsQueryBufferSizeBytes = -1;
|
||||
|
||||
// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
|
||||
bool tsRetrieveBlockingModel = 0;
|
||||
bool tsRetrieveBlockingModel = false;
|
||||
|
||||
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
|
||||
bool tsKeepOriginalColumnName = 0;
|
||||
bool tsKeepOriginalColumnName = false;
|
||||
|
||||
// kill long query
|
||||
bool tsDeadLockKillQuery = 0;
|
||||
bool tsDeadLockKillQuery = false;
|
||||
|
||||
// tsdb config
|
||||
// For backward compatibility
|
||||
|
|
@ -290,7 +303,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
|
||||
if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
|
||||
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
|
||||
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
|
||||
|
|
@ -304,6 +316,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
return -1;
|
||||
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
|
||||
|
||||
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
|
||||
tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
|
||||
if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -336,8 +353,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1;
|
||||
if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1;
|
||||
if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0) != 0) return -1;
|
||||
if (cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1;
|
||||
if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1;
|
||||
|
|
@ -360,7 +375,59 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
|
||||
// if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, 4096, INT32_MAX, 0) != 0) return -1;
|
||||
|
||||
tsNumOfRpcThreads = tsNumOfCores / 2;
|
||||
tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfCommitThreads = tsNumOfCores / 2;
|
||||
tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfMnodeQueryThreads = tsNumOfCores / 8;
|
||||
tsNumOfMnodeQueryThreads = TRANGE(tsNumOfMnodeQueryThreads, 1, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfMnodeQueryThreads", tsNumOfMnodeQueryThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfMnodeReadThreads = tsNumOfCores / 8;
|
||||
tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeQueryThreads = tsNumOfCores / 2;
|
||||
tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeFetchThreads = tsNumOfCores / 2;
|
||||
tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 2, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeWriteThreads = tsNumOfCores;
|
||||
tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeSyncThreads = tsNumOfCores / 2;
|
||||
tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfVnodeMergeThreads = tsNumOfCores / 8;
|
||||
tsNumOfVnodeMergeThreads = TRANGE(tsNumOfVnodeMergeThreads, 1, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfVnodeMergeThreads", tsNumOfVnodeMergeThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfQnodeQueryThreads = tsNumOfCores / 2;
|
||||
tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1);
|
||||
if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfQnodeFetchThreads = tsNumOfCores / 2;
|
||||
tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 2, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfSnodeSharedThreads = tsNumOfCores / 4;
|
||||
tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeSharedThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
tsNumOfSnodeUniqueThreads = tsNumOfCores / 4;
|
||||
tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4);
|
||||
if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeUniqueThreads, 1, 1024, 0) != 0) return -1;
|
||||
|
||||
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
|
||||
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
|
||||
|
|
@ -424,7 +491,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
|
||||
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
|
||||
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
|
||||
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
|
||||
|
|
@ -437,7 +503,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
|
|||
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
|
||||
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
|
||||
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
|
||||
|
||||
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -461,8 +527,6 @@ static void taosSetSystemCfg(SConfig *pCfg) {
|
|||
|
||||
static int32_t taosSetServerCfg(SConfig *pCfg) {
|
||||
tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
|
||||
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
|
||||
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
|
||||
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
|
||||
tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval;
|
||||
tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32;
|
||||
|
|
@ -485,7 +549,21 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32;
|
||||
tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32;
|
||||
tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32;
|
||||
// tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32;
|
||||
tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32;
|
||||
|
||||
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
|
||||
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
|
||||
tsNumOfMnodeQueryThreads = cfgGetItem(pCfg, "numOfMnodeQueryThreads")->i32;
|
||||
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
|
||||
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
|
||||
tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
|
||||
tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
|
||||
tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
|
||||
tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32;
|
||||
tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
|
||||
tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
|
||||
tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32;
|
||||
tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
|
||||
|
||||
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
|
||||
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
|
||||
|
|
|
|||
|
|
@ -1515,6 +1515,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
|
|||
if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->compression) < 0) return -1;
|
||||
|
|
@ -1524,6 +1525,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
|
|||
if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->streamMode) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pReq->singleSTable) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1;
|
||||
for (int32_t i = 0; i < pReq->numOfRetensions; ++i) {
|
||||
SRetention *pRetension = taosArrayGet(pReq->pRetensions, i);
|
||||
|
|
@ -1556,6 +1558,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
|
|||
if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1;
|
||||
|
|
@ -1565,6 +1568,7 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
|
|||
if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->streamMode) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pReq->singleSTable) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1;
|
||||
pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention));
|
||||
if (pReq->pRetensions == NULL) {
|
||||
|
|
@ -1942,6 +1946,152 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) {
|
|||
taosArrayDestroy(pRsp->pArray);
|
||||
}
|
||||
|
||||
int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
|
||||
SCoder encoder = {0};
|
||||
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tCoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq) {
|
||||
SCoder decoder = {0};
|
||||
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tCoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp) {
|
||||
SCoder encoder = {0};
|
||||
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->numOfVgroups) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->cacheBlockSize) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->totalBlocks) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->daysPerFile) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->daysToKeep0) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->daysToKeep1) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->daysToKeep2) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->commitTime) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->fsyncPeriod) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->quorum) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->update) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->cacheLastRow) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pRsp->streamMode) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tCoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp) {
|
||||
SCoder decoder = {0};
|
||||
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->numOfVgroups) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->cacheBlockSize) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->totalBlocks) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->daysPerFile) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->daysToKeep0) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->daysToKeep1) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->daysToKeep2) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->commitTime) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pRsp->fsyncPeriod) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->quorum) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->update) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->cacheLastRow) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &pRsp->streamMode) < 0) return -1;
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tCoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) {
|
||||
SCoder encoder = {0};
|
||||
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->indexFName) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tCoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSUserIndexReq(void* buf, int32_t bufLen, SUserIndexReq* pReq) {
|
||||
SCoder decoder = {0};
|
||||
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->indexFName) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tCoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSUserIndexRsp(void* buf, int32_t bufLen, const SUserIndexRsp* pRsp) {
|
||||
SCoder encoder = {0};
|
||||
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->tblFName) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->colName) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->indexType) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->indexExts) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tCoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSUserIndexRsp(void* buf, int32_t bufLen, SUserIndexRsp* pRsp) {
|
||||
SCoder decoder = {0};
|
||||
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->tblFName) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->colName) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->indexType) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->indexExts) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tCoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) {
|
||||
SCoder encoder = {0};
|
||||
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
|
||||
|
|
@ -2553,6 +2703,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
|
|||
int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
|
||||
taosArrayDestroy(pReq->pRetensions);
|
||||
pReq->pRetensions = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,34 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "bmInt.h"
|
||||
|
||||
void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo) {
|
||||
}
|
||||
|
||||
int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonBmInfo bmInfo = {0};
|
||||
bmGetMonitorInfo(pWrapper, &bmInfo);
|
||||
dmGetMonitorSysInfo(&bmInfo.sys);
|
||||
monGetLogs(&bmInfo.log);
|
||||
|
||||
int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonBmInfo(pRsp, rspLen, &bmInfo);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonBmInfo(&bmInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SDnode *pDnode = pWrapper->pDnode;
|
||||
SRpcMsg *pReq = &pMsg->rpcMsg;
|
||||
|
|
@ -54,4 +82,6 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
void bmInitMsgHandle(SMgmtWrapper *pWrapper) {}
|
||||
void bmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_BM_INFO, bmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,37 @@ static void bmSendErrorRsps(SMgmtWrapper *pWrapper, STaosQall *qall, int32_t num
|
|||
}
|
||||
}
|
||||
|
||||
static void bmProcessQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
static inline void bmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
|
||||
.ahandle = pMsg->rpcMsg.ahandle,
|
||||
.code = code,
|
||||
.pCont = pMsg->pRsp,
|
||||
.contLen = pMsg->rspLen};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
static void bmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
||||
SBnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
dTrace("msg:%p, get from bnode monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = &pMsg->rpcMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->rpcMsg.msgType == TDMT_MON_BM_INFO) {
|
||||
code = bmProcessGetMonBmInfoReq(pMgmt->pWrapper, pMsg);
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
bmSendRsp(pMgmt->pWrapper, pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SBnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
SMgmtWrapper *pWrapper = pMgmt->pWrapper;
|
||||
|
||||
|
|
@ -72,18 +102,37 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SBnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
|
||||
SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessQueue, .param = pMgmt};
|
||||
SMultiWorkerCfg cfg = {.max = 1, .name = "bnode-write", .fp = (FItems)bmProcessWriteQueue, .param = pMgmt};
|
||||
if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) {
|
||||
dError("failed to start bnode write worker since %s", terrstr());
|
||||
dError("failed to start bnode-write worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (tsMultiProcess) {
|
||||
SSingleWorkerCfg mCfg = {
|
||||
.min = 1, .max = 1, .name = "bnode-monitor", .fp = (FItem)bmProcessMonQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
dError("failed to start bnode-monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
dDebug("bnode workers are initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void bmStopWorker(SBnodeMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
tMultiWorkerCleanup(&pMgmt->writeWorker);
|
||||
dDebug("bnode workers are closed");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,8 +42,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
|
||||
if (pWrapper != NULL) {
|
||||
req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
|
||||
vmMonitorVnodeLoads(pWrapper, req.pVloads);
|
||||
SMonVloadInfo info = {0};
|
||||
dmGetVnodeLoads(pWrapper, &info);
|
||||
req.pVloads = info.pVloads;
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
|
||||
|
|
@ -96,6 +97,7 @@ int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
|
|||
}
|
||||
|
||||
pMgmt->statusSent = 0;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
|
||||
|
|
@ -117,7 +119,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
|
|||
return TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
}
|
||||
|
||||
|
||||
static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, EDndType ntype, SNodeMsg *pMsg) {
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
|
||||
if (pWrapper != NULL) {
|
||||
|
|
@ -209,7 +210,7 @@ void dmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
|||
dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by MNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessStatusMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,21 +14,7 @@
|
|||
*/
|
||||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "dndInt.h"
|
||||
|
||||
static int32_t dmGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
|
||||
tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
|
||||
pInfo->logdir.size = tsLogSpace.size;
|
||||
tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
|
||||
pInfo->tempdir.size = tsTempSpace.size;
|
||||
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
|
||||
if (pWrapper != NULL) {
|
||||
vmMonitorTfsInfo(pWrapper, pInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#include "dmInt.h"
|
||||
|
||||
static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
|
||||
pInfo->protocol = 1;
|
||||
|
|
@ -39,6 +25,163 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
|
|||
|
||||
static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
|
||||
pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
|
||||
if (pWrapper != NULL) {
|
||||
pInfo->has_mnode = pWrapper->required;
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
|
||||
pInfo->logdir.size = tsLogSpace.size;
|
||||
tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
|
||||
pInfo->tempdir.size = tsTempSpace.size;
|
||||
}
|
||||
|
||||
static void dmGetMonitorInfo(SDnode *pDnode, SMonDmInfo *pInfo) {
|
||||
dmGetMonitorBasicInfo(pDnode, &pInfo->basic);
|
||||
dmGetMonitorSysInfo(&pInfo->sys);
|
||||
dmGetMonitorDnodeInfo(pDnode, &pInfo->dnode);
|
||||
}
|
||||
|
||||
void dmSendMonitorReport(SDnode *pDnode) {
|
||||
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
|
||||
dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
|
||||
|
||||
SMonDmInfo dmInfo = {0};
|
||||
SMonMmInfo mmInfo = {0};
|
||||
SMonVmInfo vmInfo = {0};
|
||||
SMonQmInfo qmInfo = {0};
|
||||
SMonSmInfo smInfo = {0};
|
||||
SMonBmInfo bmInfo = {0};
|
||||
|
||||
SRpcMsg req = {0};
|
||||
SRpcMsg rsp;
|
||||
SEpSet epset = {.inUse = 0, .numOfEps = 1};
|
||||
tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
|
||||
epset.eps[0].port = tsServerPort;
|
||||
|
||||
SMgmtWrapper *pWrapper = NULL;
|
||||
dmGetMonitorInfo(pDnode, &dmInfo);
|
||||
|
||||
bool getFromAPI = !tsMultiProcess;
|
||||
pWrapper = &pDnode->wrappers[MNODE];
|
||||
if (getFromAPI) {
|
||||
if (dndMarkWrapper(pWrapper) != 0) {
|
||||
mmGetMonitorInfo(pWrapper, &mmInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
} else {
|
||||
if (pWrapper->required) {
|
||||
req.msgType = TDMT_MON_MM_INFO;
|
||||
dndSendRecv(pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonMmInfo(rsp.pCont, rsp.contLen, &mmInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
pWrapper = &pDnode->wrappers[VNODES];
|
||||
if (getFromAPI) {
|
||||
if (dndMarkWrapper(pWrapper) != 0) {
|
||||
vmGetMonitorInfo(pWrapper, &vmInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
} else {
|
||||
if (pWrapper->required) {
|
||||
req.msgType = TDMT_MON_VM_INFO;
|
||||
dndSendRecv(pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonVmInfo(rsp.pCont, rsp.contLen, &vmInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
pWrapper = &pDnode->wrappers[QNODE];
|
||||
if (getFromAPI) {
|
||||
if (dndMarkWrapper(pWrapper) != 0) {
|
||||
qmGetMonitorInfo(pWrapper, &qmInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
} else {
|
||||
if (pWrapper->required) {
|
||||
req.msgType = TDMT_MON_QM_INFO;
|
||||
dndSendRecv(pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonQmInfo(rsp.pCont, rsp.contLen, &qmInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
pWrapper = &pDnode->wrappers[SNODE];
|
||||
if (getFromAPI) {
|
||||
if (dndMarkWrapper(pWrapper) != 0) {
|
||||
smGetMonitorInfo(pWrapper, &smInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
} else {
|
||||
if (pWrapper->required) {
|
||||
req.msgType = TDMT_MON_SM_INFO;
|
||||
dndSendRecv(pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonSmInfo(rsp.pCont, rsp.contLen, &smInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
pWrapper = &pDnode->wrappers[BNODE];
|
||||
if (getFromAPI) {
|
||||
if (dndMarkWrapper(pWrapper) != 0) {
|
||||
bmGetMonitorInfo(pWrapper, &bmInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
} else {
|
||||
if (pWrapper->required) {
|
||||
req.msgType = TDMT_MON_BM_INFO;
|
||||
dndSendRecv(pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonBmInfo(rsp.pCont, rsp.contLen, &bmInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
monSetDmInfo(&dmInfo);
|
||||
monSetMmInfo(&mmInfo);
|
||||
monSetVmInfo(&vmInfo);
|
||||
monSetQmInfo(&qmInfo);
|
||||
monSetSmInfo(&smInfo);
|
||||
monSetBmInfo(&bmInfo);
|
||||
tFreeSMonMmInfo(&mmInfo);
|
||||
tFreeSMonVmInfo(&vmInfo);
|
||||
tFreeSMonQmInfo(&qmInfo);
|
||||
tFreeSMonSmInfo(&smInfo);
|
||||
tFreeSMonBmInfo(&bmInfo);
|
||||
monSendReport();
|
||||
}
|
||||
|
||||
void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) {
|
||||
bool getFromAPI = !tsMultiProcess;
|
||||
if (getFromAPI) {
|
||||
vmGetVnodeLoads(pWrapper, pInfo);
|
||||
} else {
|
||||
SRpcMsg req = {.msgType = TDMT_MON_VM_LOAD};
|
||||
SRpcMsg rsp = {0};
|
||||
SEpSet epset = {.inUse = 0, .numOfEps = 1};
|
||||
tstrncpy(epset.eps[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
|
||||
epset.eps[0].port = tsServerPort;
|
||||
|
||||
dndSendRecv(pWrapper->pDnode, &epset, &req, &rsp);
|
||||
if (rsp.code == 0 && rsp.contLen > 0) {
|
||||
tDeserializeSMonVloadInfo(rsp.pCont, rsp.contLen, pInfo);
|
||||
}
|
||||
rpcFreeCont(rsp.pCont);
|
||||
}
|
||||
}
|
||||
|
||||
void dmGetMonitorSysInfo(SMonSysInfo *pInfo) {
|
||||
taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
|
||||
taosGetCpuCores(&pInfo->cpu_cores);
|
||||
taosGetProcMemory(&pInfo->mem_engine);
|
||||
|
|
@ -47,61 +190,6 @@ static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
|
|||
pInfo->disk_engine = 0;
|
||||
pInfo->disk_used = tsDataSpace.size.used;
|
||||
pInfo->disk_total = tsDataSpace.size.total;
|
||||
taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
|
||||
taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
|
||||
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
|
||||
if (pWrapper != NULL) {
|
||||
vmMonitorVnodeReqs(pWrapper, pInfo);
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
|
||||
pWrapper = dndAcquireWrapper(pDnode, MNODE);
|
||||
if (pWrapper != NULL) {
|
||||
pInfo->has_mnode = pWrapper->required;
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
}
|
||||
|
||||
void dmSendMonitorReport(SDnode *pDnode) {
|
||||
if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
|
||||
dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
|
||||
|
||||
SMonInfo *pMonitor = monCreateMonitorInfo();
|
||||
if (pMonitor == NULL) return;
|
||||
|
||||
SMonBasicInfo basicInfo = {0};
|
||||
dmGetMonitorBasicInfo(pDnode, &basicInfo);
|
||||
monSetBasicInfo(pMonitor, &basicInfo);
|
||||
|
||||
SMonClusterInfo clusterInfo = {0};
|
||||
SMonVgroupInfo vgroupInfo = {0};
|
||||
SMonGrantInfo grantInfo = {0};
|
||||
|
||||
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
|
||||
if (pWrapper != NULL) {
|
||||
if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
|
||||
monSetClusterInfo(pMonitor, &clusterInfo);
|
||||
monSetVgroupInfo(pMonitor, &vgroupInfo);
|
||||
monSetGrantInfo(pMonitor, &grantInfo);
|
||||
}
|
||||
dndReleaseWrapper(pWrapper);
|
||||
}
|
||||
|
||||
SMonDnodeInfo dnodeInfo = {0};
|
||||
dmGetMonitorDnodeInfo(pDnode, &dnodeInfo);
|
||||
monSetDnodeInfo(pMonitor, &dnodeInfo);
|
||||
|
||||
SMonDiskInfo diskInfo = {0};
|
||||
if (dmGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
|
||||
monSetDiskInfo(pMonitor, &diskInfo);
|
||||
}
|
||||
|
||||
taosArrayDestroy(clusterInfo.dnodes);
|
||||
taosArrayDestroy(clusterInfo.mnodes);
|
||||
taosArrayDestroy(vgroupInfo.vgroups);
|
||||
taosArrayDestroy(diskInfo.datadirs);
|
||||
|
||||
monSendReport(pMonitor);
|
||||
monCleanupMonitorInfo(pMonitor);
|
||||
taosGetCardInfoDelta(&pInfo->net_in, &pInfo->net_out);
|
||||
taosGetProcIODelta(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
|
||||
}
|
||||
|
|
@ -44,6 +44,7 @@ static void *dmThreadRoutine(void *param) {
|
|||
lastMonitorTime = curTime;
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dmStartThread(SDnodeMgmt *pMgmt) {
|
||||
|
|
@ -101,9 +102,9 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) {
|
||||
dError("failed to start dnode status worker since %s", terrstr());
|
||||
SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-monitor", .fp = (FItem)dmProcessQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &scfg) != 0) {
|
||||
dError("failed to start dnode monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
|
|||
|
||||
void dmStopWorker(SDnodeMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->mgmtWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->statusWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
|
||||
if (pMgmt->threadId != NULL) {
|
||||
taosDestoryThread(pMgmt->threadId);
|
||||
|
|
@ -131,9 +132,9 @@ int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SDnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->statusWorker;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#define _TD_DND_BNODE_INT_H_
|
||||
|
||||
#include "dndInt.h"
|
||||
|
||||
#include "bnode.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
@ -29,6 +30,7 @@ typedef struct SBnodeMgmt {
|
|||
SMgmtWrapper *pWrapper;
|
||||
const char *path;
|
||||
SMultiWorker writeWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
} SBnodeMgmt;
|
||||
|
||||
// bmInt.c
|
||||
|
|
@ -39,11 +41,13 @@ int32_t bmDrop(SMgmtWrapper *pWrapper);
|
|||
void bmInitMsgHandle(SMgmtWrapper *pWrapper);
|
||||
int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t bmProcessGetMonBmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
|
||||
// bmWorker.c
|
||||
int32_t bmStartWorker(SBnodeMgmt *pMgmt);
|
||||
void bmStopWorker(SBnodeMgmt *pMgmt);
|
||||
int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ typedef struct SDnodeMgmt {
|
|||
TdThread *threadId;
|
||||
SRWLatch latch;
|
||||
SSingleWorker mgmtWorker;
|
||||
SSingleWorker statusWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
SMsgCb msgCb;
|
||||
const char *path;
|
||||
SDnode *pDnode;
|
||||
|
|
@ -54,6 +54,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
|
|||
int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg);
|
||||
|
||||
// dmMonitor.c
|
||||
void dmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo);
|
||||
void dmSendMonitorReport(SDnode *pDnode);
|
||||
|
||||
// dmWorker.c
|
||||
|
|
@ -61,7 +62,7 @@ int32_t dmStartThread(SDnodeMgmt *pMgmt);
|
|||
int32_t dmStartWorker(SDnodeMgmt *pMgmt);
|
||||
void dmStopWorker(SDnodeMgmt *pMgmt);
|
||||
int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t dmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ void dndCleanupTrans(SDnode *pDnode);
|
|||
SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper);
|
||||
SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper);
|
||||
int32_t dndInitMsgHandle(SDnode *pDnode);
|
||||
void dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
|
||||
|
||||
// mgmt
|
||||
void dmSetMgmtFp(SMgmtWrapper *pWrapper);
|
||||
|
|
@ -182,22 +183,13 @@ void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
|
|||
void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
|
||||
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
|
||||
|
||||
typedef struct {
|
||||
int32_t openVnodes;
|
||||
int32_t totalVnodes;
|
||||
int32_t masterNum;
|
||||
int64_t numOfSelectReqs;
|
||||
int64_t numOfInsertReqs;
|
||||
int64_t numOfInsertSuccessReqs;
|
||||
int64_t numOfBatchInsertReqs;
|
||||
int64_t numOfBatchInsertSuccessReqs;
|
||||
} SVnodesStat;
|
||||
|
||||
void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads);
|
||||
int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo);
|
||||
void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo);
|
||||
int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
|
||||
SMonGrantInfo *pGrantInfo);
|
||||
void dmGetMonitorSysInfo(SMonSysInfo *pInfo);
|
||||
void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo);
|
||||
void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo);
|
||||
void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo);
|
||||
void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo);
|
||||
void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo);
|
||||
void bmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonBmInfo *bmInfo);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ typedef struct SMnodeMgmt {
|
|||
SSingleWorker readWorker;
|
||||
SSingleWorker writeWorker;
|
||||
SSingleWorker syncWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
SReplica replicas[TSDB_MAX_REPLICA];
|
||||
int8_t replica;
|
||||
int8_t selfIndex;
|
||||
|
|
@ -51,6 +52,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper);
|
|||
int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);
|
||||
int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
|
||||
// mmWorker.c
|
||||
int32_t mmStartWorker(SMnodeMgmt *pMgmt);
|
||||
|
|
@ -59,6 +61,7 @@ int32_t mmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
|||
int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
|
||||
int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
|
||||
int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ typedef struct SQnodeMgmt {
|
|||
const char *path;
|
||||
SSingleWorker queryWorker;
|
||||
SSingleWorker fetchWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
} SQnodeMgmt;
|
||||
|
||||
// qmInt.c
|
||||
|
|
@ -40,6 +41,7 @@ int32_t qmDrop(SMgmtWrapper *pWrapper);
|
|||
void qmInitMsgHandle(SMgmtWrapper *pWrapper);
|
||||
int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
|
||||
// qmWorker.c
|
||||
int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
|
||||
|
|
@ -50,6 +52,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt);
|
|||
void qmStopWorker(SQnodeMgmt *pMgmt);
|
||||
int32_t qmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ typedef struct SSnodeMgmt {
|
|||
int8_t uniqueWorkerInUse;
|
||||
SArray *uniqueWorkers; // SArray<SMultiWorker*>
|
||||
SSingleWorker sharedWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
} SSnodeMgmt;
|
||||
|
||||
// smInt.c
|
||||
|
|
@ -42,6 +43,7 @@ int32_t smDrop(SMgmtWrapper *pWrapper);
|
|||
void smInitMsgHandle(SMgmtWrapper *pWrapper);
|
||||
int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
|
||||
// smWorker.c
|
||||
int32_t smStartWorker(SSnodeMgmt *pMgmt);
|
||||
|
|
@ -50,6 +52,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
|||
int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t smProcessExecMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ typedef struct SVnodesMgmt {
|
|||
SHashObj *hash;
|
||||
SRWLatch latch;
|
||||
SVnodesStat state;
|
||||
SVnodesStat lastState;
|
||||
STfs *pTfs;
|
||||
SQWorkerPool queryPool;
|
||||
SQWorkerPool fetchPool;
|
||||
|
|
@ -38,6 +39,7 @@ typedef struct SVnodesMgmt {
|
|||
SDnode *pDnode;
|
||||
SMgmtWrapper *pWrapper;
|
||||
SSingleWorker mgmtWorker;
|
||||
SSingleWorker monitorWorker;
|
||||
} SVnodesMgmt;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -91,6 +93,8 @@ int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
|
|||
int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
|
||||
int32_t vmProcessSyncVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
|
||||
int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
|
||||
int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq);
|
||||
|
||||
// vmFile.c
|
||||
int32_t vmGetVnodesFromFile(SVnodesMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
|
||||
|
|
@ -114,6 +118,7 @@ int32_t vmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
|||
int32_t vmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrappert, SNodeMsg *pMsg);
|
||||
int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -257,16 +257,11 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch
|
|||
static int32_t dndInitServer(SDnode *pDnode) {
|
||||
STransMgmt *pMgmt = &pDnode->trans;
|
||||
|
||||
int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
|
||||
if (numOfThreads < 1) {
|
||||
numOfThreads = 1;
|
||||
}
|
||||
|
||||
SRpcInit rpcInit;
|
||||
memset(&rpcInit, 0, sizeof(rpcInit));
|
||||
rpcInit.localPort = pDnode->serverPort;
|
||||
rpcInit.label = "DND";
|
||||
rpcInit.numOfThreads = numOfThreads;
|
||||
rpcInit.numOfThreads = tsNumOfRpcThreads;
|
||||
rpcInit.cfp = (RpcCfp)dndProcessMsg;
|
||||
rpcInit.sessions = tsMaxShellConns;
|
||||
rpcInit.connType = TAOS_CONN_SERVER;
|
||||
|
|
@ -488,3 +483,7 @@ SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
|
|||
.name = pWrapper->name};
|
||||
return cfg;
|
||||
}
|
||||
|
||||
void dndSendRecv(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp) {
|
||||
rpcSendRecv(pDnode->trans.clientRpc, pEpSet, pReq, pRsp);
|
||||
}
|
||||
|
|
@ -16,6 +16,36 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "mmInt.h"
|
||||
|
||||
void mmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonMmInfo *mmInfo) {
|
||||
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
mndGetMonitorInfo(pMgmt->pMnode, &mmInfo->cluster, &mmInfo->vgroup, &mmInfo->grant);
|
||||
}
|
||||
|
||||
int32_t mmProcessGetMonMmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonMmInfo mmInfo = {0};
|
||||
mmGetMonitorInfo(pWrapper, &mmInfo);
|
||||
dmGetMonitorSysInfo(&mmInfo.sys);
|
||||
monGetLogs(&mmInfo.log);
|
||||
|
||||
int32_t rspLen = tSerializeSMonMmInfo(NULL, 0, &mmInfo);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonMmInfo(pRsp, rspLen, &mmInfo);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonMmInfo(&mmInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SDnode *pDnode = pWrapper->pDnode;
|
||||
SRpcMsg *pReq = &pMsg->rpcMsg;
|
||||
|
|
@ -74,6 +104,8 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
|
|||
}
|
||||
|
||||
void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_MM_INFO, mmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by DNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
|
|
@ -146,10 +178,13 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
|||
dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by VNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
|
|
@ -161,5 +196,4 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
|||
dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,8 +241,3 @@ void mmSetMgmtFp(SMgmtWrapper *pWrapper) {
|
|||
pWrapper->fp = mgmtFp;
|
||||
}
|
||||
|
||||
int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
|
||||
SMonGrantInfo *pGrantInfo) {
|
||||
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,19 +23,18 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
|||
SRpcMsg *pRpc = &pMsg->rpcMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->rpcMsg.msgType != TDMT_DND_ALTER_MNODE) {
|
||||
if (pMsg->rpcMsg.msgType == TDMT_DND_ALTER_MNODE) {
|
||||
code = mmProcessAlterReq(pMgmt, pMsg);
|
||||
} else if (pMsg->rpcMsg.msgType == TDMT_MON_MM_INFO) {
|
||||
code = mmProcessGetMonMmInfoReq(pMgmt->pWrapper, pMsg);
|
||||
} else {
|
||||
pMsg->pNode = pMgmt->pMnode;
|
||||
code = mndProcessMsg(pMsg);
|
||||
} else {
|
||||
code = mmProcessAlterReq(pMgmt, pMsg);
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (pRpc->handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
if (code != 0) {
|
||||
code = terrno;
|
||||
dError("msg:%p, failed to process since %s", pMsg, terrstr());
|
||||
}
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
|
@ -98,6 +97,15 @@ int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SMnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) {
|
||||
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
|
||||
if (pMsg == NULL) return -1;
|
||||
|
|
@ -129,35 +137,52 @@ int32_t mmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
|
|||
}
|
||||
|
||||
int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
|
||||
SSingleWorkerCfg qCfg = {.min = 0, .max = 1, .name = "mnode-query", .fp = (FItem)mmProcessQueryQueue, .param = pMgmt};
|
||||
SSingleWorkerCfg qCfg = {.min = tsNumOfMnodeQueryThreads,
|
||||
.max = tsNumOfMnodeQueryThreads,
|
||||
.name = "mnode-query",
|
||||
.fp = (FItem)mmProcessQueryQueue,
|
||||
.param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
|
||||
dError("failed to start mnode-query worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
SSingleWorkerCfg rCfg = {.min = 0, .max = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
SSingleWorkerCfg rCfg = {.min = tsNumOfMnodeReadThreads,
|
||||
.max = tsNumOfMnodeReadThreads,
|
||||
.name = "mnode-read",
|
||||
.fp = (FItem)mmProcessQueue,
|
||||
.param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) {
|
||||
dError("failed to start mnode-read worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
SSingleWorkerCfg wCfg = {.min = 0, .max = 1, .name = "mnode-write", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
SSingleWorkerCfg wCfg = {.min = 1, .max = 1, .name = "mnode-write", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) {
|
||||
dError("failed to start mnode-write worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
SSingleWorkerCfg sCfg = {.min = 0, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
SSingleWorkerCfg sCfg = {.min = 1, .max = 1, .name = "mnode-sync", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
|
||||
dError("failed to start mnode sync-worker since %s", terrstr());
|
||||
dError("failed to start mnode mnode-sync worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (tsMultiProcess) {
|
||||
SSingleWorkerCfg mCfg = {.min = 1, .max = 1, .name = "mnode-monitor", .fp = (FItem)mmProcessQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
dError("failed to start mnode mnode-monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
dDebug("mnode workers are initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void mmStopWorker(SMnodeMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->queryWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->readWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->writeWorker);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,34 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "qmInt.h"
|
||||
|
||||
void qmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonQmInfo *qmInfo) {
|
||||
}
|
||||
|
||||
int32_t qmProcessGetMonQmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonQmInfo qmInfo = {0};
|
||||
qmGetMonitorInfo(pWrapper, &qmInfo);
|
||||
dmGetMonitorSysInfo(&qmInfo.sys);
|
||||
monGetLogs(&qmInfo.log);
|
||||
|
||||
int32_t rspLen = tSerializeSMonQmInfo(NULL, 0, &qmInfo);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonQmInfo(pRsp, rspLen, &qmInfo);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonQmInfo(&qmInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SDnode *pDnode = pWrapper->pDnode;
|
||||
SRpcMsg *pReq = &pMsg->rpcMsg;
|
||||
|
|
@ -55,6 +83,8 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
}
|
||||
|
||||
void qmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_QM_INFO, qmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by VNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE);
|
||||
|
|
|
|||
|
|
@ -16,11 +16,36 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "qmInt.h"
|
||||
|
||||
static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
|
||||
static inline void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
|
||||
.ahandle = pMsg->rpcMsg.ahandle,
|
||||
.code = code,
|
||||
.pCont = pMsg->pRsp,
|
||||
.contLen = pMsg->rspLen};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
static void qmProcessMonQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
dTrace("msg:%p, get from qnode monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = &pMsg->rpcMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) {
|
||||
code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg);
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
qmSendRsp(pMgmt->pWrapper, pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
|
|
@ -66,6 +91,15 @@ int32_t qmProcessFetchMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t qmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SQnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) {
|
||||
SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg));
|
||||
if (pMsg == NULL) {
|
||||
|
|
@ -106,13 +140,8 @@ int32_t qmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
|
|||
}
|
||||
|
||||
int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
||||
int32_t maxFetchThreads = 4;
|
||||
int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
|
||||
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
|
||||
int32_t maxQueryThreads = minQueryThreads;
|
||||
|
||||
SSingleWorkerCfg queryCfg = {.min = minQueryThreads,
|
||||
.max = maxQueryThreads,
|
||||
SSingleWorkerCfg queryCfg = {.min = tsNumOfVnodeQueryThreads,
|
||||
.max = tsNumOfVnodeQueryThreads,
|
||||
.name = "qnode-query",
|
||||
.fp = (FItem)qmProcessQueryQueue,
|
||||
.param = pMgmt};
|
||||
|
|
@ -122,8 +151,8 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
SSingleWorkerCfg fetchCfg = {.min = minFetchThreads,
|
||||
.max = maxFetchThreads,
|
||||
SSingleWorkerCfg fetchCfg = {.min = tsNumOfQnodeFetchThreads,
|
||||
.max = tsNumOfQnodeFetchThreads,
|
||||
.name = "qnode-fetch",
|
||||
.fp = (FItem)qmProcessFetchQueue,
|
||||
.param = pMgmt};
|
||||
|
|
@ -133,11 +162,21 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (tsMultiProcess) {
|
||||
SSingleWorkerCfg mCfg = {
|
||||
.min = 1, .max = 1, .name = "qnode-monitor", .fp = (FItem)qmProcessMonQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
dError("failed to start qnode-monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
dDebug("qnode workers are initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void qmStopWorker(SQnodeMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->queryWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->fetchWorker);
|
||||
dDebug("qnode workers are closed");
|
||||
|
|
|
|||
|
|
@ -16,6 +16,33 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "smInt.h"
|
||||
|
||||
void smGetMonitorInfo(SMgmtWrapper *pWrapper, SMonSmInfo *smInfo) {}
|
||||
|
||||
int32_t smProcessGetMonSmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonSmInfo smInfo = {0};
|
||||
smGetMonitorInfo(pWrapper, &smInfo);
|
||||
dmGetMonitorSysInfo(&smInfo.sys);
|
||||
monGetLogs(&smInfo.log);
|
||||
|
||||
int32_t rspLen = tSerializeSMonSmInfo(NULL, 0, &smInfo);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonSmInfo(pRsp, rspLen, &smInfo);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonSmInfo(&smInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SDnode *pDnode = pWrapper->pDnode;
|
||||
SRpcMsg *pReq = &pMsg->rpcMsg;
|
||||
|
|
@ -55,6 +82,8 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
}
|
||||
|
||||
void smInitMsgHandle(SMgmtWrapper *pWrapper) {
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_SM_INFO, smProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by SNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,36 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "smInt.h"
|
||||
|
||||
static inline void smSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
|
||||
.ahandle = pMsg->rpcMsg.ahandle,
|
||||
.code = code,
|
||||
.pCont = pMsg->pRsp,
|
||||
.contLen = pMsg->rspLen};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
static void smProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
||||
SSnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
dTrace("msg:%p, get from snode monitor queue", pMsg);
|
||||
SRpcMsg *pRpc = &pMsg->rpcMsg;
|
||||
int32_t code = -1;
|
||||
|
||||
if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) {
|
||||
code = smProcessGetMonSmInfoReq(pMgmt->pWrapper, pMsg);
|
||||
}
|
||||
|
||||
if (pRpc->msgType & 1U) {
|
||||
if (code != 0 && terrno != 0) code = terrno;
|
||||
smSendRsp(pMgmt->pWrapper, pMsg, code);
|
||||
}
|
||||
|
||||
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
}
|
||||
|
||||
static void smProcessUniqueQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
|
||||
SSnodeMgmt *pMgmt = pInfo->ahandle;
|
||||
|
||||
|
|
@ -50,7 +80,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < SND_UNIQUE_THREAD_NUM; i++) {
|
||||
for (int32_t i = 0; i < tsNumOfSnodeUniqueThreads; i++) {
|
||||
SMultiWorker *pUniqueWorker = taosMemoryMalloc(sizeof(SMultiWorker));
|
||||
if (pUniqueWorker == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
|
@ -69,8 +99,8 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
|
|||
}
|
||||
}
|
||||
|
||||
SSingleWorkerCfg cfg = {.min = SND_SHARED_THREAD_NUM,
|
||||
.max = SND_SHARED_THREAD_NUM,
|
||||
SSingleWorkerCfg cfg = {.min = tsNumOfSnodeSharedThreads,
|
||||
.max = tsNumOfSnodeSharedThreads,
|
||||
.name = "snode-shared",
|
||||
.fp = (FItem)smProcessSharedQueue,
|
||||
.param = pMgmt};
|
||||
|
|
@ -80,11 +110,21 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (tsMultiProcess) {
|
||||
SSingleWorkerCfg mCfg = {
|
||||
.min = 1, .max = 1, .name = "snode-monitor", .fp = (FItem)smProcessMonitorQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
dError("failed to start snode-monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
dDebug("snode workers are initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void smStopWorker(SSnodeMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pMgmt->uniqueWorkers); i++) {
|
||||
SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, i);
|
||||
tMultiWorkerCleanup(pWorker);
|
||||
|
|
@ -97,7 +137,7 @@ void smStopWorker(SSnodeMgmt *pMgmt) {
|
|||
static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {
|
||||
SMsgHead *pHead = pMsg->pCont;
|
||||
pHead->vgId = htonl(pHead->vgId);
|
||||
return pHead->vgId % SND_UNIQUE_THREAD_NUM;
|
||||
return pHead->vgId % tsNumOfSnodeUniqueThreads;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) {
|
||||
|
|
@ -120,6 +160,15 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SSnodeMgmt *pMgmt = pWrapper->pMgmt;
|
||||
int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,71 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "vmInt.h"
|
||||
|
||||
void vmGetMonitorInfo(SMgmtWrapper *pWrapper, SMonVmInfo *vmInfo) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
tfsGetMonitorInfo(pMgmt->pTfs, &vmInfo->tfs);
|
||||
|
||||
taosWLockLatch(&pMgmt->latch);
|
||||
vmInfo->vstat.totalVnodes = pMgmt->state.totalVnodes;
|
||||
vmInfo->vstat.masterNum = pMgmt->state.masterNum;
|
||||
vmInfo->vstat.numOfSelectReqs = pMgmt->state.numOfSelectReqs - pMgmt->lastState.numOfSelectReqs;
|
||||
vmInfo->vstat.numOfInsertReqs = pMgmt->state.numOfInsertReqs - pMgmt->lastState.numOfInsertReqs;
|
||||
vmInfo->vstat.numOfInsertSuccessReqs = pMgmt->state.numOfInsertSuccessReqs - pMgmt->lastState.numOfInsertSuccessReqs;
|
||||
vmInfo->vstat.numOfBatchInsertReqs = pMgmt->state.numOfBatchInsertReqs - pMgmt->lastState.numOfBatchInsertReqs;
|
||||
vmInfo->vstat.numOfBatchInsertSuccessReqs =
|
||||
pMgmt->state.numOfBatchInsertSuccessReqs - pMgmt->lastState.numOfBatchInsertSuccessReqs;
|
||||
pMgmt->lastState = pMgmt->state;
|
||||
taosWUnLockLatch(&pMgmt->latch);
|
||||
}
|
||||
|
||||
int32_t vmProcessGetMonVmInfoReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonVmInfo vmInfo = {0};
|
||||
vmGetMonitorInfo(pWrapper, &vmInfo);
|
||||
dmGetMonitorSysInfo(&vmInfo.sys);
|
||||
monGetLogs(&vmInfo.log);
|
||||
|
||||
int32_t rspLen = tSerializeSMonVmInfo(NULL, 0, &vmInfo);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonVmInfo(pRsp, rspLen, &vmInfo);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonVmInfo(&vmInfo);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
|
||||
SMonVloadInfo vloads = {0};
|
||||
vmGetVnodeLoads(pWrapper, &vloads);
|
||||
|
||||
int32_t rspLen = tSerializeSMonVloadInfo(NULL, 0, &vloads);
|
||||
if (rspLen < 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void *pRsp = rpcMallocCont(rspLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tSerializeSMonVloadInfo(pRsp, rspLen, &vloads);
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = rspLen;
|
||||
tFreeSMonVloadInfo(&vloads);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
|
||||
pCfg->vgId = pCreate->vgId;
|
||||
pCfg->wsize = pCreate->cacheBlockSize;
|
||||
|
|
@ -239,6 +304,9 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
|
|||
}
|
||||
|
||||
void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_VM_INFO, vmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_MON_VM_LOAD, vmProcessMonitorMsg, DEFAULT_HANDLE);
|
||||
|
||||
// Requests handled by VNODE
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
|
||||
|
|
@ -271,6 +339,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
|
|||
dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
|
||||
dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
|
||||
|
|
|
|||
|
|
@ -344,31 +344,11 @@ void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
|
|||
pWrapper->fp = mgmtFp;
|
||||
}
|
||||
|
||||
int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
if (pMgmt == NULL) return -1;
|
||||
|
||||
return tfsGetMonitorInfo(pMgmt->pTfs, pInfo);
|
||||
}
|
||||
|
||||
void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
if (pMgmt == NULL) return;
|
||||
|
||||
SVnodesStat *pStat = &pMgmt->state;
|
||||
pInfo->req_select = pStat->numOfSelectReqs;
|
||||
pInfo->req_insert = pStat->numOfInsertReqs;
|
||||
pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
|
||||
pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
|
||||
pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
|
||||
pInfo->errors = tsNumOfErrorLogs;
|
||||
pInfo->vnodes_num = pStat->totalVnodes;
|
||||
pInfo->masters = pStat->masterNum;
|
||||
}
|
||||
|
||||
void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
|
||||
void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SVnodesStat *pStat = &pMgmt->state;
|
||||
SArray *pLoads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoad));
|
||||
|
||||
int32_t totalVnodes = 0;
|
||||
int32_t masterNum = 0;
|
||||
int64_t numOfSelectReqs = 0;
|
||||
|
|
@ -377,6 +357,9 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
|
|||
int64_t numOfBatchInsertReqs = 0;
|
||||
int64_t numOfBatchInsertSuccessReqs = 0;
|
||||
|
||||
pInfo->pVloads = pLoads;
|
||||
if (pLoads == NULL) return;
|
||||
|
||||
taosRLockLatch(&pMgmt->latch);
|
||||
|
||||
void *pIter = taosHashIterate(pMgmt->hash, NULL);
|
||||
|
|
@ -402,6 +385,7 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
|
|||
|
||||
taosRUnLockLatch(&pMgmt->latch);
|
||||
|
||||
taosWLockLatch(&pMgmt->latch);
|
||||
pStat->totalVnodes = totalVnodes;
|
||||
pStat->masterNum = masterNum;
|
||||
pStat->numOfSelectReqs = numOfSelectReqs;
|
||||
|
|
@ -409,4 +393,5 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads) {
|
|||
pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
|
||||
pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
|
||||
pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
|
||||
taosWUnLockLatch(&pMgmt->latch);
|
||||
}
|
||||
|
|
@ -16,8 +16,12 @@
|
|||
#define _DEFAULT_SOURCE
|
||||
#include "vmInt.h"
|
||||
|
||||
static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
|
||||
static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
|
||||
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
|
||||
.ahandle = pMsg->rpcMsg.ahandle,
|
||||
.code = code,
|
||||
.pCont = pMsg->pRsp,
|
||||
.contLen = pMsg->rspLen};
|
||||
tmsgSendRsp(&rsp);
|
||||
}
|
||||
|
||||
|
|
@ -26,9 +30,15 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
|
|||
|
||||
int32_t code = -1;
|
||||
tmsg_t msgType = pMsg->rpcMsg.msgType;
|
||||
dTrace("msg:%p, will be processed in vnode-mgmt queue", pMsg);
|
||||
dTrace("msg:%p, will be processed in vnode-m queue", pMsg);
|
||||
|
||||
switch (msgType) {
|
||||
case TDMT_MON_VM_INFO:
|
||||
code = vmProcessGetMonVmInfoReq(pMgmt->pWrapper, pMsg);
|
||||
break;
|
||||
case TDMT_MON_VM_LOAD:
|
||||
code = vmProcessGetVnodeLoadsReq(pMgmt->pWrapper, pMsg);
|
||||
break;
|
||||
case TDMT_DND_CREATE_VNODE:
|
||||
code = vmProcessCreateVnodeReq(pMgmt, pMsg);
|
||||
break;
|
||||
|
|
@ -255,6 +265,15 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SSingleWorker *pWorker = &pMgmt->monitorWorker;
|
||||
|
||||
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
|
||||
taosWriteQitem(pWorker->queue, pMsg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) {
|
||||
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
|
||||
SMsgHead *pHead = pRpc->pCont;
|
||||
|
|
@ -379,39 +398,31 @@ void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
|
|||
}
|
||||
|
||||
int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
|
||||
int32_t maxFetchThreads = 4;
|
||||
int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
|
||||
int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
|
||||
int32_t maxQueryThreads = minQueryThreads;
|
||||
int32_t maxWriteThreads = TMAX(tsNumOfCores, 1);
|
||||
int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1);
|
||||
int32_t maxMergeThreads = 1;
|
||||
|
||||
SQWorkerPool *pQPool = &pMgmt->queryPool;
|
||||
pQPool->name = "vnode-query";
|
||||
pQPool->min = minQueryThreads;
|
||||
pQPool->max = maxQueryThreads;
|
||||
pQPool->min = tsNumOfVnodeQueryThreads;
|
||||
pQPool->max = tsNumOfVnodeQueryThreads;
|
||||
if (tQWorkerInit(pQPool) != 0) return -1;
|
||||
|
||||
SQWorkerPool *pFPool = &pMgmt->fetchPool;
|
||||
pFPool->name = "vnode-fetch";
|
||||
pFPool->min = minFetchThreads;
|
||||
pFPool->max = maxFetchThreads;
|
||||
pFPool->min = tsNumOfVnodeFetchThreads;
|
||||
pFPool->max = tsNumOfVnodeFetchThreads;
|
||||
if (tQWorkerInit(pFPool) != 0) return -1;
|
||||
|
||||
SWWorkerPool *pWPool = &pMgmt->writePool;
|
||||
pWPool->name = "vnode-write";
|
||||
pWPool->max = maxWriteThreads;
|
||||
pWPool->max = tsNumOfVnodeWriteThreads;
|
||||
if (tWWorkerInit(pWPool) != 0) return -1;
|
||||
|
||||
pWPool = &pMgmt->syncPool;
|
||||
pWPool->name = "vnode-sync";
|
||||
pWPool->max = maxSyncThreads;
|
||||
pWPool->max = tsNumOfVnodeSyncThreads;
|
||||
if (tWWorkerInit(pWPool) != 0) return -1;
|
||||
|
||||
pWPool = &pMgmt->mergePool;
|
||||
pWPool->name = "vnode-merge";
|
||||
pWPool->max = maxMergeThreads;
|
||||
pWPool->max = tsNumOfVnodeMergeThreads;
|
||||
if (tWWorkerInit(pWPool) != 0) return -1;
|
||||
|
||||
SSingleWorkerCfg cfg = {.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
|
||||
|
|
@ -420,11 +431,21 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
if (tsMultiProcess) {
|
||||
SSingleWorkerCfg mCfg = {
|
||||
.min = 1, .max = 1, .name = "vnode-monitor", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
|
||||
if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) {
|
||||
dError("failed to start mnode vnode-monitor worker since %s", terrstr());
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
dDebug("vnode workers are initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
void vmStopWorker(SVnodesMgmt *pMgmt) {
|
||||
tSingleWorkerCleanup(&pMgmt->monitorWorker);
|
||||
tSingleWorkerCleanup(&pMgmt->mgmtWorker);
|
||||
tQWorkerCleanup(&pMgmt->fetchPool);
|
||||
tQWorkerCleanup(&pMgmt->queryPool);
|
||||
|
|
|
|||
|
|
@ -260,6 +260,7 @@ typedef struct {
|
|||
int32_t maxRows;
|
||||
int32_t commitTime;
|
||||
int32_t fsyncPeriod;
|
||||
int32_t ttl;
|
||||
int8_t walLevel;
|
||||
int8_t precision;
|
||||
int8_t compression;
|
||||
|
|
@ -268,6 +269,7 @@ typedef struct {
|
|||
int8_t update;
|
||||
int8_t cacheLastRow;
|
||||
int8_t streamMode;
|
||||
int8_t singleSTable;
|
||||
int32_t numOfRetensions;
|
||||
SArray* pRetensions;
|
||||
} SDbCfg;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode);
|
|||
void mndCleanupSma(SMnode *pMnode);
|
||||
SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName);
|
||||
void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma);
|
||||
int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include "mndTrans.h"
|
||||
#include "mndUser.h"
|
||||
#include "mndVgroup.h"
|
||||
#include "mndSma.h"
|
||||
|
||||
#define TSDB_DB_VER_NUMBER 1
|
||||
#define TSDB_DB_RESERVE_SIZE 64
|
||||
|
|
@ -40,6 +41,8 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq);
|
|||
static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
|
||||
static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
|
||||
static void mndCancelGetNextDb(SMnode *pMnode, void *pIter);
|
||||
static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq);
|
||||
static int32_t mndProcessGetIndexReq(SNodeMsg *pReq);
|
||||
|
||||
int32_t mndInitDb(SMnode *pMnode) {
|
||||
SSdbTable table = {.sdbType = SDB_DB,
|
||||
|
|
@ -56,6 +59,8 @@ int32_t mndInitDb(SMnode *pMnode) {
|
|||
mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq);
|
||||
|
||||
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta);
|
||||
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
|
||||
|
|
@ -268,6 +273,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
|
|||
if (pCfg->minRows > pCfg->maxRows) return -1;
|
||||
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
|
||||
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
|
||||
if (pCfg->ttl < TSDB_MIN_DB_TTL_OPTION && pCfg->ttl != TSDB_DEFAULT_DB_TTL_OPTION) return -1;
|
||||
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
|
||||
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
|
||||
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
|
||||
|
|
@ -278,6 +284,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
|
|||
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
|
||||
if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
|
||||
if (pCfg->streamMode < TSDB_MIN_DB_STREAM_MODE || pCfg->streamMode > TSDB_MAX_DB_STREAM_MODE) return -1;
|
||||
if (pCfg->singleSTable < TSDB_MIN_DB_SINGLE_STABLE_OPTION || pCfg->streamMode > TSDB_MAX_DB_SINGLE_STABLE_OPTION) return -1;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -293,6 +300,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
|
|||
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK;
|
||||
if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
|
||||
if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
|
||||
if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL_OPTION;
|
||||
if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
|
||||
if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
|
||||
if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
|
||||
|
|
@ -301,6 +309,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
|
|||
if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION;
|
||||
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
|
||||
if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE;
|
||||
if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE_OPTION;
|
||||
if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
|
||||
}
|
||||
|
||||
|
|
@ -437,6 +446,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
|
|||
.maxRows = pCreate->maxRows,
|
||||
.commitTime = pCreate->commitTime,
|
||||
.fsyncPeriod = pCreate->fsyncPeriod,
|
||||
.ttl = pCreate->ttl,
|
||||
.walLevel = pCreate->walLevel,
|
||||
.precision = pCreate->precision,
|
||||
.compression = pCreate->compression,
|
||||
|
|
@ -445,6 +455,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
|
|||
.update = pCreate->update,
|
||||
.cacheLastRow = pCreate->cacheLastRow,
|
||||
.streamMode = pCreate->streamMode,
|
||||
.singleSTable = pCreate->singleSTable,
|
||||
};
|
||||
|
||||
dbObj.cfg.numOfRetensions = pCreate->numOfRetensions;
|
||||
|
|
@ -730,6 +741,71 @@ ALTER_DB_OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
|
||||
SMnode *pMnode = pReq->pNode;
|
||||
int32_t code = -1;
|
||||
SDbObj *pDb = NULL;
|
||||
SDbCfgReq cfgReq = {0};
|
||||
SDbCfgRsp cfgRsp = {0};
|
||||
|
||||
if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
goto GET_DB_CFG_OVER;
|
||||
}
|
||||
|
||||
pDb = mndAcquireDb(pMnode, cfgReq.db);
|
||||
if (pDb == NULL) {
|
||||
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
|
||||
goto GET_DB_CFG_OVER;
|
||||
}
|
||||
|
||||
cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups;
|
||||
cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize;
|
||||
cfgRsp.totalBlocks = pDb->cfg.totalBlocks;
|
||||
cfgRsp.daysPerFile = pDb->cfg.daysPerFile;
|
||||
cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0;
|
||||
cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1;
|
||||
cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2;
|
||||
cfgRsp.minRows = pDb->cfg.minRows;
|
||||
cfgRsp.maxRows = pDb->cfg.maxRows;
|
||||
cfgRsp.commitTime = pDb->cfg.commitTime;
|
||||
cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod;
|
||||
cfgRsp.ttl = pDb->cfg.ttl;
|
||||
cfgRsp.walLevel = pDb->cfg.walLevel;
|
||||
cfgRsp.precision = pDb->cfg.precision;
|
||||
cfgRsp.compression = pDb->cfg.compression;
|
||||
cfgRsp.replications = pDb->cfg.replications;
|
||||
cfgRsp.quorum = pDb->cfg.quorum;
|
||||
cfgRsp.update = pDb->cfg.update;
|
||||
cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
|
||||
cfgRsp.streamMode = pDb->cfg.streamMode;
|
||||
cfgRsp.singleSTable = pDb->cfg.singleSTable;
|
||||
|
||||
int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp);
|
||||
void *pRsp = rpcMallocCont(contLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
code = -1;
|
||||
goto GET_DB_CFG_OVER;
|
||||
}
|
||||
|
||||
tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp);
|
||||
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = contLen;
|
||||
|
||||
GET_DB_CFG_OVER:
|
||||
|
||||
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
|
||||
mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr());
|
||||
}
|
||||
|
||||
mndReleaseDb(pMnode, pDb);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
|
||||
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
|
||||
if (pRedoRaw == NULL) return -1;
|
||||
|
|
@ -1509,6 +1585,23 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
|
|||
STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2);
|
||||
cols++;
|
||||
|
||||
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
|
||||
*(int32_t *)pWrite = pDb->cfg.ttl;
|
||||
cols++;
|
||||
|
||||
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
|
||||
*(int8_t *)pWrite = pDb->cfg.singleSTable;
|
||||
cols++;
|
||||
|
||||
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
|
||||
*(int8_t *)pWrite = pDb->cfg.streamMode;
|
||||
cols++;
|
||||
|
||||
pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
|
||||
char *status = "ready";
|
||||
STR_WITH_SIZE_TO_VARSTR(pWrite, status, strlen(status));
|
||||
cols++;
|
||||
|
||||
// pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
|
||||
// *(int8_t *)pWrite = pDb->cfg.update;
|
||||
}
|
||||
|
|
@ -1571,3 +1664,50 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) {
|
|||
SSdb *pSdb = pMnode->pSdb;
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
}
|
||||
|
||||
static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) {
|
||||
SUserIndexReq indexReq = {0};
|
||||
SMnode *pMnode = pReq->pNode;
|
||||
int32_t code = -1;
|
||||
SUserIndexRsp rsp = {0};
|
||||
bool exist = false;
|
||||
|
||||
if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist);
|
||||
if (code) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
if (!exist) {
|
||||
//TODO GET INDEX FROM FULLTEXT
|
||||
code = -1;
|
||||
terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST;
|
||||
} else {
|
||||
int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp);
|
||||
void *pRsp = rpcMallocCont(contLen);
|
||||
if (pRsp == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
code = -1;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
tSerializeSUserIndexRsp(pRsp, contLen, &rsp);
|
||||
|
||||
pReq->pRsp = pRsp;
|
||||
pReq->rspLen = contLen;
|
||||
|
||||
code = 0;
|
||||
}
|
||||
|
||||
_OVER:
|
||||
if (code != 0) {
|
||||
mError("failed to get index %s since %s", indexReq.indexFName, terrstr());
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
|
|||
|
||||
bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
|
||||
int64_t interval = TABS(pDnode->lastAccessTime - curMs);
|
||||
if (interval > 3500 * tsStatusInterval) {
|
||||
if (interval > 30000 * tsStatusInterval) {
|
||||
if (pDnode->rebootTime > 0) {
|
||||
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
|
||||
}
|
||||
|
|
@ -630,6 +630,7 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) {
|
|||
|
||||
static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
|
||||
mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t mndGetConfigMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
|
||||
|
||||
//!!!! Note: only APPEND columns in below tables, NO insert !!!!
|
||||
static const SInfosTableSchema dnodesSchema[] = {
|
||||
|
|
@ -63,7 +64,11 @@ static const SInfosTableSchema userDBSchema[] = {
|
|||
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
|
||||
{.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
|
||||
{.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
|
||||
{.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
|
||||
{.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
|
||||
{.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
|
||||
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
|
||||
// {.name = "update", .bytes = 1, .type =
|
||||
// TSDB_DATA_TYPE_TINYINT}, // disable update
|
||||
};
|
||||
|
|
@ -75,11 +80,11 @@ static const SInfosTableSchema userFuncSchema[] = {
|
|||
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
|
||||
};
|
||||
static const SInfosTableSchema userIdxSchema[] = {
|
||||
{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
|
||||
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -416,8 +416,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
|
|||
streamObj.version = 1;
|
||||
streamObj.sql = pCreate->sql;
|
||||
streamObj.createdBy = STREAM_CREATED_BY__SMA;
|
||||
// TODO
|
||||
streamObj.fixedSinkVgId = 0;
|
||||
streamObj.fixedSinkVgId = smaObj.dstVgId;
|
||||
streamObj.smaId = smaObj.uid;
|
||||
/*streamObj.physicalPlan = "";*/
|
||||
streamObj.logicalPlan = "not implemented";
|
||||
|
|
@ -689,6 +688,39 @@ _OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) {
|
||||
int32_t code = -1;
|
||||
SSmaObj *pSma = NULL;
|
||||
|
||||
pSma = mndAcquireSma(pMnode, indexReq->indexFName);
|
||||
if (pSma == NULL) {
|
||||
*exist = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy(rsp->dbFName, pSma->db, sizeof(pSma->db));
|
||||
memcpy(rsp->tblFName, pSma->stb, sizeof(pSma->stb));
|
||||
strcpy(rsp->indexType, TSDB_INDEX_TYPE_SMA);
|
||||
|
||||
SNodeList *pList = NULL;
|
||||
int32_t extOffset = 0;
|
||||
code = nodesStringToList(pSma->expr, &pList);
|
||||
if (0 == code) {
|
||||
SNode *node = NULL;
|
||||
FOREACH(node, pList) {
|
||||
SFunctionNode *pFunc = (SFunctionNode *)node;
|
||||
extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? ",":""), pFunc->functionName);
|
||||
}
|
||||
|
||||
*exist = true;
|
||||
}
|
||||
|
||||
mndReleaseSma(pMnode, pSma);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
|
||||
static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) {
|
||||
mndTransProcessRsp(pRsp);
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -60,7 +60,8 @@ static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);
|
|||
static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
|
||||
const SMqConsumerEp *pConsumerEp);
|
||||
|
||||
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
|
||||
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName);
|
||||
static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName);
|
||||
|
||||
int32_t mndInitSubscribe(SMnode *pMnode) {
|
||||
SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
|
||||
|
|
@ -74,6 +75,7 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
|
|||
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);
|
||||
mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
|
||||
mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);
|
||||
|
|
@ -100,12 +102,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
|
|||
return pSub;
|
||||
}
|
||||
|
||||
static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) {
|
||||
static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, const char* topicName) {
|
||||
SMqMVRebReq req = {
|
||||
.vgId = pConsumerEp->vgId,
|
||||
.oldConsumerId = pConsumerEp->oldConsumerId,
|
||||
.newConsumerId = pConsumerEp->consumerId,
|
||||
};
|
||||
req.topic = strdup(topicName);
|
||||
|
||||
int32_t tlen = tEncodeSMqMVRebReq(NULL, &req);
|
||||
void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
|
||||
|
|
@ -120,6 +123,7 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
|
|||
|
||||
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
tEncodeSMqMVRebReq(&abuf, &req);
|
||||
taosMemoryFree(req.topic);
|
||||
|
||||
*pBuf = buf;
|
||||
*pLen = tlen;
|
||||
|
|
@ -127,12 +131,12 @@ static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsume
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) {
|
||||
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* topicName) {
|
||||
ASSERT(pConsumerEp->oldConsumerId != -1);
|
||||
|
||||
void *buf;
|
||||
int32_t tlen;
|
||||
if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp) < 0) {
|
||||
if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -154,11 +158,14 @@ static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqC
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp) {
|
||||
SMqSetCVgReq req = {0};
|
||||
static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, const char* oldTopicName) {
|
||||
SMqCancelConnReq req = {0};
|
||||
req.consumerId = pConsumerEp->consumerId;
|
||||
req.vgId = pConsumerEp->vgId;
|
||||
req.epoch = pConsumerEp->epoch;
|
||||
strcpy(req.topicName, oldTopicName);
|
||||
|
||||
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
|
||||
int32_t tlen = tEncodeSMqCancelConnReq(NULL, &req);
|
||||
void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen);
|
||||
if (buf == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
|
@ -169,16 +176,16 @@ static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsum
|
|||
pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
|
||||
pMsgHead->vgId = htonl(pConsumerEp->vgId);
|
||||
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
|
||||
tEncodeSMqSetCVgReq(&abuf, &req);
|
||||
tEncodeSMqCancelConnReq(&abuf, &req);
|
||||
*pBuf = buf;
|
||||
*pLen = tlen;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp) {
|
||||
static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, const char* oldTopicName) {
|
||||
void *buf;
|
||||
int32_t tlen;
|
||||
if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp) < 0) {
|
||||
if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp, oldTopicName) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -189,7 +196,7 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
|
|||
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
|
||||
action.pCont = buf;
|
||||
action.contLen = sizeof(SMsgHead) + tlen;
|
||||
action.msgType = TDMT_VND_MQ_SET_CONN;
|
||||
action.msgType = TDMT_VND_MQ_CANCEL_CONN;
|
||||
|
||||
mndReleaseVgroup(pMnode, pVgObj);
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
|
|
@ -365,7 +372,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
|
|||
if (status == MQ_CONSUMER_STATUS__MODIFY) {
|
||||
int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics);
|
||||
for (int32_t i = 0; i < removeSz; i++) {
|
||||
char *topicName = taosArrayGet(pConsumer->recentRemovedTopics, i);
|
||||
char *topicName = taosArrayGetP(pConsumer->recentRemovedTopics, i);
|
||||
taosMemoryFree(topicName);
|
||||
}
|
||||
taosArrayClear(pConsumer->recentRemovedTopics);
|
||||
|
|
@ -497,10 +504,10 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
|
|||
pConsumerEp->epoch = 0;
|
||||
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
|
||||
|
||||
if (pConsumerEp->oldConsumerId == -1) {
|
||||
char topic[TSDB_TOPIC_FNAME_LEN];
|
||||
char cgroup[TSDB_CGROUP_LEN];
|
||||
mndSplitSubscribeKey(pSub->key, topic, cgroup);
|
||||
if (pConsumerEp->oldConsumerId == -1) {
|
||||
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
|
||||
|
||||
mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId,
|
||||
|
|
@ -512,7 +519,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
|
|||
mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "",
|
||||
pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
|
||||
|
||||
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
|
||||
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -797,7 +804,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
|
|||
int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
|
||||
for (int32_t vgi = 0; vgi < vgsz; vgi++) {
|
||||
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi);
|
||||
mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp);
|
||||
mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName);
|
||||
taosArrayPush(pSub->unassignedVg, pConsumerEp);
|
||||
}
|
||||
taosArrayRemove(pSub->consumers, ci);
|
||||
|
|
@ -844,7 +851,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
|
|||
pConsumerEp->consumerId);
|
||||
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
|
||||
} else {
|
||||
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp);
|
||||
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName);
|
||||
}
|
||||
// to trigger rebalance at once, do not set status active
|
||||
/*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/
|
||||
|
|
@ -859,7 +866,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
|
|||
}
|
||||
}
|
||||
|
||||
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);
|
||||
/*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/
|
||||
|
||||
// persist consumerObj
|
||||
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
|
||||
|
|
|
|||
|
|
@ -502,7 +502,11 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
|
|||
|
||||
SMonVgroupDesc desc = {0};
|
||||
desc.vgroup_id = pVgroup->vgId;
|
||||
strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name));
|
||||
|
||||
SName name = {0};
|
||||
tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
tNameGetDbName(&name, desc.database_name);
|
||||
|
||||
desc.tables_num = pVgroup->numOfTables;
|
||||
pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
|
||||
tstrncpy(desc.status, "unsynced", sizeof(desc.status));
|
||||
|
|
|
|||
|
|
@ -1,54 +1,76 @@
|
|||
set(META_DB_IMPL_LIST "BDB" "TDB")
|
||||
set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation")
|
||||
set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
|
||||
# vnode
|
||||
add_library(vnode STATIC "")
|
||||
target_sources(
|
||||
vnode
|
||||
PRIVATE
|
||||
# vnode
|
||||
"src/vnd/vnodeArenaMAImpl.c"
|
||||
"src/vnd/vnodeBufferPool.c"
|
||||
"src/vnd/vnodeCfg.c"
|
||||
"src/vnd/vnodeCommit.c"
|
||||
"src/vnd/vnodeInt.c"
|
||||
"src/vnd/vnodeMain.c"
|
||||
"src/vnd/vnodeMgr.c"
|
||||
"src/vnd/vnodeQuery.c"
|
||||
"src/vnd/vnodeStateMgr.c"
|
||||
"src/vnd/vnodeWrite.c"
|
||||
|
||||
if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
|
||||
message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
|
||||
else()
|
||||
message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
|
||||
endif()
|
||||
# meta
|
||||
# "src/meta/metaBDBImpl.c"
|
||||
"src/meta/metaCache.c"
|
||||
"src/meta/metaCfg.c"
|
||||
"src/meta/metaIdx.c"
|
||||
"src/meta/metaMain.c"
|
||||
"src/meta/metaQuery.c"
|
||||
"src/meta/metaTable.c"
|
||||
"src/meta/metaTbCfg.c"
|
||||
"src/meta/metaTbTag.c"
|
||||
"src/meta/metaTbUid.c"
|
||||
"src/meta/metaTDBImpl.c"
|
||||
|
||||
aux_source_directory(src/meta META_SRC)
|
||||
if(${META_DB_IMPL} STREQUAL "BDB")
|
||||
list(REMOVE_ITEM META_SRC "src/meta/metaTDBImpl.c")
|
||||
elseif(${META_DB_IMPL} STREQUAL "TDB")
|
||||
list(REMOVE_ITEM META_SRC "src/meta/metaBDBImpl.c")
|
||||
endif()
|
||||
# tsdb
|
||||
# "src/tsdb/tsdbBDBImpl.c"
|
||||
"src/tsdb/tsdbCommit.c"
|
||||
"src/tsdb/tsdbCompact.c"
|
||||
"src/tsdb/tsdbFile.c"
|
||||
"src/tsdb/tsdbFS.c"
|
||||
"src/tsdb/tsdbMain.c"
|
||||
"src/tsdb/tsdbMemTable.c"
|
||||
"src/tsdb/tsdbOptions.c"
|
||||
"src/tsdb/tsdbRead.c"
|
||||
"src/tsdb/tsdbReadImpl.c"
|
||||
"src/tsdb/tsdbScan.c"
|
||||
# "src/tsdb/tsdbSma.c"
|
||||
"src/tsdb/tsdbWrite.c"
|
||||
|
||||
aux_source_directory(src/tq TQ_SRC)
|
||||
aux_source_directory(src/tsdb TSDB_SRC)
|
||||
aux_source_directory(src/vnd VND_SRC)
|
||||
|
||||
list(APPEND
|
||||
VNODE_SRC
|
||||
${META_SRC}
|
||||
${TQ_SRC}
|
||||
${TSDB_SRC}
|
||||
${VND_SRC}
|
||||
# tq
|
||||
"src/tq/tq.c"
|
||||
"src/tq/tqCommit.c"
|
||||
"src/tq/tqMetaStore.c"
|
||||
"src/tq/tqOffset.c"
|
||||
"src/tq/tqPush.c"
|
||||
"src/tq/tqRead.c"
|
||||
)
|
||||
|
||||
add_library(vnode STATIC ${VNODE_SRC})
|
||||
target_include_directories(
|
||||
vnode
|
||||
PUBLIC inc
|
||||
PRIVATE src/inc
|
||||
PUBLIC "inc"
|
||||
PRIVATE "src/inc"
|
||||
)
|
||||
target_link_libraries(
|
||||
vnode
|
||||
PUBLIC os
|
||||
PUBLIC util
|
||||
PUBLIC common
|
||||
PUBLIC transport
|
||||
PUBLIC tfs
|
||||
PUBLIC wal
|
||||
PUBLIC scheduler
|
||||
PUBLIC executor
|
||||
PUBLIC stream
|
||||
PUBLIC qworker
|
||||
PUBLIC sync
|
||||
# TODO: get rid of BDB
|
||||
PUBLIC bdb
|
||||
PUBLIC executor
|
||||
PUBLIC scheduler
|
||||
PUBLIC tdb
|
||||
#PUBLIC bdb
|
||||
PUBLIC transport
|
||||
PUBLIC stream
|
||||
)
|
||||
|
||||
if(${BUILD_TEST})
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_META_H_
|
||||
#define _TD_META_H_
|
||||
|
||||
#include "tmallocator.h"
|
||||
#include "tmsg.h"
|
||||
#include "trow.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define META_SUPER_TABLE TD_SUPER_TABLE
|
||||
#define META_CHILD_TABLE TD_CHILD_TABLE
|
||||
#define META_NORMAL_TABLE TD_NORMAL_TABLE
|
||||
|
||||
// Types exported
|
||||
typedef struct SMeta SMeta;
|
||||
|
||||
typedef struct SMetaCfg {
|
||||
/// LRU cache size
|
||||
uint64_t lruSize;
|
||||
} SMetaCfg;
|
||||
|
||||
typedef struct SMTbCursor SMTbCursor;
|
||||
typedef struct SMCtbCursor SMCtbCursor;
|
||||
typedef struct SMSmaCursor SMSmaCursor;
|
||||
|
||||
typedef SVCreateTbReq STbCfg;
|
||||
typedef SVCreateTSmaReq SSmaCfg;
|
||||
|
||||
// SMeta operations
|
||||
SMeta * metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
|
||||
void metaClose(SMeta *pMeta);
|
||||
void metaRemove(const char *path);
|
||||
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
|
||||
int metaDropTable(SMeta *pMeta, tb_uid_t uid);
|
||||
int metaCommit(SMeta *pMeta);
|
||||
int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
|
||||
int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid);
|
||||
|
||||
// For Query
|
||||
STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
|
||||
STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
|
||||
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
|
||||
STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
|
||||
STSma * metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid);
|
||||
STSmaWrapper * metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid);
|
||||
SArray * metaGetSmaTbUids(SMeta *pMeta, bool isDup);
|
||||
int metaGetTbNum(SMeta *pMeta);
|
||||
|
||||
SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
|
||||
void metaCloseTbCursor(SMTbCursor *pTbCur);
|
||||
char * metaTbCursorNext(SMTbCursor *pTbCur);
|
||||
|
||||
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
|
||||
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
|
||||
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur);
|
||||
|
||||
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
|
||||
void metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
|
||||
const char * metaSmaCursorNext(SMSmaCursor *pSmaCur);
|
||||
|
||||
// Options
|
||||
void metaOptionsInit(SMetaCfg *pMetaCfg);
|
||||
void metaOptionsClear(SMetaCfg *pMetaCfg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_META_H_*/
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_H_
|
||||
#define _TD_TSDB_H_
|
||||
|
||||
#include "tmallocator.h"
|
||||
#include "meta.h"
|
||||
#include "tcommon.h"
|
||||
#include "tfs.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct SDataStatis {
|
||||
int16_t colId;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
} SDataStatis;
|
||||
|
||||
typedef struct STable {
|
||||
uint64_t tid;
|
||||
uint64_t uid;
|
||||
STSchema *pSchema;
|
||||
} STable;
|
||||
|
||||
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
|
||||
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
|
||||
#define BLOCK_LOAD_TABLE_RR_ORDER 3
|
||||
|
||||
#define TABLE_TID(t) (t)->tid
|
||||
#define TABLE_UID(t) (t)->uid
|
||||
|
||||
// TYPES EXPOSED
|
||||
typedef struct STsdb STsdb;
|
||||
|
||||
typedef struct STsdbCfg {
|
||||
int8_t precision;
|
||||
int8_t update;
|
||||
int8_t compression;
|
||||
int32_t daysPerFile;
|
||||
int32_t minRowsPerFileBlock;
|
||||
int32_t maxRowsPerFileBlock;
|
||||
int32_t keep;
|
||||
int32_t keep1;
|
||||
int32_t keep2;
|
||||
uint64_t lruCacheSize;
|
||||
SArray *retentions;
|
||||
} STsdbCfg;
|
||||
|
||||
// query condition to build multi-table data block iterator
|
||||
typedef struct STsdbQueryCond {
|
||||
STimeWindow twindow;
|
||||
int32_t order; // desc|asc order to iterate the data block
|
||||
int32_t numOfCols;
|
||||
SColumnInfo *colList;
|
||||
bool loadExternalRows; // load external rows or not
|
||||
int32_t type; // data block load type:
|
||||
} STsdbQueryCond;
|
||||
|
||||
typedef struct {
|
||||
TSKEY lastKey;
|
||||
uint64_t uid;
|
||||
} STableKeyInfo;
|
||||
|
||||
// STsdb
|
||||
STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
|
||||
void tsdbClose(STsdb *);
|
||||
void tsdbRemove(const char *path);
|
||||
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
|
||||
int tsdbPrepareCommit(STsdb *pTsdb);
|
||||
int tsdbCommit(STsdb *pTsdb);
|
||||
|
||||
/**
|
||||
* @brief When submit msg received, update the relative expired window synchronously.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg);
|
||||
|
||||
/**
|
||||
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param indexUid
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
|
||||
|
||||
/**
|
||||
* @brief Drop tSma data and local cache.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param indexUid
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
|
||||
|
||||
/**
|
||||
* @brief Insert RSma(Rollup SMA) data.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
|
||||
|
||||
// TODO: This is the basic params, and should wrap the params to a queryHandle.
|
||||
/**
|
||||
* @brief Get tSma(Time-range-wise SMA) data.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param pData
|
||||
* @param indexUid
|
||||
* @param querySKey
|
||||
* @param nMaxResult
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
|
||||
|
||||
// STsdbCfg
|
||||
int tsdbOptionsInit(STsdbCfg *);
|
||||
void tsdbOptionsClear(STsdbCfg *);
|
||||
|
||||
typedef void* tsdbReaderT;
|
||||
|
||||
/**
|
||||
* Get the data block iterator, starting from position according to the query condition
|
||||
*
|
||||
* @param tsdb tsdb handle
|
||||
* @param pCond query condition, including time window, result set order, and basic required columns for each block
|
||||
* @param tableInfoGroup table object list in the form of set, grouped into different sets according to the
|
||||
* group by condition
|
||||
* @param qinfo query info handle from query processor
|
||||
* @return
|
||||
*/
|
||||
tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId);
|
||||
|
||||
/**
|
||||
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
|
||||
* Note that only one data block with only row will be returned while invoking retrieve data block function for
|
||||
* all tables in this group.
|
||||
*
|
||||
* @param tsdb tsdb handle
|
||||
* @param pCond query condition, including time window, result set order, and basic required columns for each block
|
||||
* @param tableInfo table list.
|
||||
* @return
|
||||
*/
|
||||
//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
|
||||
// SMemRef *pRef);
|
||||
|
||||
|
||||
tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
|
||||
|
||||
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo);
|
||||
|
||||
bool isTsdbCacheLastRow(tsdbReaderT* pReader);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param tsdb
|
||||
* @param uid
|
||||
* @param skey
|
||||
* @param pTagCond
|
||||
* @param len
|
||||
* @param tagNameRelType
|
||||
* @param tbnameCond
|
||||
* @param pGroupInfo
|
||||
* @param pColIndex
|
||||
* @param numOfCols
|
||||
* @param reqId
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
|
||||
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
|
||||
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId);
|
||||
/**
|
||||
* get num of rows in mem table
|
||||
*
|
||||
* @param pHandle
|
||||
* @return row size
|
||||
*/
|
||||
|
||||
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle);
|
||||
|
||||
/**
|
||||
* move to next block if exists
|
||||
*
|
||||
* @param pTsdbReadHandle
|
||||
* @return
|
||||
*/
|
||||
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
|
||||
|
||||
/**
|
||||
* Get current data block information
|
||||
*
|
||||
* @param pTsdbReadHandle
|
||||
* @param pBlockInfo
|
||||
* @return
|
||||
*/
|
||||
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
|
||||
|
||||
/**
|
||||
*
|
||||
* Get the pre-calculated information w.r.t. current data block.
|
||||
*
|
||||
* In case of data block in cache, the pBlockStatis will always be NULL.
|
||||
* If a block is not completed loaded from disk, the pBlockStatis will be NULL.
|
||||
|
||||
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis);
|
||||
|
||||
/**
|
||||
*
|
||||
* The query condition with primary timestamp is passed to iterator during its constructor function,
|
||||
* the returned data block must be satisfied with the time window condition in any cases,
|
||||
* which means the SData data block is not actually the completed disk data blocks.
|
||||
*
|
||||
* @param pTsdbReadHandle query handle
|
||||
* @param pColumnIdList required data columns id list
|
||||
* @return
|
||||
*/
|
||||
SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
|
||||
|
||||
/**
|
||||
* destroy the created table group list, which is generated by tag query
|
||||
* @param pGroupList
|
||||
*/
|
||||
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
|
||||
|
||||
/**
|
||||
* create the table group result including only one table, used to handle the normal table query
|
||||
*
|
||||
* @param tsdb tsdbHandle
|
||||
* @param uid table uid
|
||||
* @param pGroupInfo the generated result
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param tsdb
|
||||
* @param pTableIdList
|
||||
* @param pGroupInfo
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
|
||||
|
||||
/**
|
||||
* clean up the query handle
|
||||
* @param queryHandle
|
||||
*/
|
||||
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
|
||||
|
||||
int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_H_*/
|
||||
|
|
@ -21,12 +21,17 @@
|
|||
#include "tqueue.h"
|
||||
#include "trpc.h"
|
||||
|
||||
#include "meta.h"
|
||||
#include "tarray.h"
|
||||
#include "tfs.h"
|
||||
#include "tsdb.h"
|
||||
#include "wal.h"
|
||||
|
||||
#include "tmallocator.h"
|
||||
#include "tmsg.h"
|
||||
#include "trow.h"
|
||||
#include "tmallocator.h"
|
||||
#include "tcommon.h"
|
||||
#include "tfs.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
|
@ -34,6 +39,82 @@ extern "C" {
|
|||
/* ------------------------ TYPES EXPOSED ------------------------ */
|
||||
typedef struct SMgmtWrapper SMgmtWrapper;
|
||||
typedef struct SVnode SVnode;
|
||||
|
||||
#define META_SUPER_TABLE TD_SUPER_TABLE
|
||||
#define META_CHILD_TABLE TD_CHILD_TABLE
|
||||
#define META_NORMAL_TABLE TD_NORMAL_TABLE
|
||||
|
||||
// Types exported
|
||||
typedef struct SMeta SMeta;
|
||||
|
||||
typedef struct SMetaCfg {
|
||||
/// LRU cache size
|
||||
uint64_t lruSize;
|
||||
} SMetaCfg;
|
||||
|
||||
typedef struct SMTbCursor SMTbCursor;
|
||||
typedef struct SMCtbCursor SMCtbCursor;
|
||||
typedef struct SMSmaCursor SMSmaCursor;
|
||||
|
||||
typedef SVCreateTbReq STbCfg;
|
||||
typedef SVCreateTSmaReq SSmaCfg;
|
||||
|
||||
typedef struct SDataStatis {
|
||||
int16_t colId;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
} SDataStatis;
|
||||
|
||||
typedef struct STsdbQueryCond {
|
||||
STimeWindow twindow;
|
||||
int32_t order; // desc|asc order to iterate the data block
|
||||
int32_t numOfCols;
|
||||
SColumnInfo *colList;
|
||||
bool loadExternalRows; // load external rows or not
|
||||
int32_t type; // data block load type:
|
||||
} STsdbQueryCond;
|
||||
|
||||
typedef struct {
|
||||
TSKEY lastKey;
|
||||
uint64_t uid;
|
||||
} STableKeyInfo;
|
||||
|
||||
|
||||
typedef struct STable {
|
||||
uint64_t tid;
|
||||
uint64_t uid;
|
||||
STSchema *pSchema;
|
||||
} STable;
|
||||
|
||||
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
|
||||
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2
|
||||
#define BLOCK_LOAD_TABLE_RR_ORDER 3
|
||||
|
||||
#define TABLE_TID(t) (t)->tid
|
||||
#define TABLE_UID(t) (t)->uid
|
||||
|
||||
// TYPES EXPOSED
|
||||
typedef struct STsdb STsdb;
|
||||
|
||||
typedef struct STsdbCfg {
|
||||
int8_t precision;
|
||||
int8_t update;
|
||||
int8_t compression;
|
||||
int32_t daysPerFile;
|
||||
int32_t minRowsPerFileBlock;
|
||||
int32_t maxRowsPerFileBlock;
|
||||
int32_t keep;
|
||||
int32_t keep1;
|
||||
int32_t keep2;
|
||||
uint64_t lruCacheSize;
|
||||
SArray *retentions;
|
||||
} STsdbCfg;
|
||||
|
||||
|
||||
typedef struct {
|
||||
// TODO
|
||||
int32_t reserved;
|
||||
|
|
@ -263,6 +344,241 @@ int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockIn
|
|||
// return SArray<SColumnInfoData>
|
||||
SArray *tqRetrieveDataBlock(STqReadHandle *pHandle);
|
||||
|
||||
// meta.h
|
||||
SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
|
||||
void metaClose(SMeta *pMeta);
|
||||
void metaRemove(const char *path);
|
||||
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
|
||||
int metaDropTable(SMeta *pMeta, tb_uid_t uid);
|
||||
int metaCommit(SMeta *pMeta);
|
||||
int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
|
||||
int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid);
|
||||
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
|
||||
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
|
||||
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
|
||||
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
|
||||
STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid);
|
||||
STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid);
|
||||
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup);
|
||||
int metaGetTbNum(SMeta *pMeta);
|
||||
SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
|
||||
void metaCloseTbCursor(SMTbCursor *pTbCur);
|
||||
char *metaTbCursorNext(SMTbCursor *pTbCur);
|
||||
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
|
||||
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
|
||||
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur);
|
||||
|
||||
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
|
||||
void metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
|
||||
const char *metaSmaCursorNext(SMSmaCursor *pSmaCur);
|
||||
|
||||
// Options
|
||||
void metaOptionsInit(SMetaCfg *pMetaCfg);
|
||||
void metaOptionsClear(SMetaCfg *pMetaCfg);
|
||||
|
||||
// query condition to build multi-table data block iterator
|
||||
// STsdb
|
||||
STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs);
|
||||
void tsdbClose(STsdb *);
|
||||
void tsdbRemove(const char *path);
|
||||
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
|
||||
int tsdbPrepareCommit(STsdb *pTsdb);
|
||||
int tsdbCommit(STsdb *pTsdb);
|
||||
|
||||
|
||||
int32_t tsdbInitSma(STsdb *pTsdb);
|
||||
int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
|
||||
int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
|
||||
/**
|
||||
* @brief When submit msg received, update the relative expired window synchronously.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg);
|
||||
|
||||
/**
|
||||
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param indexUid
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
|
||||
|
||||
/**
|
||||
* @brief Drop tSma data and local cache.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param indexUid
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
|
||||
|
||||
/**
|
||||
* @brief Insert RSma(Rollup SMA) data.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param msg
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
|
||||
|
||||
// TODO: This is the basic params, and should wrap the params to a queryHandle.
|
||||
/**
|
||||
* @brief Get tSma(Time-range-wise SMA) data.
|
||||
*
|
||||
* @param pTsdb
|
||||
* @param pData
|
||||
* @param indexUid
|
||||
* @param querySKey
|
||||
* @param nMaxResult
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
|
||||
|
||||
// STsdbCfg
|
||||
int tsdbOptionsInit(STsdbCfg *);
|
||||
void tsdbOptionsClear(STsdbCfg *);
|
||||
|
||||
typedef void* tsdbReaderT;
|
||||
|
||||
/**
|
||||
* Get the data block iterator, starting from position according to the query condition
|
||||
*
|
||||
* @param tsdb tsdb handle
|
||||
* @param pCond query condition, including time window, result set order, and basic required columns for each block
|
||||
* @param tableInfoGroup table object list in the form of set, grouped into different sets according to the
|
||||
* group by condition
|
||||
* @param qinfo query info handle from query processor
|
||||
* @return
|
||||
*/
|
||||
tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId);
|
||||
|
||||
/**
|
||||
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
|
||||
* Note that only one data block with only row will be returned while invoking retrieve data block function for
|
||||
* all tables in this group.
|
||||
*
|
||||
* @param tsdb tsdb handle
|
||||
* @param pCond query condition, including time window, result set order, and basic required columns for each block
|
||||
* @param tableInfo table list.
|
||||
* @return
|
||||
*/
|
||||
//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId,
|
||||
// SMemRef *pRef);
|
||||
|
||||
|
||||
tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
|
||||
|
||||
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* pReader, STableBlockDistInfo* pTableBlockInfo);
|
||||
|
||||
bool isTsdbCacheLastRow(tsdbReaderT* pReader);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param tsdb
|
||||
* @param uid
|
||||
* @param skey
|
||||
* @param pTagCond
|
||||
* @param len
|
||||
* @param tagNameRelType
|
||||
* @param tbnameCond
|
||||
* @param pGroupInfo
|
||||
* @param pColIndex
|
||||
* @param numOfCols
|
||||
* @param reqId
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
|
||||
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
|
||||
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId);
|
||||
/**
|
||||
* get num of rows in mem table
|
||||
*
|
||||
* @param pHandle
|
||||
* @return row size
|
||||
*/
|
||||
|
||||
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle);
|
||||
|
||||
/**
|
||||
* move to next block if exists
|
||||
*
|
||||
* @param pTsdbReadHandle
|
||||
* @return
|
||||
*/
|
||||
bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
|
||||
|
||||
/**
|
||||
* Get current data block information
|
||||
*
|
||||
* @param pTsdbReadHandle
|
||||
* @param pBlockInfo
|
||||
* @return
|
||||
*/
|
||||
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
|
||||
|
||||
/**
|
||||
*
|
||||
* Get the pre-calculated information w.r.t. current data block.
|
||||
*
|
||||
* In case of data block in cache, the pBlockStatis will always be NULL.
|
||||
* If a block is not completed loaded from disk, the pBlockStatis will be NULL.
|
||||
|
||||
* @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis);
|
||||
|
||||
/**
|
||||
*
|
||||
* The query condition with primary timestamp is passed to iterator during its constructor function,
|
||||
* the returned data block must be satisfied with the time window condition in any cases,
|
||||
* which means the SData data block is not actually the completed disk data blocks.
|
||||
*
|
||||
* @param pTsdbReadHandle query handle
|
||||
* @param pColumnIdList required data columns id list
|
||||
* @return
|
||||
*/
|
||||
SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
|
||||
|
||||
/**
|
||||
* destroy the created table group list, which is generated by tag query
|
||||
* @param pGroupList
|
||||
*/
|
||||
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
|
||||
|
||||
/**
|
||||
* create the table group result including only one table, used to handle the normal table query
|
||||
*
|
||||
* @param tsdb tsdbHandle
|
||||
* @param uid table uid
|
||||
* @param pGroupInfo the generated result
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param tsdb
|
||||
* @param pTableIdList
|
||||
* @param pGroupInfo
|
||||
* @return
|
||||
*/
|
||||
int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
|
||||
|
||||
/**
|
||||
* clean up the query handle
|
||||
* @param queryHandle
|
||||
*/
|
||||
void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
|
||||
|
||||
int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -13,12 +13,8 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_META_DEF_H_
|
||||
#define _TD_META_DEF_H_
|
||||
|
||||
#include "tmallocator.h"
|
||||
|
||||
#include "meta.h"
|
||||
#ifndef _TD_VNODE_META_H_
|
||||
#define _TD_VNODE_META_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -78,4 +74,4 @@ struct SMeta {
|
|||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_META_DEF_H_*/
|
||||
#endif /*_TD_VNODE_META_H_*/
|
||||
|
|
@ -13,18 +13,14 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TQ_INT_H_
|
||||
#define _TD_TQ_INT_H_
|
||||
|
||||
#include "meta.h"
|
||||
#include "tlog.h"
|
||||
#include "tqPush.h"
|
||||
#include "vnd.h"
|
||||
#ifndef _TD_VNODE_TQ_H_
|
||||
#define _TD_VNODE_TQ_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// tqInt.h
|
||||
#define tqFatal(...) \
|
||||
{ \
|
||||
if (tqDebugFlag & DEBUG_FATAL) { \
|
||||
|
|
@ -67,7 +63,7 @@ extern "C" {
|
|||
} \
|
||||
}
|
||||
|
||||
#define TQ_BUFFER_SIZE 8
|
||||
#define TQ_BUFFER_SIZE 4
|
||||
|
||||
#define TQ_BUCKET_MASK 0xFF
|
||||
#define TQ_BUCKET_SIZE 256
|
||||
|
|
@ -106,6 +102,9 @@ static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
|
|||
|
||||
typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
|
||||
|
||||
typedef struct STqOffsetCfg STqOffsetCfg;
|
||||
typedef struct STqOffsetStore STqOffsetStore;
|
||||
|
||||
typedef struct {
|
||||
int16_t ver;
|
||||
int16_t action;
|
||||
|
|
@ -211,13 +210,93 @@ typedef struct {
|
|||
SArray* topics; // SArray<STqTopic>
|
||||
} STqConsumer;
|
||||
|
||||
enum {
|
||||
TQ_PUSHER_TYPE__CLIENT = 1,
|
||||
TQ_PUSHER_TYPE__STREAM,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
int8_t reserved[3];
|
||||
int32_t ttl;
|
||||
int64_t consumerId;
|
||||
SRpcMsg* pMsg;
|
||||
// SMqPollRsp* rsp;
|
||||
} STqClientPusher;
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
int8_t nodeType;
|
||||
int8_t reserved[6];
|
||||
int64_t streamId;
|
||||
qTaskInfo_t task;
|
||||
// TODO sync function
|
||||
} STqStreamPusher;
|
||||
|
||||
typedef struct {
|
||||
int8_t type; // mq or stream
|
||||
} STqPusher;
|
||||
|
||||
typedef struct {
|
||||
SHashObj* pHash; // <id, STqPush*>
|
||||
} STqPushMgr;
|
||||
|
||||
typedef struct {
|
||||
int8_t inited;
|
||||
tmr_h timer;
|
||||
} STqPushMgmt;
|
||||
|
||||
static STqPushMgmt tqPushMgmt;
|
||||
|
||||
|
||||
int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
|
||||
int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);
|
||||
|
||||
static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }
|
||||
|
||||
// tqMetaStore.h
|
||||
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
|
||||
FTqDelete pDeleter, int32_t tqConfigFlag);
|
||||
int32_t tqStoreClose(STqMetaStore*);
|
||||
// int32_t tqStoreDelete(TqMetaStore*);
|
||||
// int32_t tqStoreCommitAll(TqMetaStore*);
|
||||
int32_t tqStorePersist(STqMetaStore*);
|
||||
// clean deleted idx and data from persistent file
|
||||
int32_t tqStoreCompact(STqMetaStore*);
|
||||
|
||||
void* tqHandleGet(STqMetaStore*, int64_t key);
|
||||
// make it unpersist
|
||||
void* tqHandleTouchGet(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
|
||||
int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
|
||||
// delete committed kv pair
|
||||
// notice that a delete action still needs to be committed
|
||||
int32_t tqHandleDel(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandlePurge(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleCommit(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleAbort(STqMetaStore*, int64_t key);
|
||||
|
||||
// tqOffset
|
||||
STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
|
||||
void STqOffsetClose(STqOffsetStore*);
|
||||
|
||||
int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset);
|
||||
int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
|
||||
|
||||
// tqPush
|
||||
int32_t tqPushMgrInit();
|
||||
void tqPushMgrCleanUp();
|
||||
|
||||
STqPushMgr* tqPushMgrOpen();
|
||||
void tqPushMgrClose(STqPushMgr* pushMgr);
|
||||
|
||||
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
|
||||
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TQ_INT_H_*/
|
||||
#endif /*_TD_VNODE_TQ_H_*/
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TQ_META_STORE_H_
|
||||
#define _TQ_META_STORE_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tqInt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
|
||||
FTqDelete pDeleter, int32_t tqConfigFlag);
|
||||
int32_t tqStoreClose(STqMetaStore*);
|
||||
// int32_t tqStoreDelete(TqMetaStore*);
|
||||
// int32_t tqStoreCommitAll(TqMetaStore*);
|
||||
int32_t tqStorePersist(STqMetaStore*);
|
||||
// clean deleted idx and data from persistent file
|
||||
int32_t tqStoreCompact(STqMetaStore*);
|
||||
|
||||
void* tqHandleGet(STqMetaStore*, int64_t key);
|
||||
// make it unpersist
|
||||
void* tqHandleTouchGet(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
|
||||
int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
|
||||
// delete committed kv pair
|
||||
// notice that a delete action still needs to be committed
|
||||
int32_t tqHandleDel(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandlePurge(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleCommit(STqMetaStore*, int64_t key);
|
||||
int32_t tqHandleAbort(STqMetaStore*, int64_t key);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ifndef _TQ_META_STORE_H_ */
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TQ_OFFSET_H_
|
||||
#define _TD_TQ_OFFSET_H_
|
||||
|
||||
#include "tqInt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct STqOffsetCfg STqOffsetCfg;
|
||||
typedef struct STqOffsetStore STqOffsetStore;
|
||||
|
||||
STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
|
||||
void STqOffsetClose(STqOffsetStore*);
|
||||
|
||||
int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset);
|
||||
int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TQ_OFFSET_H_*/
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TQ_PUSH_H_
|
||||
#define _TQ_PUSH_H_
|
||||
|
||||
#include "executor.h"
|
||||
#include "thash.h"
|
||||
#include "trpc.h"
|
||||
#include "ttimer.h"
|
||||
#include "vnode.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum {
|
||||
TQ_PUSHER_TYPE__CLIENT = 1,
|
||||
TQ_PUSHER_TYPE__STREAM,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
int8_t reserved[3];
|
||||
int32_t ttl;
|
||||
int64_t consumerId;
|
||||
SRpcMsg* pMsg;
|
||||
// SMqPollRsp* rsp;
|
||||
} STqClientPusher;
|
||||
|
||||
typedef struct {
|
||||
int8_t type;
|
||||
int8_t nodeType;
|
||||
int8_t reserved[6];
|
||||
int64_t streamId;
|
||||
qTaskInfo_t task;
|
||||
// TODO sync function
|
||||
} STqStreamPusher;
|
||||
|
||||
typedef struct {
|
||||
int8_t type; // mq or stream
|
||||
} STqPusher;
|
||||
|
||||
typedef struct {
|
||||
SHashObj* pHash; // <id, STqPush*>
|
||||
} STqPushMgr;
|
||||
|
||||
typedef struct {
|
||||
int8_t inited;
|
||||
tmr_h timer;
|
||||
} STqPushMgmt;
|
||||
|
||||
static STqPushMgmt tqPushMgmt;
|
||||
|
||||
int32_t tqPushMgrInit();
|
||||
void tqPushMgrCleanUp();
|
||||
|
||||
STqPushMgr* tqPushMgrOpen();
|
||||
void tqPushMgrClose(STqPushMgr* pushMgr);
|
||||
|
||||
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
|
||||
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TQ_PUSH_H_*/
|
||||
|
|
@ -0,0 +1,936 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_VNODE_TSDB_H_
|
||||
#define _TD_VNODE_TSDB_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct SSmaStat SSmaStat;
|
||||
typedef struct SSmaEnv SSmaEnv;
|
||||
typedef struct SSmaEnvs SSmaEnvs;
|
||||
|
||||
typedef enum {
|
||||
TSDB_FILE_HEAD = 0, // .head
|
||||
TSDB_FILE_DATA, // .data
|
||||
TSDB_FILE_LAST, // .last
|
||||
TSDB_FILE_SMAD, // .smad(Block-wise SMA)
|
||||
TSDB_FILE_SMAL, // .smal(Block-wise SMA)
|
||||
TSDB_FILE_MAX, //
|
||||
TSDB_FILE_META, // meta
|
||||
TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA
|
||||
TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
|
||||
} E_TSDB_FILE_T;
|
||||
|
||||
typedef struct {
|
||||
uint32_t magic;
|
||||
uint32_t fver;
|
||||
uint32_t len;
|
||||
uint32_t totalBlocks;
|
||||
uint32_t totalSubBlocks;
|
||||
uint32_t offset;
|
||||
uint64_t size;
|
||||
uint64_t tombSize;
|
||||
} SDFInfo;
|
||||
|
||||
typedef struct {
|
||||
SDFInfo info;
|
||||
STfsFile f;
|
||||
TdFilePtr pFile;
|
||||
uint8_t state;
|
||||
} SDFile;
|
||||
|
||||
struct SSmaEnvs {
|
||||
int16_t nTSma;
|
||||
int16_t nRSma;
|
||||
SSmaEnv *pTSmaEnv;
|
||||
SSmaEnv *pRSmaEnv;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int fid;
|
||||
int8_t state; // -128~127
|
||||
uint8_t ver; // 0~255, DFileSet version
|
||||
uint16_t reserve;
|
||||
SDFile files[TSDB_FILE_MAX];
|
||||
} SDFileSet;
|
||||
|
||||
typedef struct {
|
||||
int minFid;
|
||||
int midFid;
|
||||
int maxFid;
|
||||
TSKEY minKey;
|
||||
} SRtn;
|
||||
|
||||
typedef struct STbData {
|
||||
tb_uid_t uid;
|
||||
TSKEY keyMin;
|
||||
TSKEY keyMax;
|
||||
int64_t nrows;
|
||||
SSkipList *pData;
|
||||
} STbData;
|
||||
|
||||
typedef struct STsdbMemTable {
|
||||
T_REF_DECLARE()
|
||||
SRWLatch latch;
|
||||
TSKEY keyMin;
|
||||
TSKEY keyMax;
|
||||
uint64_t nRow;
|
||||
SMemAllocator *pMA;
|
||||
// Container
|
||||
SSkipList *pSlIdx; // SSkiplist<STbData>
|
||||
SHashObj * pHashIdx;
|
||||
} STsdbMemTable;
|
||||
|
||||
typedef struct {
|
||||
uint32_t version; // Commit version from 0 to increase
|
||||
int64_t totalPoints; // total points
|
||||
int64_t totalStorage; // Uncompressed total storage
|
||||
} STsdbFSMeta;
|
||||
|
||||
// ==================
|
||||
typedef struct {
|
||||
STsdbFSMeta meta; // FS meta
|
||||
SArray * df; // data file array
|
||||
SArray * sf; // sma data file array v2f1900.index_name_1
|
||||
} SFSStatus;
|
||||
|
||||
typedef struct {
|
||||
TdThreadRwlock lock;
|
||||
|
||||
SFSStatus *cstatus; // current status
|
||||
SHashObj * metaCache; // meta cache
|
||||
SHashObj * metaCacheComp; // meta cache for compact
|
||||
bool intxn;
|
||||
SFSStatus *nstatus; // new status
|
||||
} STsdbFS;
|
||||
|
||||
struct STsdb {
|
||||
int32_t vgId;
|
||||
bool repoLocked;
|
||||
TdThreadMutex mutex;
|
||||
char * path;
|
||||
STsdbCfg config;
|
||||
STsdbMemTable * mem;
|
||||
STsdbMemTable * imem;
|
||||
SRtn rtn;
|
||||
SMemAllocatorFactory *pmaf;
|
||||
STsdbFS * fs;
|
||||
SMeta * pMeta;
|
||||
STfs * pTfs;
|
||||
SSmaEnvs smaEnvs;
|
||||
};
|
||||
|
||||
#define REPO_ID(r) ((r)->vgId)
|
||||
#define REPO_CFG(r) (&(r)->config)
|
||||
#define REPO_FS(r) ((r)->fs)
|
||||
#define REPO_META(r) ((r)->pMeta)
|
||||
#define REPO_TFS(r) ((r)->pTfs)
|
||||
#define IS_REPO_LOCKED(r) ((r)->repoLocked)
|
||||
#define REPO_TSMA_NUM(r) ((r)->smaEnvs.nTSma)
|
||||
#define REPO_RSMA_NUM(r) ((r)->smaEnvs.nRSma)
|
||||
#define REPO_TSMA_ENV(r) ((r)->smaEnvs.pTSmaEnv)
|
||||
#define REPO_RSMA_ENV(r) ((r)->smaEnvs.pRSmaEnv)
|
||||
|
||||
int tsdbLockRepo(STsdb *pTsdb);
|
||||
int tsdbUnlockRepo(STsdb *pTsdb);
|
||||
|
||||
static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) {
|
||||
return pTable->pSchema;
|
||||
}
|
||||
|
||||
// tsdbLog
|
||||
extern int32_t tsdbDebugFlag;
|
||||
|
||||
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
|
||||
// tsdbMemTable.h
|
||||
typedef struct {
|
||||
int rowsInserted;
|
||||
int rowsUpdated;
|
||||
int rowsDeleteSucceed;
|
||||
int rowsDeleteFailed;
|
||||
int nOperations;
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SMergeInfo;
|
||||
|
||||
static void * taosTMalloc(size_t size);
|
||||
static void * taosTCalloc(size_t nmemb, size_t size);
|
||||
static void * taosTRealloc(void *ptr, size_t size);
|
||||
static void * taosTZfree(void *ptr);
|
||||
static size_t taosTSizeof(void *ptr);
|
||||
static void taosTMemset(void *ptr, int c);
|
||||
|
||||
STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
|
||||
void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
|
||||
int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
|
||||
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
|
||||
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
|
||||
|
||||
static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
|
||||
if (pIter == NULL) return NULL;
|
||||
|
||||
SSkipListNode *node = tSkipListIterGet(pIter);
|
||||
if (node == NULL) return NULL;
|
||||
|
||||
return (STSRow *)SL_GET_NODE_DATA(node);
|
||||
}
|
||||
|
||||
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
|
||||
STSRow *row = tsdbNextIterRow(pIter);
|
||||
if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
|
||||
|
||||
return TD_ROW_KEY(row);
|
||||
}
|
||||
|
||||
// tsdbOptions
|
||||
extern const STsdbCfg defautlTsdbOptions;
|
||||
|
||||
int tsdbValidateOptions(const STsdbCfg *);
|
||||
void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
|
||||
|
||||
// tsdbReadImpl
|
||||
typedef struct SReadH SReadH;
|
||||
|
||||
typedef struct {
|
||||
uint32_t len;
|
||||
uint32_t offset;
|
||||
uint32_t hasLast : 2;
|
||||
uint32_t numOfBlocks : 30;
|
||||
uint64_t uid;
|
||||
TSKEY maxKey;
|
||||
} SBlockIdx;
|
||||
|
||||
#ifdef TD_REFACTOR_3
|
||||
typedef struct {
|
||||
int64_t last : 1;
|
||||
int64_t offset : 63;
|
||||
int32_t algorithm : 8;
|
||||
int32_t numOfRows : 24;
|
||||
int32_t len;
|
||||
int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
|
||||
int16_t numOfSubBlocks;
|
||||
int16_t numOfCols; // not including timestamp column
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SBlock;
|
||||
|
||||
#else
|
||||
|
||||
typedef enum {
|
||||
TSDB_SBLK_VER_0 = 0,
|
||||
TSDB_SBLK_VER_MAX,
|
||||
} ESBlockVer;
|
||||
|
||||
#define SBlockVerLatest TSDB_SBLK_VER_0
|
||||
|
||||
typedef struct {
|
||||
uint8_t last : 1;
|
||||
uint8_t blkVer : 7;
|
||||
uint8_t numOfSubBlocks;
|
||||
col_id_t numOfCols; // not including timestamp column
|
||||
uint32_t len; // data block length
|
||||
uint32_t keyLen : 20; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
|
||||
uint32_t algorithm : 4;
|
||||
uint32_t reserve : 8;
|
||||
col_id_t numOfBSma;
|
||||
uint16_t numOfRows;
|
||||
int64_t offset;
|
||||
uint64_t aggrStat : 1;
|
||||
uint64_t aggrOffset : 63;
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SBlockV0;
|
||||
|
||||
#define SBlock SBlockV0 // latest SBlock definition
|
||||
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int32_t delimiter; // For recovery usage
|
||||
int32_t tid;
|
||||
uint64_t uid;
|
||||
SBlock blocks[];
|
||||
} SBlockInfo;
|
||||
|
||||
#ifdef TD_REFACTOR_3
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
|
||||
uint16_t reserve : 15;
|
||||
int32_t len;
|
||||
uint32_t type : 8;
|
||||
uint32_t offset : 24;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
uint8_t offsetH;
|
||||
char padding[1];
|
||||
} SBlockCol;
|
||||
#else
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
uint16_t type : 6;
|
||||
uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input)
|
||||
uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
|
||||
uint32_t len : 31; // data length + bitmap length
|
||||
uint32_t offset;
|
||||
} SBlockColV0;
|
||||
|
||||
#define SBlockCol SBlockColV0 // latest SBlockCol definition
|
||||
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
} SAggrBlkColV0;
|
||||
|
||||
#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition
|
||||
|
||||
#endif
|
||||
|
||||
// Code here just for back-ward compatibility
|
||||
static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
|
||||
#ifdef TD_REFACTOR_3
|
||||
pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
|
||||
pBlockCol->offsetH = (uint8_t)(offset >> 24);
|
||||
#else
|
||||
pBlockCol->offset = offset;
|
||||
#endif
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
|
||||
#ifdef TD_REFACTOR_3
|
||||
uint32_t offset1 = pBlockCol->offset;
|
||||
uint32_t offset2 = pBlockCol->offsetH;
|
||||
return (offset1 | (offset2 << 24));
|
||||
#else
|
||||
return pBlockCol->offset;
|
||||
#endif
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int32_t delimiter; // For recovery usage
|
||||
int32_t numOfCols; // For recovery usage
|
||||
uint64_t uid; // For recovery usage
|
||||
SBlockCol cols[];
|
||||
} SBlockData;
|
||||
|
||||
typedef void SAggrBlkData; // SBlockCol cols[];
|
||||
|
||||
struct SReadH {
|
||||
STsdb *pRepo;
|
||||
SDFileSet rSet; // FSET to read
|
||||
SArray *aBlkIdx; // SBlockIdx array
|
||||
STable *pTable; // table to read
|
||||
SBlockIdx *pBlkIdx; // current reading table SBlockIdx
|
||||
int cidx;
|
||||
SBlockInfo *pBlkInfo;
|
||||
SBlockData *pBlkData; // Block info
|
||||
SAggrBlkData *pAggrBlkData; // Aggregate Block info
|
||||
SDataCols *pDCols[2];
|
||||
void *pBuf; // buffer
|
||||
void *pCBuf; // compression buffer
|
||||
void *pExBuf; // extra buffer
|
||||
};
|
||||
|
||||
#define TSDB_READ_REPO(rh) ((rh)->pRepo)
|
||||
#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
|
||||
#define TSDB_READ_FSET(rh) (&((rh)->rSet))
|
||||
#define TSDB_READ_TABLE(rh) ((rh)->pTable)
|
||||
#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
|
||||
#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
|
||||
#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
|
||||
#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
|
||||
#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
|
||||
#define TSDB_READ_BUF(rh) ((rh)->pBuf)
|
||||
#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
|
||||
#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf)
|
||||
|
||||
#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
|
||||
(sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
|
||||
|
||||
static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
|
||||
switch (blkVer) {
|
||||
case TSDB_SBLK_VER_0:
|
||||
default:
|
||||
return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
|
||||
|
||||
static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
|
||||
switch (blkVer) {
|
||||
case TSDB_SBLK_VER_0:
|
||||
default:
|
||||
return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
|
||||
}
|
||||
}
|
||||
|
||||
int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
|
||||
void tsdbDestroyReadH(SReadH *pReadh);
|
||||
int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
|
||||
void tsdbCloseAndUnsetFSet(SReadH *pReadh);
|
||||
int tsdbLoadBlockIdx(SReadH *pReadh);
|
||||
int tsdbSetReadTable(SReadH *pReadh, STable *pTable);
|
||||
int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
|
||||
int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
|
||||
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
|
||||
int numOfColsIds);
|
||||
int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
|
||||
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
|
||||
void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
|
||||
void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
|
||||
|
||||
static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
|
||||
void *pBuf = *ppBuf;
|
||||
size_t tsize = taosTSizeof(pBuf);
|
||||
|
||||
if (tsize < size) {
|
||||
if (tsize == 0) tsize = 1024;
|
||||
|
||||
while (tsize < size) {
|
||||
tsize *= 2;
|
||||
}
|
||||
|
||||
*ppBuf = taosTRealloc(pBuf, tsize);
|
||||
if (*ppBuf == NULL) {
|
||||
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// tsdbMemory
|
||||
static FORCE_INLINE void *taosTMalloc(size_t size) {
|
||||
if (size <= 0) return NULL;
|
||||
|
||||
void *ret = taosMemoryMalloc(size + sizeof(size_t));
|
||||
if (ret == NULL) return NULL;
|
||||
|
||||
*(size_t *)ret = size;
|
||||
|
||||
return (void *)((char *)ret + sizeof(size_t));
|
||||
}
|
||||
|
||||
static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
|
||||
size_t tsize = nmemb * size;
|
||||
void * ret = taosTMalloc(tsize);
|
||||
if (ret == NULL) return NULL;
|
||||
|
||||
taosTMemset(ret, 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
|
||||
|
||||
static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
|
||||
|
||||
static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
|
||||
if (ptr == NULL) return taosTMalloc(size);
|
||||
|
||||
if (size <= taosTSizeof(ptr)) return ptr;
|
||||
|
||||
void * tptr = (void *)((char *)ptr - sizeof(size_t));
|
||||
size_t tsize = size + sizeof(size_t);
|
||||
void* tptr1 = taosMemoryRealloc(tptr, tsize);
|
||||
if (tptr1 == NULL) return NULL;
|
||||
tptr = tptr1;
|
||||
|
||||
*(size_t *)tptr = size;
|
||||
|
||||
return (void *)((char *)tptr + sizeof(size_t));
|
||||
}
|
||||
|
||||
static FORCE_INLINE void* taosTZfree(void* ptr) {
|
||||
if (ptr) {
|
||||
taosMemoryFree((void*)((char*)ptr - sizeof(size_t)));
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// tsdbCommit
|
||||
|
||||
typedef struct {
|
||||
uint64_t uid;
|
||||
int64_t offset;
|
||||
int64_t size;
|
||||
} SKVRecord;
|
||||
|
||||
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
|
||||
|
||||
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
|
||||
if (key < 0) {
|
||||
return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
|
||||
} else {
|
||||
return (int)((key / tsTickPerDay[precision] / days));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
|
||||
if (fid >= pRtn->maxFid) {
|
||||
return 0;
|
||||
} else if (fid >= pRtn->midFid) {
|
||||
return 1;
|
||||
} else if (fid >= pRtn->minFid) {
|
||||
return 2;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
// tsdbDBDef
|
||||
// typedef struct SDBFile SDBFile;
|
||||
// typedef DB_ENV* TDBEnv;
|
||||
|
||||
// struct SDBFile {
|
||||
// int32_t fid;
|
||||
// DB* pDB;
|
||||
// char* path;
|
||||
// };
|
||||
|
||||
// int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
|
||||
// void tsdbCloseDBF(SDBFile* pDBF);
|
||||
// int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path);
|
||||
// void tsdbCloseBDBEnv(DB_ENV* pEnv);
|
||||
// int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize);
|
||||
// void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize);
|
||||
|
||||
// tsdbFile
|
||||
#define TSDB_FILE_HEAD_SIZE 512
|
||||
#define TSDB_FILE_DELIMITER 0xF00AFA0F
|
||||
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
|
||||
#define TSDB_IVLD_FID INT_MIN
|
||||
#define TSDB_FILE_STATE_OK 0
|
||||
#define TSDB_FILE_STATE_BAD 1
|
||||
|
||||
#define TSDB_FILE_INFO(tf) (&((tf)->info))
|
||||
#define TSDB_FILE_F(tf) (&((tf)->f))
|
||||
#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
|
||||
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
|
||||
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
|
||||
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
|
||||
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
|
||||
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
|
||||
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
|
||||
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
|
||||
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
|
||||
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
|
||||
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
|
||||
#define TSDB_FILE_STATE(tf) ((tf)->state)
|
||||
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
|
||||
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
|
||||
#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
|
||||
|
||||
typedef int32_t TSDB_FILE_T;
|
||||
typedef enum {
|
||||
TSDB_FS_VER_0 = 0,
|
||||
TSDB_FS_VER_MAX,
|
||||
} ETsdbFsVer;
|
||||
|
||||
#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile
|
||||
#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file
|
||||
|
||||
static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile
|
||||
switch (fType) {
|
||||
case TSDB_FILE_HEAD: // .head
|
||||
case TSDB_FILE_DATA: // .data
|
||||
case TSDB_FILE_LAST: // .last
|
||||
case TSDB_FILE_SMAD: // .smad(Block-wise SMA)
|
||||
case TSDB_FILE_SMAL: // .smal(Block-wise SMA)
|
||||
default:
|
||||
return TSDB_LATEST_FVER;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
|
||||
void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
|
||||
int tsdbEncodeSDFile(void** buf, SDFile* pDFile);
|
||||
void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
|
||||
int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType);
|
||||
int tsdbUpdateDFileHeader(SDFile* pDFile);
|
||||
int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
|
||||
int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
|
||||
|
||||
static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
|
||||
|
||||
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
|
||||
ASSERT(!TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
|
||||
if (pDFile->pFile == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
|
||||
if (TSDB_FILE_OPENED(pDFile)) {
|
||||
taosCloseFile(&pDFile->pFile);
|
||||
TSDB_FILE_SET_CLOSED(pDFile);
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
|
||||
// ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
|
||||
if (loffset < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return loffset;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
|
||||
if (nwrite < nbyte) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nwrite;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
|
||||
pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t toffset;
|
||||
|
||||
if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ASSERT(pDFile->info.size == toffset);
|
||||
|
||||
if (offset) {
|
||||
*offset = toffset;
|
||||
}
|
||||
|
||||
if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
pDFile->info.size += nbyte;
|
||||
|
||||
return (int)nbyte;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
|
||||
|
||||
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
|
||||
if (nread < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nread;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
|
||||
if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
|
||||
return 0;
|
||||
}
|
||||
|
||||
// =============== SDFileSet
|
||||
|
||||
typedef struct {
|
||||
int fid;
|
||||
int8_t state;
|
||||
uint8_t ver;
|
||||
uint16_t reserve;
|
||||
#if 0
|
||||
SDFInfo info;
|
||||
#endif
|
||||
STfsFile f;
|
||||
TdFilePtr pFile;
|
||||
|
||||
} SSFile; // files split by days with fid
|
||||
|
||||
#define TSDB_LATEST_FSET_VER 0
|
||||
|
||||
#define TSDB_FSET_FID(s) ((s)->fid)
|
||||
#define TSDB_FSET_STATE(s) ((s)->state)
|
||||
#define TSDB_FSET_VER(s) ((s)->ver)
|
||||
#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
|
||||
#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
|
||||
#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
|
||||
#define TSDB_FSET_SET_CLOSED(s) \
|
||||
do { \
|
||||
for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
|
||||
TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \
|
||||
} \
|
||||
} while (0);
|
||||
#define TSDB_FSET_FSYNC(s) \
|
||||
do { \
|
||||
for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
|
||||
TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver);
|
||||
void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
|
||||
int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
|
||||
void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet);
|
||||
int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
|
||||
void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
|
||||
int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
|
||||
int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader);
|
||||
int tsdbUpdateDFileSetHeader(SDFileSet* pSet);
|
||||
int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
|
||||
|
||||
static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
|
||||
tsdbCloseDFileSet(pSet);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
(void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
|
||||
tsdbRemoveDFileSet(pDest);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
|
||||
*minKey = fid * days * tsTickPerDay[precision];
|
||||
*maxKey = *minKey + days * tsTickPerDay[precision] - 1;
|
||||
}
|
||||
|
||||
static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// tsdbFS
|
||||
// ================== TSDB global config
|
||||
extern bool tsdbForceKeepFile;
|
||||
|
||||
// ================== CURRENT file header info
|
||||
typedef struct {
|
||||
uint32_t version; // Current file system version (relating to code)
|
||||
uint32_t len; // Encode content length (including checksum)
|
||||
} SFSHeader;
|
||||
|
||||
// ================== TSDB File System Meta
|
||||
|
||||
/**
|
||||
* @brief Directory structure of .tsma data files.
|
||||
*
|
||||
* /vnode2/tsdb $ tree tsma/
|
||||
* tsma/
|
||||
* ├── v2f100.index_name_1
|
||||
* ├── v2f101.index_name_1
|
||||
* ├── v2f102.index_name_1
|
||||
* ├── v2f1900.index_name_3
|
||||
* ├── v2f1901.index_name_3
|
||||
* ├── v2f1902.index_name_3
|
||||
* ├── v2f200.index_name_2
|
||||
* ├── v2f201.index_name_2
|
||||
* └── v2f202.index_name_2
|
||||
*
|
||||
* 0 directories, 9 files
|
||||
*/
|
||||
|
||||
#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
|
||||
#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
|
||||
#define FS_IN_TXN(pfs) (pfs)->intxn
|
||||
#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
|
||||
#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
|
||||
|
||||
typedef struct {
|
||||
int direction;
|
||||
uint64_t version; // current FS version
|
||||
STsdbFS * pfs;
|
||||
int index; // used to position next fset when version the same
|
||||
int fid; // used to seek when version is changed
|
||||
SDFileSet *pSet;
|
||||
} SFSIter;
|
||||
|
||||
#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
|
||||
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
|
||||
|
||||
STsdbFS *tsdbNewFS(const STsdbCfg *pCfg);
|
||||
void * tsdbFreeFS(STsdbFS *pfs);
|
||||
int tsdbOpenFS(STsdb *pRepo);
|
||||
void tsdbCloseFS(STsdb *pRepo);
|
||||
void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
|
||||
int tsdbEndFSTxn(STsdb *pRepo);
|
||||
int tsdbEndFSTxnWithError(STsdbFS *pfs);
|
||||
void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
|
||||
// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
|
||||
int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
|
||||
|
||||
void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
|
||||
void tsdbFSIterSeek(SFSIter *pIter, int fid);
|
||||
SDFileSet *tsdbFSIterNext(SFSIter *pIter);
|
||||
int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
|
||||
|
||||
static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockRdlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockWrlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockUnlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// tsdbSma
|
||||
// #define TSDB_SMA_TEST // remove after test finished
|
||||
|
||||
|
||||
// struct SSmaEnv {
|
||||
// TdThreadRwlock lock;
|
||||
// SDiskID did;
|
||||
// TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
|
||||
// char *path; // relative path
|
||||
// SSmaStat *pStat;
|
||||
// };
|
||||
|
||||
// #define SMA_ENV_LOCK(env) ((env)->lock)
|
||||
// #define SMA_ENV_DID(env) ((env)->did)
|
||||
// #define SMA_ENV_ENV(env) ((env)->dbEnv)
|
||||
// #define SMA_ENV_PATH(env) ((env)->path)
|
||||
// #define SMA_ENV_STAT(env) ((env)->pStat)
|
||||
// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
|
||||
|
||||
|
||||
// void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
|
||||
// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
|
||||
// #if 0
|
||||
// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
|
||||
// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
|
||||
// #endif
|
||||
|
||||
// // internal func
|
||||
// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
|
||||
// int32_t len = 0;
|
||||
// len += taosEncodeFixedI64(pData, tsKey);
|
||||
// len += taosEncodeFixedI64(pData, groupId);
|
||||
// return len;
|
||||
// }
|
||||
|
||||
// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
|
||||
// int code = taosThreadRwlockRdlock(&(pEnv->lock));
|
||||
// if (code != 0) {
|
||||
// terrno = TAOS_SYSTEM_ERROR(code);
|
||||
// return -1;
|
||||
// }
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
|
||||
// int code = taosThreadRwlockWrlock(&(pEnv->lock));
|
||||
// if (code != 0) {
|
||||
// terrno = TAOS_SYSTEM_ERROR(code);
|
||||
// return -1;
|
||||
// }
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
|
||||
// int code = taosThreadRwlockUnlock(&(pEnv->lock));
|
||||
// if (code != 0) {
|
||||
// terrno = TAOS_SYSTEM_ERROR(code);
|
||||
// return -1;
|
||||
// }
|
||||
// return 0;
|
||||
// }
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#endif /*_TD_VNODE_TSDB_H_*/
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_COMMIT_H_
|
||||
#define _TD_TSDB_COMMIT_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int minFid;
|
||||
int midFid;
|
||||
int maxFid;
|
||||
TSKEY minKey;
|
||||
} SRtn;
|
||||
|
||||
typedef struct {
|
||||
uint64_t uid;
|
||||
int64_t offset;
|
||||
int64_t size;
|
||||
} SKVRecord;
|
||||
|
||||
void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
|
||||
|
||||
static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
|
||||
if (key < 0) {
|
||||
return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
|
||||
} else {
|
||||
return (int)((key / tsTickPerDay[precision] / days));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
|
||||
if (fid >= pRtn->maxFid) {
|
||||
return 0;
|
||||
} else if (fid >= pRtn->midFid) {
|
||||
return 1;
|
||||
} else if (fid >= pRtn->minFid) {
|
||||
return 2;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
|
||||
|
||||
int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
|
||||
void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
|
||||
void *tsdbCommitData(STsdbRepo *pRepo);
|
||||
int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
|
||||
int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, SBlockIdx *pIdx);
|
||||
int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
|
||||
int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock,
|
||||
bool isLast, bool isSuper, void **ppBuf, void **ppCBuf);
|
||||
int tsdbApplyRtn(STsdbRepo *pRepo);
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TD_TSDB_COMMIT_H_ */
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#if 0
|
||||
|
||||
#ifndef _TD_TSDB_COMPACT_H_
|
||||
#define _TD_TSDB_COMPACT_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void *tsdbCompactImpl(STsdbRepo *pRepo);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TD_TSDB_COMPACT_H_ */
|
||||
|
||||
#endif
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_DB_DEF_H_
|
||||
#define _TD_TSDB_DB_DEF_H_
|
||||
|
||||
#include "db.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct SDBFile SDBFile;
|
||||
typedef DB_ENV* TDBEnv;
|
||||
|
||||
struct SDBFile {
|
||||
int32_t fid;
|
||||
DB* pDB;
|
||||
char* path;
|
||||
};
|
||||
|
||||
int32_t tsdbOpenDBF(TDBEnv pEnv, SDBFile* pDBF);
|
||||
void tsdbCloseDBF(SDBFile* pDBF);
|
||||
int32_t tsdbOpenBDBEnv(DB_ENV** ppEnv, const char* path);
|
||||
void tsdbCloseBDBEnv(DB_ENV* pEnv);
|
||||
int32_t tsdbSaveSmaToDB(SDBFile* pDBF, void* key, uint32_t keySize, void* data, uint32_t dataSize);
|
||||
void* tsdbGetSmaDataByKey(SDBFile* pDBF, void* key, uint32_t keySize, uint32_t* valueSize);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_DB_DEF_H_*/
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_DEF_H_
|
||||
#define _TD_TSDB_DEF_H_
|
||||
|
||||
#include "tsdbDBDef.h"
|
||||
#include "tmallocator.h"
|
||||
#include "meta.h"
|
||||
#include "tcompression.h"
|
||||
#include "tglobal.h"
|
||||
#include "thash.h"
|
||||
#include "tlist.h"
|
||||
#include "tmsg.h"
|
||||
#include "tskiplist.h"
|
||||
#include "ttime.h"
|
||||
|
||||
#include "tsdb.h"
|
||||
#include "tsdbCommit.h"
|
||||
#include "tsdbFS.h"
|
||||
#include "tsdbFile.h"
|
||||
#include "tsdbLog.h"
|
||||
#include "tsdbMemTable.h"
|
||||
#include "tsdbMemory.h"
|
||||
#include "tsdbOptions.h"
|
||||
#include "tsdbReadImpl.h"
|
||||
#include "tsdbSma.h"
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct STsdb {
|
||||
int32_t vgId;
|
||||
bool repoLocked;
|
||||
TdThreadMutex mutex;
|
||||
char * path;
|
||||
STsdbCfg config;
|
||||
STsdbMemTable * mem;
|
||||
STsdbMemTable * imem;
|
||||
SRtn rtn;
|
||||
SMemAllocatorFactory *pmaf;
|
||||
STsdbFS * fs;
|
||||
SMeta * pMeta;
|
||||
STfs * pTfs;
|
||||
SSmaEnv * pTSmaEnv;
|
||||
SSmaEnv * pRSmaEnv;
|
||||
};
|
||||
|
||||
#define REPO_ID(r) ((r)->vgId)
|
||||
#define REPO_CFG(r) (&(r)->config)
|
||||
#define REPO_FS(r) (r)->fs
|
||||
#define REPO_META(r) (r)->pMeta
|
||||
#define REPO_TFS(r) (r)->pTfs
|
||||
#define IS_REPO_LOCKED(r) (r)->repoLocked
|
||||
#define REPO_SMA_ENV(r, t) ((TSDB_SMA_TYPE_ROLLUP == (t)) ? (r)->pRSmaEnv : (r)->pTSmaEnv)
|
||||
|
||||
int tsdbLockRepo(STsdb *pTsdb);
|
||||
int tsdbUnlockRepo(STsdb *pTsdb);
|
||||
|
||||
static FORCE_INLINE STSchema *tsdbGetTableSchemaImpl(STable *pTable, bool lock, bool copy, int32_t version) {
|
||||
return pTable->pSchema;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_DEF_H_*/
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_FS_H_
|
||||
#define _TD_TSDB_FS_H_
|
||||
|
||||
#include "tsdbFile.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// ================== TSDB global config
|
||||
extern bool tsdbForceKeepFile;
|
||||
|
||||
// ================== CURRENT file header info
|
||||
typedef struct {
|
||||
uint32_t version; // Current file system version (relating to code)
|
||||
uint32_t len; // Encode content length (including checksum)
|
||||
} SFSHeader;
|
||||
|
||||
// ================== TSDB File System Meta
|
||||
typedef struct {
|
||||
uint32_t version; // Commit version from 0 to increase
|
||||
int64_t totalPoints; // total points
|
||||
int64_t totalStorage; // Uncompressed total storage
|
||||
} STsdbFSMeta;
|
||||
|
||||
// ==================
|
||||
typedef struct {
|
||||
STsdbFSMeta meta; // FS meta
|
||||
SArray * df; // data file array
|
||||
SArray * sf; // sma data file array v2f1900.index_name_1
|
||||
} SFSStatus;
|
||||
|
||||
/**
|
||||
* @brief Directory structure of .tsma data files.
|
||||
*
|
||||
* /vnode2/tsdb $ tree tsma/
|
||||
* tsma/
|
||||
* ├── v2f100.index_name_1
|
||||
* ├── v2f101.index_name_1
|
||||
* ├── v2f102.index_name_1
|
||||
* ├── v2f1900.index_name_3
|
||||
* ├── v2f1901.index_name_3
|
||||
* ├── v2f1902.index_name_3
|
||||
* ├── v2f200.index_name_2
|
||||
* ├── v2f201.index_name_2
|
||||
* └── v2f202.index_name_2
|
||||
*
|
||||
* 0 directories, 9 files
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
TdThreadRwlock lock;
|
||||
|
||||
SFSStatus *cstatus; // current status
|
||||
SHashObj * metaCache; // meta cache
|
||||
SHashObj * metaCacheComp; // meta cache for compact
|
||||
bool intxn;
|
||||
SFSStatus *nstatus; // new status
|
||||
} STsdbFS;
|
||||
|
||||
#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
|
||||
#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
|
||||
#define FS_IN_TXN(pfs) (pfs)->intxn
|
||||
#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
|
||||
#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
|
||||
|
||||
typedef struct {
|
||||
int direction;
|
||||
uint64_t version; // current FS version
|
||||
STsdbFS * pfs;
|
||||
int index; // used to position next fset when version the same
|
||||
int fid; // used to seek when version is changed
|
||||
SDFileSet *pSet;
|
||||
} SFSIter;
|
||||
|
||||
#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
|
||||
#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
|
||||
|
||||
STsdbFS *tsdbNewFS(const STsdbCfg *pCfg);
|
||||
void * tsdbFreeFS(STsdbFS *pfs);
|
||||
int tsdbOpenFS(STsdb *pRepo);
|
||||
void tsdbCloseFS(STsdb *pRepo);
|
||||
void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd);
|
||||
int tsdbEndFSTxn(STsdb *pRepo);
|
||||
int tsdbEndFSTxnWithError(STsdbFS *pfs);
|
||||
void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
|
||||
// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
|
||||
int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
|
||||
|
||||
void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
|
||||
void tsdbFSIterSeek(SFSIter *pIter, int fid);
|
||||
SDFileSet *tsdbFSIterNext(SFSIter *pIter);
|
||||
int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta);
|
||||
|
||||
static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockRdlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockWrlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
|
||||
int code = taosThreadRwlockUnlock(&(pFs->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TD_TSDB_FS_H_ */
|
||||
|
|
@ -1,435 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TS_TSDB_FILE_H_
|
||||
#define _TS_TSDB_FILE_H_
|
||||
|
||||
#include "tchecksum.h"
|
||||
#include "tfs.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define TSDB_FILE_HEAD_SIZE 512
|
||||
#define TSDB_FILE_DELIMITER 0xF00AFA0F
|
||||
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
|
||||
#define TSDB_IVLD_FID INT_MIN
|
||||
#define TSDB_FILE_STATE_OK 0
|
||||
#define TSDB_FILE_STATE_BAD 1
|
||||
|
||||
#define TSDB_FILE_INFO(tf) (&((tf)->info))
|
||||
#define TSDB_FILE_F(tf) (&((tf)->f))
|
||||
#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
|
||||
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
|
||||
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
|
||||
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
|
||||
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
|
||||
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
|
||||
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
|
||||
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
|
||||
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
|
||||
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
|
||||
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
|
||||
#define TSDB_FILE_STATE(tf) ((tf)->state)
|
||||
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
|
||||
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
|
||||
#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD)
|
||||
|
||||
typedef enum {
|
||||
TSDB_FILE_HEAD = 0, // .head
|
||||
TSDB_FILE_DATA, // .data
|
||||
TSDB_FILE_LAST, // .last
|
||||
TSDB_FILE_SMAD, // .smad(Block-wise SMA)
|
||||
TSDB_FILE_SMAL, // .smal(Block-wise SMA)
|
||||
TSDB_FILE_MAX, //
|
||||
TSDB_FILE_META, // meta
|
||||
TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA
|
||||
TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
|
||||
} E_TSDB_FILE_T;
|
||||
|
||||
typedef int32_t TSDB_FILE_T;
|
||||
typedef enum {
|
||||
TSDB_FS_VER_0 = 0,
|
||||
TSDB_FS_VER_MAX,
|
||||
} ETsdbFsVer;
|
||||
|
||||
#define TSDB_LATEST_FVER TSDB_FS_VER_0 // latest version for DFile
|
||||
#define TSDB_LATEST_SFS_VER TSDB_FS_VER_0 // latest version for 'current' file
|
||||
|
||||
static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest version for DFile
|
||||
switch (fType) {
|
||||
case TSDB_FILE_HEAD: // .head
|
||||
case TSDB_FILE_DATA: // .data
|
||||
case TSDB_FILE_LAST: // .last
|
||||
case TSDB_FILE_SMAD: // .smad(Block-wise SMA)
|
||||
case TSDB_FILE_SMAL: // .smal(Block-wise SMA)
|
||||
default:
|
||||
return TSDB_LATEST_FVER;
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
// =============== SMFile
|
||||
typedef struct {
|
||||
int64_t size;
|
||||
int64_t tombSize;
|
||||
int64_t nRecords;
|
||||
int64_t nDels;
|
||||
uint32_t magic;
|
||||
} SMFInfo;
|
||||
|
||||
typedef struct {
|
||||
SMFInfo info;
|
||||
STfsFile f;
|
||||
int fd;
|
||||
uint8_t state;
|
||||
} SMFile;
|
||||
|
||||
void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
|
||||
void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile);
|
||||
int tsdbEncodeSMFile(void** buf, SMFile* pMFile);
|
||||
void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
|
||||
int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
|
||||
void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
|
||||
int tsdbApplyMFileChange(SMFile* from, SMFile* to);
|
||||
int tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
|
||||
int tsdbUpdateMFileHeader(SMFile* pMFile);
|
||||
int tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
|
||||
int tsdbScanAndTryFixMFile(STsdb* pRepo);
|
||||
int tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
|
||||
void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
|
||||
|
||||
static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
|
||||
|
||||
static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
|
||||
ASSERT(TSDB_FILE_CLOSED(pMFile));
|
||||
|
||||
pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
|
||||
if (pMFile->fd < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
|
||||
if (TSDB_FILE_OPENED(pMFile)) {
|
||||
close(pMFile->fd);
|
||||
TSDB_FILE_SET_CLOSED(pMFile);
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
|
||||
ASSERT(TSDB_FILE_OPENED(pMFile));
|
||||
|
||||
int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pMFile), offset, whence);
|
||||
if (loffset < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return loffset;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pMFile));
|
||||
|
||||
int64_t nwrite = taosWriteFile(pMFile->fd, buf, nbyte);
|
||||
if (nwrite < nbyte) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nwrite;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
|
||||
pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
|
||||
ASSERT(TSDB_FILE_OPENED(pMFile));
|
||||
|
||||
int64_t toffset;
|
||||
|
||||
if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ASSERT(pMFile->info.size == toffset);
|
||||
|
||||
if (offset) {
|
||||
*offset = toffset;
|
||||
}
|
||||
|
||||
if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
pMFile->info.size += nbyte;
|
||||
|
||||
return (int)nbyte;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
|
||||
|
||||
static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pMFile));
|
||||
|
||||
int64_t nread = taosReadFile(pMFile->fd, buf, nbyte);
|
||||
if (nread < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nread;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// =============== SDFile
|
||||
typedef struct {
|
||||
uint32_t magic;
|
||||
uint32_t fver;
|
||||
uint32_t len;
|
||||
uint32_t totalBlocks;
|
||||
uint32_t totalSubBlocks;
|
||||
uint32_t offset;
|
||||
uint64_t size;
|
||||
uint64_t tombSize;
|
||||
} SDFInfo;
|
||||
|
||||
typedef struct {
|
||||
SDFInfo info;
|
||||
STfsFile f;
|
||||
TdFilePtr pFile;
|
||||
uint8_t state;
|
||||
} SDFile;
|
||||
|
||||
void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype);
|
||||
void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
|
||||
int tsdbEncodeSDFile(void** buf, SDFile* pDFile);
|
||||
void* tsdbDecodeSDFile(STsdb *pRepo, void* buf, SDFile* pDFile);
|
||||
int tsdbCreateDFile(STsdb *pRepo, SDFile* pDFile, bool updateHeader, TSDB_FILE_T fType);
|
||||
int tsdbUpdateDFileHeader(SDFile* pDFile);
|
||||
int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
|
||||
int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
|
||||
|
||||
static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
|
||||
|
||||
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
|
||||
ASSERT(!TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
|
||||
if (pDFile->pFile == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
|
||||
if (TSDB_FILE_OPENED(pDFile)) {
|
||||
taosCloseFile(&pDFile->pFile);
|
||||
TSDB_FILE_SET_CLOSED(pDFile);
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
|
||||
// ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
|
||||
if (loffset < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return loffset;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
|
||||
if (nwrite < nbyte) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nwrite;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
|
||||
pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t toffset;
|
||||
|
||||
if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
ASSERT(pDFile->info.size == toffset);
|
||||
|
||||
if (offset) {
|
||||
*offset = toffset;
|
||||
}
|
||||
|
||||
if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
pDFile->info.size += nbyte;
|
||||
|
||||
return (int)nbyte;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); }
|
||||
|
||||
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
|
||||
ASSERT(TSDB_FILE_OPENED(pDFile));
|
||||
|
||||
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
|
||||
if (nread < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return nread;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
|
||||
if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
|
||||
tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
|
||||
return 0;
|
||||
}
|
||||
|
||||
// =============== SDFileSet
|
||||
typedef struct {
|
||||
int fid;
|
||||
int8_t state; // -128~127
|
||||
uint8_t ver; // 0~255, DFileSet version
|
||||
uint16_t reserve;
|
||||
SDFile files[TSDB_FILE_MAX];
|
||||
} SDFileSet;
|
||||
|
||||
typedef struct {
|
||||
int fid;
|
||||
int8_t state;
|
||||
uint8_t ver;
|
||||
uint16_t reserve;
|
||||
#if 0
|
||||
SDFInfo info;
|
||||
#endif
|
||||
STfsFile f;
|
||||
TdFilePtr pFile;
|
||||
|
||||
} SSFile; // files split by days with fid
|
||||
|
||||
#define TSDB_LATEST_FSET_VER 0
|
||||
|
||||
#define TSDB_FSET_FID(s) ((s)->fid)
|
||||
#define TSDB_FSET_STATE(s) ((s)->state)
|
||||
#define TSDB_FSET_VER(s) ((s)->ver)
|
||||
#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
|
||||
#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
|
||||
#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
|
||||
#define TSDB_FSET_SET_CLOSED(s) \
|
||||
do { \
|
||||
for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
|
||||
TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \
|
||||
} \
|
||||
} while (0);
|
||||
#define TSDB_FSET_FSYNC(s) \
|
||||
do { \
|
||||
for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
|
||||
TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
void tsdbInitDFileSet(STsdb *pRepo, SDFileSet* pSet, SDiskID did, int fid, uint32_t ver);
|
||||
void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
|
||||
int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
|
||||
void* tsdbDecodeDFileSet(STsdb *pRepo, void* buf, SDFileSet* pSet);
|
||||
int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
|
||||
void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
|
||||
int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
|
||||
int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet* pSet, bool updateHeader);
|
||||
int tsdbUpdateDFileSetHeader(SDFileSet* pSet);
|
||||
int tsdbScanAndTryFixDFileSet(STsdb* pRepo, SDFileSet* pSet);
|
||||
|
||||
static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
|
||||
tsdbCloseDFileSet(pSet);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
(void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
|
||||
tsdbRemoveDFileSet(pDest);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
|
||||
*minKey = fid * days * tsTickPerDay[precision];
|
||||
*maxKey = *minKey + days * tsTickPerDay[precision] - 1;
|
||||
}
|
||||
|
||||
static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
|
||||
for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
|
||||
if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TS_TSDB_FILE_H_ */
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_LOG_H_
|
||||
#define _TD_TSDB_LOG_H_
|
||||
|
||||
#include "tlog.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int32_t tsdbDebugFlag;
|
||||
|
||||
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
|
||||
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TD_TSDB_LOG_H_ */
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_MEM_TABLE_H_
|
||||
#define _TD_TSDB_MEM_TABLE_H_
|
||||
|
||||
#include "tsdb.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int rowsInserted;
|
||||
int rowsUpdated;
|
||||
int rowsDeleteSucceed;
|
||||
int rowsDeleteFailed;
|
||||
int nOperations;
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SMergeInfo;
|
||||
|
||||
typedef struct STbData {
|
||||
tb_uid_t uid;
|
||||
TSKEY keyMin;
|
||||
TSKEY keyMax;
|
||||
int64_t nrows;
|
||||
SSkipList *pData;
|
||||
} STbData;
|
||||
|
||||
typedef struct STsdbMemTable {
|
||||
T_REF_DECLARE()
|
||||
SRWLatch latch;
|
||||
TSKEY keyMin;
|
||||
TSKEY keyMax;
|
||||
uint64_t nRow;
|
||||
SMemAllocator *pMA;
|
||||
// Container
|
||||
SSkipList *pSlIdx; // SSkiplist<STbData>
|
||||
SHashObj * pHashIdx;
|
||||
} STsdbMemTable;
|
||||
|
||||
STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb);
|
||||
void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable);
|
||||
int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp);
|
||||
int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
|
||||
TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
|
||||
|
||||
static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
|
||||
if (pIter == NULL) return NULL;
|
||||
|
||||
SSkipListNode *node = tSkipListIterGet(pIter);
|
||||
if (node == NULL) return NULL;
|
||||
|
||||
return (STSRow *)SL_GET_NODE_DATA(node);
|
||||
}
|
||||
|
||||
static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
|
||||
STSRow *row = tsdbNextIterRow(pIter);
|
||||
if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
|
||||
|
||||
return TD_ROW_KEY(row);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_MEM_TABLE_H_*/
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_MEMORY_H_
|
||||
#define _TD_TSDB_MEMORY_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static void * taosTMalloc(size_t size);
|
||||
static void * taosTCalloc(size_t nmemb, size_t size);
|
||||
static void * taosTRealloc(void *ptr, size_t size);
|
||||
static void * taosTZfree(void *ptr);
|
||||
static size_t taosTSizeof(void *ptr);
|
||||
static void taosTMemset(void *ptr, int c);
|
||||
|
||||
static FORCE_INLINE void *taosTMalloc(size_t size) {
|
||||
if (size <= 0) return NULL;
|
||||
|
||||
void *ret = taosMemoryMalloc(size + sizeof(size_t));
|
||||
if (ret == NULL) return NULL;
|
||||
|
||||
*(size_t *)ret = size;
|
||||
|
||||
return (void *)((char *)ret + sizeof(size_t));
|
||||
}
|
||||
|
||||
static FORCE_INLINE void *taosTCalloc(size_t nmemb, size_t size) {
|
||||
size_t tsize = nmemb * size;
|
||||
void * ret = taosTMalloc(tsize);
|
||||
if (ret == NULL) return NULL;
|
||||
|
||||
taosTMemset(ret, 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static FORCE_INLINE size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
|
||||
|
||||
static FORCE_INLINE void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); }
|
||||
|
||||
static FORCE_INLINE void * taosTRealloc(void *ptr, size_t size) {
|
||||
if (ptr == NULL) return taosTMalloc(size);
|
||||
|
||||
if (size <= taosTSizeof(ptr)) return ptr;
|
||||
|
||||
void * tptr = (void *)((char *)ptr - sizeof(size_t));
|
||||
size_t tsize = size + sizeof(size_t);
|
||||
void* tptr1 = taosMemoryRealloc(tptr, tsize);
|
||||
if (tptr1 == NULL) return NULL;
|
||||
tptr = tptr1;
|
||||
|
||||
*(size_t *)tptr = size;
|
||||
|
||||
return (void *)((char *)tptr + sizeof(size_t));
|
||||
}
|
||||
|
||||
static FORCE_INLINE void* taosTZfree(void* ptr) {
|
||||
if (ptr) {
|
||||
taosMemoryFree((void*)((char*)ptr - sizeof(size_t)));
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _TD_TSDB_MEMORY_H_ */
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_OPTIONS_H_
|
||||
#define _TD_TSDB_OPTIONS_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern const STsdbCfg defautlTsdbOptions;
|
||||
|
||||
int tsdbValidateOptions(const STsdbCfg *);
|
||||
void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_OPTIONS_H_*/
|
||||
|
|
@ -1,256 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_READ_IMPL_H_
|
||||
#define _TD_TSDB_READ_IMPL_H_
|
||||
|
||||
#include "os.h"
|
||||
#include "tcommon.h"
|
||||
#include "tfs.h"
|
||||
#include "tsdb.h"
|
||||
#include "tsdbFile.h"
|
||||
#include "tsdbMemory.h"
|
||||
#include "tskiplist.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct SReadH SReadH;
|
||||
|
||||
typedef struct {
|
||||
uint32_t len;
|
||||
uint32_t offset;
|
||||
uint32_t hasLast : 2;
|
||||
uint32_t numOfBlocks : 30;
|
||||
uint64_t uid;
|
||||
TSKEY maxKey;
|
||||
} SBlockIdx;
|
||||
|
||||
#ifdef TD_REFACTOR_3
|
||||
typedef struct {
|
||||
int64_t last : 1;
|
||||
int64_t offset : 63;
|
||||
int32_t algorithm : 8;
|
||||
int32_t numOfRows : 24;
|
||||
int32_t len;
|
||||
int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
|
||||
int16_t numOfSubBlocks;
|
||||
int16_t numOfCols; // not including timestamp column
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SBlock;
|
||||
|
||||
#else
|
||||
|
||||
typedef enum {
|
||||
TSDB_SBLK_VER_0 = 0,
|
||||
TSDB_SBLK_VER_MAX,
|
||||
} ESBlockVer;
|
||||
|
||||
#define SBlockVerLatest TSDB_SBLK_VER_0
|
||||
|
||||
typedef struct {
|
||||
uint8_t last : 1;
|
||||
uint8_t blkVer : 7;
|
||||
uint8_t numOfSubBlocks;
|
||||
col_id_t numOfCols; // not including timestamp column
|
||||
uint32_t len; // data block length
|
||||
uint32_t keyLen : 20; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
|
||||
uint32_t algorithm : 4;
|
||||
uint32_t reserve : 8;
|
||||
col_id_t numOfBSma;
|
||||
uint16_t numOfRows;
|
||||
int64_t offset;
|
||||
uint64_t aggrStat : 1;
|
||||
uint64_t aggrOffset : 63;
|
||||
TSKEY keyFirst;
|
||||
TSKEY keyLast;
|
||||
} SBlockV0;
|
||||
|
||||
#define SBlock SBlockV0 // latest SBlock definition
|
||||
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int32_t delimiter; // For recovery usage
|
||||
int32_t tid;
|
||||
uint64_t uid;
|
||||
SBlock blocks[];
|
||||
} SBlockInfo;
|
||||
|
||||
#ifdef TD_REFACTOR_3
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
|
||||
uint16_t reserve : 15;
|
||||
int32_t len;
|
||||
uint32_t type : 8;
|
||||
uint32_t offset : 24;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
uint8_t offsetH;
|
||||
char padding[1];
|
||||
} SBlockCol;
|
||||
#else
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
uint16_t type : 6;
|
||||
uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input)
|
||||
uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
|
||||
uint32_t len : 31; // data length + bitmap length
|
||||
uint32_t offset;
|
||||
} SBlockColV0;
|
||||
|
||||
#define SBlockCol SBlockColV0 // latest SBlockCol definition
|
||||
|
||||
typedef struct {
|
||||
int16_t colId;
|
||||
int16_t maxIndex;
|
||||
int16_t minIndex;
|
||||
int16_t numOfNull;
|
||||
int64_t sum;
|
||||
int64_t max;
|
||||
int64_t min;
|
||||
} SAggrBlkColV0;
|
||||
|
||||
#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition
|
||||
|
||||
#endif
|
||||
|
||||
// Code here just for back-ward compatibility
|
||||
static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) {
|
||||
#ifdef TD_REFACTOR_3
|
||||
pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1);
|
||||
pBlockCol->offsetH = (uint8_t)(offset >> 24);
|
||||
#else
|
||||
pBlockCol->offset = offset;
|
||||
#endif
|
||||
}
|
||||
|
||||
static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) {
|
||||
#ifdef TD_REFACTOR_3
|
||||
uint32_t offset1 = pBlockCol->offset;
|
||||
uint32_t offset2 = pBlockCol->offsetH;
|
||||
return (offset1 | (offset2 << 24));
|
||||
#else
|
||||
return pBlockCol->offset;
|
||||
#endif
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int32_t delimiter; // For recovery usage
|
||||
int32_t numOfCols; // For recovery usage
|
||||
uint64_t uid; // For recovery usage
|
||||
SBlockCol cols[];
|
||||
} SBlockData;
|
||||
|
||||
typedef void SAggrBlkData; // SBlockCol cols[];
|
||||
|
||||
struct SReadH {
|
||||
STsdb *pRepo;
|
||||
SDFileSet rSet; // FSET to read
|
||||
SArray *aBlkIdx; // SBlockIdx array
|
||||
STable *pTable; // table to read
|
||||
SBlockIdx *pBlkIdx; // current reading table SBlockIdx
|
||||
int cidx;
|
||||
SBlockInfo *pBlkInfo;
|
||||
SBlockData *pBlkData; // Block info
|
||||
SAggrBlkData *pAggrBlkData; // Aggregate Block info
|
||||
SDataCols *pDCols[2];
|
||||
void *pBuf; // buffer
|
||||
void *pCBuf; // compression buffer
|
||||
void *pExBuf; // extra buffer
|
||||
};
|
||||
|
||||
#define TSDB_READ_REPO(rh) ((rh)->pRepo)
|
||||
#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
|
||||
#define TSDB_READ_FSET(rh) (&((rh)->rSet))
|
||||
#define TSDB_READ_TABLE(rh) ((rh)->pTable)
|
||||
#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
|
||||
#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
|
||||
#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
|
||||
#define TSDB_READ_SMAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAD)
|
||||
#define TSDB_READ_SMAL_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_SMAL)
|
||||
#define TSDB_READ_BUF(rh) ((rh)->pBuf)
|
||||
#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
|
||||
#define TSDB_READ_EXBUF(rh) ((rh)->pExBuf)
|
||||
|
||||
#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \
|
||||
(sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM))
|
||||
|
||||
static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) {
|
||||
switch (blkVer) {
|
||||
case TSDB_SBLK_VER_0:
|
||||
default:
|
||||
return TSDB_BLOCK_STATIS_SIZE(nCols, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM))
|
||||
|
||||
static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) {
|
||||
switch (blkVer) {
|
||||
case TSDB_SBLK_VER_0:
|
||||
default:
|
||||
return TSDB_BLOCK_AGGR_SIZE(nCols, 0);
|
||||
}
|
||||
}
|
||||
|
||||
int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo);
|
||||
void tsdbDestroyReadH(SReadH *pReadh);
|
||||
int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
|
||||
void tsdbCloseAndUnsetFSet(SReadH *pReadh);
|
||||
int tsdbLoadBlockIdx(SReadH *pReadh);
|
||||
int tsdbSetReadTable(SReadH *pReadh, STable *pTable);
|
||||
int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
|
||||
int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
|
||||
int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
|
||||
int numOfColsIds);
|
||||
int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
|
||||
int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
|
||||
void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
|
||||
void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
|
||||
|
||||
static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
|
||||
void *pBuf = *ppBuf;
|
||||
size_t tsize = taosTSizeof(pBuf);
|
||||
|
||||
if (tsize < size) {
|
||||
if (tsize == 0) tsize = 1024;
|
||||
|
||||
while (tsize < size) {
|
||||
tsize *= 2;
|
||||
}
|
||||
|
||||
*ppBuf = taosTRealloc(pBuf, tsize);
|
||||
if (*ppBuf == NULL) {
|
||||
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_TSDB_READ_IMPL_H_*/
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_TSDB_SMA_H_
|
||||
#define _TD_TSDB_SMA_H_
|
||||
|
||||
#define TSDB_SMA_TEST // remove after test finished
|
||||
|
||||
typedef struct SSmaStat SSmaStat;
|
||||
typedef struct SSmaEnv SSmaEnv;
|
||||
|
||||
struct SSmaEnv {
|
||||
TdThreadRwlock lock;
|
||||
SDiskID did;
|
||||
TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level?
|
||||
char *path; // relative path
|
||||
SSmaStat *pStat;
|
||||
};
|
||||
|
||||
#define SMA_ENV_LOCK(env) ((env)->lock)
|
||||
#define SMA_ENV_DID(env) ((env)->did)
|
||||
#define SMA_ENV_ENV(env) ((env)->dbEnv)
|
||||
#define SMA_ENV_PATH(env) ((env)->path)
|
||||
#define SMA_ENV_STAT(env) ((env)->pStat)
|
||||
#define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
|
||||
|
||||
void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
|
||||
void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
|
||||
#if 0
|
||||
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
|
||||
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
|
||||
#endif
|
||||
|
||||
// internal func
|
||||
static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
|
||||
int32_t len = 0;
|
||||
len += taosEncodeFixedI64(pData, tsKey);
|
||||
len += taosEncodeFixedI64(pData, groupId);
|
||||
return len;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
|
||||
int code = taosThreadRwlockRdlock(&(pEnv->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
|
||||
int code = taosThreadRwlockWrlock(&(pEnv->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
|
||||
int code = taosThreadRwlockUnlock(&(pEnv->lock));
|
||||
if (code != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(code);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* _TD_TSDB_SMA_H_ */
|
||||
|
|
@ -16,17 +16,23 @@
|
|||
#ifndef _TD_VNODE_DEF_H_
|
||||
#define _TD_VNODE_DEF_H_
|
||||
|
||||
#include "tmallocator.h"
|
||||
// #include "sync.h"
|
||||
#include "executor.h"
|
||||
#include "tchecksum.h"
|
||||
#include "tcoding.h"
|
||||
#include "tcompression.h"
|
||||
#include "tdatablock.h"
|
||||
#include "tfs.h"
|
||||
#include "tglobal.h"
|
||||
#include "tlist.h"
|
||||
#include "tlockfree.h"
|
||||
#include "tmacro.h"
|
||||
#include "tmallocator.h"
|
||||
#include "tskiplist.h"
|
||||
#include "ttime.h"
|
||||
#include "ttimer.h"
|
||||
#include "vnode.h"
|
||||
#include "vnodeQuery.h"
|
||||
#include "wal.h"
|
||||
#include "qworker.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -36,6 +42,7 @@ typedef struct STQ STQ;
|
|||
|
||||
typedef struct SVState SVState;
|
||||
typedef struct SVBufPool SVBufPool;
|
||||
typedef struct SQWorkerMgmt SQHandle;
|
||||
|
||||
typedef struct SVnodeTask {
|
||||
TD_DLIST_NODE(SVnodeTask);
|
||||
|
|
@ -96,6 +103,8 @@ struct SVnode {
|
|||
};
|
||||
|
||||
int vnodeScheduleTask(SVnodeTask* task);
|
||||
int vnodeQueryOpen(SVnode* pVnode);
|
||||
void vnodeQueryClose(SVnode* pVnode);
|
||||
|
||||
#define vFatal(...) \
|
||||
do { \
|
||||
|
|
@ -197,6 +206,7 @@ int tqCommit(STQ*);
|
|||
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
|
||||
int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
|
||||
int32_t tqProcessRebReq(STQ* pTq, char* msg);
|
||||
int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
|
||||
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
|
||||
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
|
||||
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
|
||||
|
|
@ -204,6 +214,12 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo
|
|||
// sma
|
||||
void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
|
||||
|
||||
#include "meta.h"
|
||||
|
||||
#include "tsdb.h"
|
||||
|
||||
#include "tq.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef _TD_VNODE_READ_H_
|
||||
#define _TD_VNODE_READ_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
#include "qworker.h"
|
||||
#include "vnode.h"
|
||||
|
||||
|
||||
typedef struct SQWorkerMgmt SQHandle;
|
||||
|
||||
int vnodeQueryOpen(SVnode *pVnode);
|
||||
void vnodeQueryClose(SVnode *pVnode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /*_TD_VNODE_READ_H_*/
|
||||
|
|
@ -912,7 +912,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
|
|||
pCur->uid = uid;
|
||||
// TODO: lock?
|
||||
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0);
|
||||
if (ret != 0) {
|
||||
if ((ret != 0) || (pCur->pCur == NULL)) {
|
||||
taosMemoryFree(pCur);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -996,32 +996,31 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
|
|||
}
|
||||
|
||||
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
|
||||
SArray * pUids = NULL;
|
||||
SArray *pUids = NULL;
|
||||
SMetaDB *pDB = pMeta->pDB;
|
||||
DBC * pCur = NULL;
|
||||
DBC *pCur = NULL;
|
||||
DBT pkey = {0}, pval = {0};
|
||||
uint32_t mode = isDup ? DB_NEXT_DUP : DB_NEXT_NODUP;
|
||||
int ret;
|
||||
|
||||
pUids = taosArrayInit(16, sizeof(tb_uid_t));
|
||||
|
||||
if (!pUids) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// TODO: lock?
|
||||
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
|
||||
if (ret != 0) {
|
||||
taosArrayDestroy(pUids);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *pBuf = NULL;
|
||||
|
||||
// TODO: lock?
|
||||
|
||||
while ((ret = pCur->get(pCur, &pkey, &pval, mode)) == 0) {
|
||||
if (!pUids) {
|
||||
pUids = taosArrayInit(16, sizeof(tb_uid_t));
|
||||
if (!pUids) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
taosArrayPush(pUids, pkey.data);
|
||||
}
|
||||
// TODO: lock?
|
||||
|
||||
if (pCur) {
|
||||
pCur->close(pCur);
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "meta.h"
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
struct SMetaCache {
|
||||
// TODO
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
const SMetaCfg defaultMetaOptions = {.lruSize = 0};
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
#ifdef USE_INVERTED_INDEX
|
||||
#include "index.h"
|
||||
#endif
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
struct SMetaIdx {
|
||||
#ifdef USE_INVERTED_INDEX
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
#include "tcoding.h"
|
||||
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
|
||||
static void metaFree(SMeta *pMeta);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
#include "tdbInt.h"
|
||||
typedef struct SPoolMem {
|
||||
|
|
@ -603,7 +603,7 @@ void metaCloseSmaCurosr(SMSmaCursor *pCur) {
|
|||
|
||||
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
|
||||
// TODO
|
||||
ASSERT(0);
|
||||
// ASSERT(0); // comment this line to pass CI
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) {
|
||||
// Validate the tbOptions
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
#include "tcoding.h"
|
||||
|
||||
int metaValidateTbCfg(SMeta *pMeta, const STbCfg *pTbOptions) {
|
||||
|
|
|
|||
|
|
@ -13,8 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "meta.h"
|
||||
#include "metaDef.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
int metaOpenUidGnrt(SMeta *pMeta) {
|
||||
// Init a generator
|
||||
|
|
|
|||
|
|
@ -15,9 +15,8 @@
|
|||
|
||||
#include "tcompare.h"
|
||||
#include "tdatablock.h"
|
||||
#include "tqInt.h"
|
||||
#include "tqMetaStore.h"
|
||||
#include "tstream.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
int32_t tqInit() { return tqPushMgrInit(); }
|
||||
|
||||
|
|
@ -81,6 +80,13 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
|
|||
return -1;
|
||||
}
|
||||
memcpy(data, msg, msgLen);
|
||||
|
||||
if (msgType == TDMT_VND_SUBMIT) {
|
||||
// if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg) != 0) {
|
||||
// return -1;
|
||||
// }
|
||||
}
|
||||
|
||||
SRpcMsg req = {
|
||||
.msgType = TDMT_VND_STREAM_TRIGGER,
|
||||
.pCont = data,
|
||||
|
|
@ -265,7 +271,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
fetchOffset = pReq->currentOffset + 1;
|
||||
}
|
||||
|
||||
vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
|
||||
vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
|
||||
pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
|
||||
|
||||
SMqPollRsp rsp = {
|
||||
/*.consumerId = consumerId,*/
|
||||
|
|
@ -292,7 +299,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
int sz = taosArrayGetSize(pConsumer->topics);
|
||||
for (int32_t i = 0; i < sz; i++) {
|
||||
STqTopic* topic = taosArrayGet(pConsumer->topics, i);
|
||||
//TODO race condition
|
||||
// TODO race condition
|
||||
ASSERT(pConsumer->consumerId == consumerId);
|
||||
if (strcmp(topic->topicName, pReq->topic) == 0) {
|
||||
pTopic = topic;
|
||||
|
|
@ -300,7 +307,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
}
|
||||
}
|
||||
if (pTopic == NULL) {
|
||||
vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, pTq->pVnode->vgId);
|
||||
vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
|
||||
pTq->pVnode->vgId);
|
||||
pMsg->pCont = NULL;
|
||||
pMsg->contLen = 0;
|
||||
pMsg->code = -1;
|
||||
|
|
@ -308,17 +316,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
vDebug("poll topic %s from consumer %ld (epoch %d)", pTopic->topicName, consumerId, pReq->epoch);
|
||||
vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, pTq->pVnode->vgId);
|
||||
|
||||
rsp.reqOffset = pReq->currentOffset;
|
||||
rsp.skipLogNum = 0;
|
||||
|
||||
while (1) {
|
||||
/*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
|
||||
//TODO
|
||||
// TODO
|
||||
consumerEpoch = atomic_load_32(&pConsumer->epoch);
|
||||
if (consumerEpoch > pReq->epoch) {
|
||||
//TODO: return
|
||||
if (consumerEpoch > reqEpoch) {
|
||||
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch);
|
||||
break;
|
||||
}
|
||||
SWalReadHead* pHead;
|
||||
|
|
@ -421,22 +429,62 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
|
||||
int32_t tqProcessRebReq(STQ* pTq, char* msg) {
|
||||
SMqMVRebReq req = {0};
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
tDecodeSMqMVRebReq(msg, &req);
|
||||
|
||||
vDebug("vg %d set from consumer %ld to consumer %ld", req.vgId, req.oldConsumerId ,req.newConsumerId);
|
||||
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId);
|
||||
ASSERT(pConsumer);
|
||||
ASSERT(pConsumer->consumerId == req.oldConsumerId);
|
||||
int32_t numOfTopics = taosArrayGetSize(pConsumer->topics);
|
||||
if (numOfTopics == 1) {
|
||||
STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
|
||||
ASSERT(strcmp(pTopic->topicName, req.topic) == 0);
|
||||
STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
|
||||
if (pNewConsumer == NULL) {
|
||||
pConsumer->consumerId = req.newConsumerId;
|
||||
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
|
||||
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
|
||||
tqHandlePurge(pTq->tqMeta, req.oldConsumerId);
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
return 0;
|
||||
} else {
|
||||
taosArrayPush(pNewConsumer->topics, pTopic);
|
||||
}
|
||||
} else {
|
||||
for (int32_t i = 0; i < numOfTopics; i++) {
|
||||
STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
|
||||
if (strcmp(pTopic->topicName, req.topic) == 0) {
|
||||
STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId);
|
||||
if (pNewConsumer == NULL) {
|
||||
pNewConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
|
||||
if (pNewConsumer == NULL) {
|
||||
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
strcpy(pNewConsumer->cgroup, pConsumer->cgroup);
|
||||
pNewConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
|
||||
pNewConsumer->consumerId = req.newConsumerId;
|
||||
pNewConsumer->epoch = 0;
|
||||
|
||||
taosArrayPush(pNewConsumer->topics, pTopic);
|
||||
tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer);
|
||||
tqHandleCommit(pTq->tqMeta, req.newConsumerId);
|
||||
return 0;
|
||||
}
|
||||
ASSERT(pNewConsumer->consumerId == req.newConsumerId);
|
||||
taosArrayPush(pNewConsumer->topics, pTopic);
|
||||
break;
|
||||
}
|
||||
}
|
||||
//
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
|
||||
SMqSetCVgReq req = {0};
|
||||
tDecodeSMqSetCVgReq(msg, &req);
|
||||
bool create = false;
|
||||
|
||||
vDebug("vg %d set to consumer %ld", req.vgId, req.consumerId);
|
||||
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.consumerId);
|
||||
|
|
@ -450,6 +498,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
|
|||
pConsumer->topics = taosArrayInit(0, sizeof(STqTopic));
|
||||
pConsumer->consumerId = req.consumerId;
|
||||
pConsumer->epoch = 0;
|
||||
create = true;
|
||||
}
|
||||
|
||||
STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic));
|
||||
|
|
@ -483,10 +532,17 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
|
|||
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
|
||||
ASSERT(pTopic->buffer.output[i].task);
|
||||
}
|
||||
/*printf("set topic %s to consumer %ld on vg %d\n", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);*/
|
||||
vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);
|
||||
taosArrayPush(pConsumer->topics, pTopic);
|
||||
if (create) {
|
||||
tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
|
||||
tqHandleCommit(pTq->tqMeta, req.consumerId);
|
||||
}
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) {
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include "tqMetaStore.h"
|
||||
#include "vnodeInt.h"
|
||||
// TODO:replace by an abstract file layer
|
||||
#include <fcntl.h>
|
||||
#include <string.h>
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
*/
|
||||
#define _DEFAULT_SOURCE
|
||||
|
||||
#include "tqOffset.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
enum ETqOffsetPersist {
|
||||
TQ_OFFSET_PERSIST__LAZY = 1,
|
||||
|
|
@ -39,4 +39,3 @@ STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) {
|
|||
pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
|
||||
return pStore;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "tqPush.h"
|
||||
#include "vnodeInt.h"
|
||||
|
||||
int32_t tqPushMgrInit() {
|
||||
//
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue