xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0_query_integrate' into feature/scheduler

This commit is contained in:
dapan1121 2022-03-09 16:21:41 +08:00
parent 7e0d9a7e7c cbc3b52db5
commit ee8d007801
1603 changed files with 333840 additions and 5955 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -24,7 +24,6 @@ mac/
*.orig *.orig
src/connector/nodejs/node_modules/ src/connector/nodejs/node_modules/
src/connector/nodejs/out/ src/connector/nodejs/out/
tests/test/
tests/taoshebei/ tests/taoshebei/
tests/taoscsv/ tests/taoscsv/
tests/taosdalipu/ tests/taosdalipu/

7
.gitmodules vendored
View File

@ -10,10 +10,3 @@
[submodule "deps/TSZ"] [submodule "deps/TSZ"]
path = deps/TSZ path = deps/TSZ
url = https://github.com/taosdata/TSZ.git url = https://github.com/taosdata/TSZ.git
[submodule "tests"]
path = tests
url = https://github.com/taosdata/tests
branch = 3.0
[submodule "examples/rust"]
path = examples/rust
url = https://github.com/songtianyi/tdengine-rust-bindings.git

1
CMakeLists.txt
View File

@ -9,6 +9,7 @@ project(
set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
include(${CMAKE_SUPPORT_DIR}/cmake.options) include(${CMAKE_SUPPORT_DIR}/cmake.options)
include(${CMAKE_SUPPORT_DIR}/cmake.version)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3")

31
Jenkinsfile2
View File

@ -74,37 +74,9 @@ def pre_test(){
git pull >/dev/null git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD git checkout -qf FETCH_HEAD
git submodule update --init --recursive --remote git submodule update --init --recursive
''' '''
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
cd ${WKCT}
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
cd ${WKCT}
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
cd ${WKCT}
git checkout 3.0
'''
}
else{
sh '''
cd ${WKCT}
git checkout develop
'''
}
}
sh''' sh'''
cd ${WKCT}
git pull >/dev/null
cd ${WKC} cd ${WKC}
export TZ=Asia/Harbin export TZ=Asia/Harbin
date date
@ -123,7 +95,6 @@ pipeline {
environment{ environment{
WK = '/var/lib/jenkins/workspace/TDinternal' WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDengine' WKC= '/var/lib/jenkins/workspace/TDengine'
WKCT= '/var/lib/jenkins/workspace/TDengine/tests'
} }
stages { stages {
stage('pre_build'){ stage('pre_build'){

77
cmake/cmake.version Normal file
View File

@ -0,0 +1,77 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.0.0.0")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)
SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE})
ELSE ()
SET(TD_VER_COMPATIBLE "3.0.0.0")
ENDIF ()
find_program(HAVE_GIT NAMES git)
IF (DEFINED GITINFO)
SET(TD_VER_GIT ${GITINFO})
ELSEIF (HAVE_GIT)
execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
#message(STATUS "git log result:${GIT_COMMITID}")
IF (GIT_COMMITID)
string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
SET(TD_VER_GIT ${GIT_COMMITID})
ELSE ()
message(STATUS "not a git repository")
SET(TD_VER_GIT "no git commit id")
ENDIF ()
ELSE ()
message(STATUS "no git cmd")
SET(TD_VER_GIT "no git commit id")
ENDIF ()
IF (DEFINED VERDATE)
SET(TD_VER_DATE ${VERDATE})
ELSE ()
STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S")
ENDIF ()
IF (DEFINED VERTYPE)
SET(TD_VER_VERTYPE ${VERTYPE})
ELSE ()
SET(TD_VER_VERTYPE "stable")
ENDIF ()
IF (DEFINED CPUTYPE)
SET(TD_VER_CPUTYPE ${CPUTYPE})
ELSE ()
IF (TD_WINDOWS_32)
SET(TD_VER_CPUTYPE "x86")
ELSEIF (TD_LINUX_32)
SET(TD_VER_CPUTYPE "x86")
ELSEIF (TD_ARM_32)
SET(TD_VER_CPUTYPE "x86")
ELSEIF (TD_MIPS_32)
SET(TD_VER_CPUTYPE "x86")
ELSE ()
SET(TD_VER_CPUTYPE "x64")
ENDIF ()
ENDIF ()
IF (DEFINED OSTYPE)
SET(TD_VER_OSTYPE ${OSTYPE})
ELSE ()
SET(TD_VER_OSTYPE "Linux")
ENDIF ()
MESSAGE(STATUS "============= compile version parameter information start ============= ")
MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER})
MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE})
MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT})
MESSAGE(STATUS "build date:" ${TD_VER_DATE})
MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE})
MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE})
MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE})
MESSAGE(STATUS "============= compile version parameter information end ============= ")
STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER})

2
contrib/test/craft/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c") add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode craft lz4 uv_a) target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)

2
contrib/test/traft/single_node/CMakeLists.txt
View File

@ -3,4 +3,4 @@ target_sources(singleNode
PRIVATE PRIVATE
"singleNode.c" "singleNode.c"
) )
target_link_libraries(singleNode traft lz4 uv_a) target_link_libraries(singleNode PUBLIC traft lz4 uv_a)

1
examples/rust

@ -1 +0,0 @@
Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df

78
include/common/tcommon.h
View File

@ -25,24 +25,6 @@
extern "C" { extern "C" {
#endif #endif
// typedef struct STimeWindow {
// TSKEY skey;
// TSKEY ekey;
// } STimeWindow;
// typedef struct {
// int32_t dataLen;
// char name[TSDB_TABLE_FNAME_LEN];
// char *data;
// } STagData;
// typedef struct SSchema {
// uint8_t type;
// char name[TSDB_COL_NAME_LEN];
// int16_t colId;
// int16_t bytes;
// } SSchema;
enum { enum {
TMQ_CONF__RESET_OFFSET__LATEST = -1, TMQ_CONF__RESET_OFFSET__LATEST = -1,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
@ -50,7 +32,8 @@ enum {
}; };
enum { enum {
TMQ_MSG_TYPE__POLL_RSP = 0, TMQ_MSG_TYPE__DUMMY = 0,
TMQ_MSG_TYPE__POLL_RSP,
TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__EP_RSP,
}; };
@ -73,8 +56,9 @@ typedef struct SColumnDataAgg {
typedef struct SDataBlockInfo { typedef struct SDataBlockInfo {
STimeWindow window; STimeWindow window;
int32_t rows; int32_t rows;
int32_t tupleSize;
int32_t numOfCols; int32_t numOfCols;
int64_t uid; union {int64_t uid; int64_t blockId;};
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SConstantItem { typedef struct SConstantItem {
@ -125,7 +109,7 @@ static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlo
SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
tlen += taosEncodeFixedI16(buf, pColData->info.colId); tlen += taosEncodeFixedI16(buf, pColData->info.colId);
tlen += taosEncodeFixedI16(buf, pColData->info.type); tlen += taosEncodeFixedI16(buf, pColData->info.type);
tlen += taosEncodeFixedI16(buf, pColData->info.bytes); tlen += taosEncodeFixedI32(buf, pColData->info.bytes);
int32_t colSz = rows * pColData->info.bytes; int32_t colSz = rows * pColData->info.bytes;
tlen += taosEncodeBinary(buf, pColData->pData, colSz); tlen += taosEncodeBinary(buf, pColData->pData, colSz);
} }
@ -144,7 +128,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
SColumnInfoData data = {0}; SColumnInfoData data = {0};
buf = taosDecodeFixedI16(buf, &data.info.colId); buf = taosDecodeFixedI16(buf, &data.info.colId);
buf = taosDecodeFixedI16(buf, &data.info.type); buf = taosDecodeFixedI16(buf, &data.info.type);
buf = taosDecodeFixedI16(buf, &data.info.bytes); buf = taosDecodeFixedI32(buf, &data.info.bytes);
int32_t colSz = pBlock->info.rows * data.info.bytes; int32_t colSz = pBlock->info.rows * data.info.bytes;
buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); buf = taosDecodeBinary(buf, (void**)&data.pData, colSz);
taosArrayPush(pBlock->pDataBlock, &data); taosArrayPush(pBlock->pDataBlock, &data);
@ -229,10 +213,23 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) {
//====================================================================================================================== //======================================================================================================================
// the following structure shared by parser and executor // the following structure shared by parser and executor
typedef struct SColumn { typedef struct SColumn {
uint64_t uid; union {
char name[TSDB_COL_NAME_LEN]; uint64_t uid;
int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) int64_t dataBlockId;
SColumnInfo info; };
char name[TSDB_COL_NAME_LEN];
int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string)
union {
int16_t colId;
int16_t slotId;
};
int16_t type;
int32_t bytes;
uint8_t precision;
uint8_t scale;
// SColumnInfo info;
} SColumn; } SColumn;
typedef struct SLimit { typedef struct SLimit {
@ -250,21 +247,24 @@ typedef struct SGroupbyExpr {
bool groupbyTag; // group by tag or column bool groupbyTag; // group by tag or column
} SGroupbyExpr; } SGroupbyExpr;
// the structure for sql function in select clause typedef struct SFunctParam {
typedef struct SSqlExpr { int32_t type;
char token[TSDB_COL_NAME_LEN]; // original token SColumn *pCol;
SSchema resSchema; SVariant param;
} SFunctParam;
int32_t numOfCols; // the structure for sql function in select clause
SColumn* pColumns; // data columns that are required by query typedef struct SExprBasicInfo {
int32_t interBytes; // inter result buffer size SSchema resSchema; // TODO refactor
int16_t numOfParams; // argument value of each function int32_t interBytes; // inter result buffer size, TODO remove it
SVariant param[3]; // parameters are not more than 3 int16_t numOfParams; // argument value of each function
} SSqlExpr; SFunctParam *pParam;
// SVariant param[3]; // parameters are not more than 3
} SExprBasicInfo;
typedef struct SExprInfo { typedef struct SExprInfo {
struct SSqlExpr base; struct SExprBasicInfo base;
struct tExprNode* pExpr; struct tExprNode *pExpr;
} SExprInfo; } SExprInfo;
typedef struct SStateWindow { typedef struct SStateWindow {
@ -285,4 +285,4 @@ typedef struct SSessionWindow {
} }
#endif #endif
#endif /*_TD_COMMON_DEF_H_*/ #endif /*_TD_COMMON_DEF_H_*/

4
include/common/tep.h → include/common/tdatablock.h
View File

@ -94,8 +94,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
void colDataTrim(SColumnInfoData* pColumnInfoData); void colDataTrim(SColumnInfoData* pColumnInfoData);
size_t colDataGetNumOfCols(const SSDataBlock* pBlock); size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
size_t colDataGetNumOfRows(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,

2
include/common/tglobal.h
View File

@ -58,6 +58,7 @@ extern int32_t tsMonitorInterval;
extern char tsMonitorFqdn[]; extern char tsMonitorFqdn[];
extern uint16_t tsMonitorPort; extern uint16_t tsMonitorPort;
extern int32_t tsMonitorMaxLogs; extern int32_t tsMonitorMaxLogs;
extern bool tsMonitorComp;
// query buffer management // query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
@ -83,7 +84,6 @@ extern int64_t tsMaxRetentWindow;
extern char version[]; extern char version[];
extern char compatible_version[]; extern char compatible_version[];
extern char gitinfo[]; extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[]; extern char buildinfo[];
// lossy // lossy

286
include/common/tmsg.h
View File

@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
typedef enum { typedef enum {
HEARTBEAT_TYPE_MQ = 0, HEARTBEAT_TYPE_MQ = 0,
HEARTBEAT_TYPE_QUERY = 1, HEARTBEAT_TYPE_QUERY,
// types can be added here // types can be added here
// //
HEARTBEAT_TYPE_MAX HEARTBEAT_TYPE_MAX
@ -424,9 +424,10 @@ typedef struct {
int16_t slotId; int16_t slotId;
}; };
int16_t type; int16_t type;
int16_t bytes; int32_t bytes;
SColumnFilterList flist; uint8_t precision;
uint8_t scale;
} SColumnInfo; } SColumnInfo;
typedef struct { typedef struct {
@ -456,57 +457,6 @@ typedef struct {
int64_t offset; int64_t offset;
} SInterval; } SInterval;
typedef struct {
SMsgHead head;
char version[TSDB_VERSION_LEN];
bool stableQuery; // super table query or not
bool topBotQuery; // TODO used bitwise flag
bool interpQuery; // interp query or not
bool groupbyColumn; // denote if this is a groupby normal column query
bool hasTagResults; // if there are tag values in final result or not
bool timeWindowInterpo; // if the time window start/end required interpolation
bool queryBlockDist; // if query data block distribution
bool stabledev; // super table stddev query
bool tsCompQuery; // is tscomp query
bool simpleAgg;
bool pointInterpQuery; // point interpolation query
bool needReverseScan; // need reverse scan
bool stateWindow; // state window flag
STimeWindow window;
int32_t numOfTables;
int16_t order;
int16_t orderColId;
int16_t numOfCols; // the number of columns will be load from vnode
SInterval interval;
// SSessionWindow sw; // session window
int16_t tagCondLen; // tag length in current query
int16_t colCondLen; // column length in current query
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query.
int16_t prjOrder; // global order in super table projection query.
int64_t limit;
int64_t offset;
int32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t fillType; // interpolate type
int64_t fillVal; // default value array list
int32_t secondStageOutput;
STsBufInfo tsBuf; // tsBuf info
int32_t numOfTags; // number of tags columns involved
int32_t sqlstrLen; // sql query string
int32_t prevResultLen; // previous result length
int32_t numOfOperator;
int32_t tableScanOperator; // table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t udfContentOffset;
int32_t udfContentLen;
SColumnInfo tableCols[];
} SQueryTableReq;
typedef struct { typedef struct {
int32_t code; int32_t code;
} SQueryTableRsp; } SQueryTableRsp;
@ -696,6 +646,11 @@ typedef struct {
int64_t totalStorage; int64_t totalStorage;
int64_t compStorage; int64_t compStorage;
int64_t pointsWritten; int64_t pointsWritten;
int64_t numOfSelectReqs;
int64_t numOfInsertReqs;
int64_t numOfInsertSuccessReqs;
int64_t numOfBatchInsertReqs;
int64_t numOfBatchInsertSuccessReqs;
} SVnodeLoad; } SVnodeLoad;
typedef struct { typedef struct {
@ -766,7 +721,7 @@ typedef struct {
int32_t fsyncPeriod; int32_t fsyncPeriod;
uint32_t hashBegin; uint32_t hashBegin;
uint32_t hashEnd; uint32_t hashEnd;
int8_t hashMethod; int8_t hashMethod;
int8_t walLevel; int8_t walLevel;
int8_t precision; int8_t precision;
int8_t compression; int8_t compression;
@ -777,7 +732,7 @@ typedef struct {
int8_t selfIndex; int8_t selfIndex;
int8_t streamMode; int8_t streamMode;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
} SCreateVnodeReq, SAlterVnodeReq; } SCreateVnodeReq, SAlterVnodeReq;
int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
@ -898,6 +853,8 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
void tFreeSShowRsp(SShowRsp* pRsp); void tFreeSShowRsp(SShowRsp* pRsp);
typedef struct { typedef struct {
int32_t type;
char db[TSDB_DB_FNAME_LEN];
int64_t showId; int64_t showId;
int8_t free; int8_t free;
} SRetrieveTableReq; } SRetrieveTableReq;
@ -925,6 +882,8 @@ int32_t tDeserializeSCreateDnodeReq(void* buf, int32_t bufLen, SCreateDnodeReq*
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
char fqdn[TSDB_FQDN_LEN];
int32_t port;
} SDropDnodeReq; } SDropDnodeReq;
int32_t tSerializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq); int32_t tSerializeSDropDnodeReq(void* buf, int32_t bufLen, SDropDnodeReq* pReq);
@ -1450,7 +1409,7 @@ typedef struct {
typedef struct SMqCMGetSubEpReq { typedef struct SMqCMGetSubEpReq {
int64_t consumerId; int64_t consumerId;
int32_t epoch; int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
} SMqCMGetSubEpReq; } SMqCMGetSubEpReq;
static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
@ -1750,7 +1709,7 @@ typedef struct {
int32_t vgId; int32_t vgId;
int64_t consumerId; int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
char* sql; char* sql;
char* logicalPlan; char* logicalPlan;
char* physicalPlan; char* physicalPlan;
@ -1813,7 +1772,7 @@ typedef struct {
int32_t vgId; int32_t vgId;
int64_t consumerId; int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
} SMqSetCVgRsp; } SMqSetCVgRsp;
typedef struct { typedef struct {
@ -1821,14 +1780,14 @@ typedef struct {
int32_t vgId; int32_t vgId;
int64_t consumerId; int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
} SMqMVRebRsp; } SMqMVRebRsp;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int64_t offset; int64_t offset;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
} SMqOffset; } SMqOffset;
typedef struct { typedef struct {
@ -1904,6 +1863,205 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
} }
return buf; return buf;
} }
typedef enum {
TD_TIME_UNIT_UNKNOWN = -1,
TD_TIME_UNIT_YEAR = 0,
TD_TIME_UNIT_SEASON = 1,
TD_TIME_UNIT_MONTH = 2,
TD_TIME_UNIT_WEEK = 3,
TD_TIME_UNIT_DAY = 4,
TD_TIME_UNIT_HOUR = 5,
TD_TIME_UNIT_MINUTE = 6,
TD_TIME_UNIT_SEC = 7,
TD_TIME_UNIT_MILLISEC = 8,
TD_TIME_UNIT_MICROSEC = 9,
TD_TIME_UNIT_NANOSEC = 10
} ETDTimeUnit;
typedef struct {
uint8_t version; // for compatibility
uint8_t intervalUnit;
uint8_t slidingUnit;
char indexName[TSDB_INDEX_NAME_LEN + 1];
col_id_t numOfColIds;
uint16_t numOfFuncIds;
uint64_t tableUid; // super/common table uid
int64_t interval;
int64_t sliding;
col_id_t* colIds; // sorted column ids
uint16_t* funcIds; // sorted sma function ids
} STSma; // Time-range-wise SMA
typedef struct {
int64_t ver; // use a general definition
STSma tSma;
} SVCreateTSmaReq;
typedef struct {
int8_t type; // 0 status report, 1 update data
char indexName[TSDB_INDEX_NAME_LEN + 1]; //
STimeWindow windows;
} STSmaMsg;
typedef struct {
int64_t ver; // use a general definition
char indexName[TSDB_INDEX_NAME_LEN + 1];
} SVDropTSmaReq;
typedef struct {
} SVCreateTSmaRsp, SVDropTSmaRsp;
int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
typedef struct {
STimeWindow tsWindow; // [skey, ekey]
uint64_t tableUid; // sub/common table uid
int32_t numOfBlocks; // number of sma blocks for each column, total number is numOfBlocks*numOfColId
int32_t dataLen; // total data length
col_id_t* colIds; // e.g. 2,4,9,10
col_id_t numOfColIds; // e.g. 4
char data[]; // the sma blocks
} STSmaData;
// TODO: move to the final location afte schema of STSma/STSmaData defined
static FORCE_INLINE void tdDestroySmaData(STSmaData* pSmaData) {
if (pSmaData) {
if (pSmaData->colIds) {
tfree(pSmaData->colIds);
}
tfree(pSmaData);
}
}
// RSma: Rollup SMA
typedef struct {
int64_t interval;
int32_t retention; // unit: day
uint16_t days; // unit: day
int8_t intervalUnit;
} SSmaParams;
typedef struct {
STSma tsma;
float xFilesFactor;
SArray* smaParams; // SSmaParams
} SRSma;
typedef struct {
uint32_t number;
STSma* tSma;
} STSmaWrapper;
static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
if (pSma) {
tfree(pSma->colIds);
tfree(pSma->funcIds);
}
}
static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
if (pSW) {
if (pSW->tSma) {
for (uint32_t i = 0; i < pSW->number; ++i) {
tdDestroyTSma(pSW->tSma + i);
}
tfree(pSW->tSma);
}
}
}
static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
int32_t tlen = 0;
tlen += taosEncodeFixedU8(buf, pSma->version);
tlen += taosEncodeFixedU8(buf, pSma->intervalUnit);
tlen += taosEncodeFixedU8(buf, pSma->slidingUnit);
tlen += taosEncodeString(buf, pSma->indexName);
tlen += taosEncodeFixedU16(buf, pSma->numOfColIds);
tlen += taosEncodeFixedU16(buf, pSma->numOfFuncIds);
tlen += taosEncodeFixedU64(buf, pSma->tableUid);
tlen += taosEncodeFixedI64(buf, pSma->interval);
tlen += taosEncodeFixedI64(buf, pSma->sliding);
for (col_id_t i = 0; i < pSma->numOfColIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i));
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i));
}
return tlen;
}
static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
int32_t tlen = 0;
tlen += taosEncodeFixedU32(buf, pSW->number);
for (uint32_t i = 0; i < pSW->number; ++i) {
tlen += tEncodeTSma(buf, pSW->tSma + i);
}
return tlen;
}
static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedU8(buf, &pSma->version);
buf = taosDecodeFixedU8(buf, &pSma->intervalUnit);
buf = taosDecodeFixedU8(buf, &pSma->slidingUnit);
buf = taosDecodeStringTo(buf, pSma->indexName);
buf = taosDecodeFixedU16(buf, &pSma->numOfColIds);
buf = taosDecodeFixedU16(buf, &pSma->numOfFuncIds);
buf = taosDecodeFixedU64(buf, &pSma->tableUid);
buf = taosDecodeFixedI64(buf, &pSma->interval);
buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->numOfColIds > 0) {
pSma->colIds = (col_id_t*)calloc(pSma->numOfColIds, sizeof(STSma));
if (pSma->colIds == NULL) {
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfColIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->colIds + i);
}
} else {
pSma->colIds = NULL;
}
if (pSma->numOfFuncIds > 0) {
pSma->funcIds = (uint16_t*)calloc(pSma->numOfFuncIds, sizeof(STSma));
if (pSma->funcIds == NULL) {
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->funcIds + i);
}
} else {
pSma->funcIds = NULL;
}
return buf;
}
static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->number);
pSW->tSma = (STSma*)calloc(pSW->number, sizeof(STSma));
if (pSW->tSma == NULL) {
return NULL;
}
for (uint32_t i = 0; i < pSW->number; ++i) {
if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
for (uint32_t j = i; j >= 0; --i) {
tdDestroyTSma(pSW->tSma + j);
}
free(pSW->tSma);
return NULL;
}
}
return buf;
}
typedef struct { typedef struct {
int64_t uid; int64_t uid;
@ -1945,7 +2103,7 @@ typedef struct {
int64_t consumerId; int64_t consumerId;
int64_t blockingTime; int64_t blockingTime;
int32_t epoch; int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
int64_t currentOffset; int64_t currentOffset;
char topic[TSDB_TOPIC_FNAME_LEN]; char topic[TSDB_TOPIC_FNAME_LEN];
@ -1964,7 +2122,7 @@ typedef struct {
typedef struct { typedef struct {
int64_t consumerId; int64_t consumerId;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
SArray* topics; // SArray<SMqSubTopicEp> SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp; } SMqCMGetSubEpRsp;

4
include/common/tmsgdef.h
View File

@ -135,6 +135,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp) TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "mnode-systable-retrieve", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
@ -186,6 +187,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
// Requests handled by QNODE // Requests handled by QNODE
TD_NEW_MSG_SEG(TDMT_QND_MSG) TD_NEW_MSG_SEG(TDMT_QND_MSG)

6
include/common/trow.h
View File

@ -118,6 +118,8 @@ typedef struct {
} SKvRow; } SKvRow;
typedef struct { typedef struct {
/// timestamp
TSKEY ts;
union { union {
/// union field for encode and decode /// union field for encode and decode
uint32_t info; uint32_t info;
@ -138,8 +140,6 @@ typedef struct {
uint32_t len; uint32_t len;
/// row version /// row version
uint64_t ver; uint64_t ver;
/// timestamp
TSKEY ts;
/// the inline data, maybe a tuple or a k-v tuple /// the inline data, maybe a tuple or a k-v tuple
char data[]; char data[];
} STSRow; } STSRow;
@ -173,7 +173,7 @@ typedef struct {
#define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_DATA(r) ((r)->data)
#define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_LEN(r) ((r)->len)
#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY(r) ((r)->ts)
#define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16) #define TD_ROW_KEY_ADDR(r) (r)
// N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.

224
include/common/ttokendef.h
View File

@ -34,110 +34,121 @@
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_DATABASE 19 #define TK_USER 19
#define TK_IF 20 #define TK_PASS 20
#define TK_NOT 21 #define TK_NK_STRING 21
#define TK_EXISTS 22 #define TK_ALTER 22
#define TK_BLOCKS 23 #define TK_PRIVILEGE 23
#define TK_NK_INTEGER 24 #define TK_DROP 24
#define TK_CACHE 25 #define TK_SHOW 25
#define TK_CACHELAST 26 #define TK_USERS 26
#define TK_COMP 27 #define TK_DNODE 27
#define TK_DAYS 28 #define TK_PORT 28
#define TK_FSYNC 29 #define TK_NK_INTEGER 29
#define TK_MAXROWS 30 #define TK_DNODES 30
#define TK_MINROWS 31 #define TK_NK_ID 31
#define TK_KEEP 32 #define TK_NK_IPTOKEN 32
#define TK_PRECISION 33 #define TK_DATABASE 33
#define TK_NK_STRING 34 #define TK_DATABASES 34
#define TK_QUORUM 35 #define TK_USE 35
#define TK_REPLICA 36 #define TK_IF 36
#define TK_TTL 37 #define TK_NOT 37
#define TK_WAL 38 #define TK_EXISTS 38
#define TK_VGROUPS 39 #define TK_BLOCKS 39
#define TK_SINGLE_STABLE 40 #define TK_CACHE 40
#define TK_STREAM_MODE 41 #define TK_CACHELAST 41
#define TK_USE 42 #define TK_COMP 42
#define TK_TABLE 43 #define TK_DAYS 43
#define TK_NK_LP 44 #define TK_FSYNC 44
#define TK_NK_RP 45 #define TK_MAXROWS 45
#define TK_STABLE 46 #define TK_MINROWS 46
#define TK_USING 47 #define TK_KEEP 47
#define TK_TAGS 48 #define TK_PRECISION 48
#define TK_NK_ID 49 #define TK_QUORUM 49
#define TK_NK_DOT 50 #define TK_REPLICA 50
#define TK_NK_COMMA 51 #define TK_TTL 51
#define TK_COMMENT 52 #define TK_WAL 52
#define TK_BOOL 53 #define TK_VGROUPS 53
#define TK_TINYINT 54 #define TK_SINGLE_STABLE 54
#define TK_SMALLINT 55 #define TK_STREAM_MODE 55
#define TK_INT 56 #define TK_TABLE 56
#define TK_INTEGER 57 #define TK_NK_LP 57
#define TK_BIGINT 58 #define TK_NK_RP 58
#define TK_FLOAT 59 #define TK_STABLE 59
#define TK_DOUBLE 60 #define TK_TABLES 60
#define TK_BINARY 61 #define TK_STABLES 61
#define TK_TIMESTAMP 62 #define TK_USING 62
#define TK_NCHAR 63 #define TK_TAGS 63
#define TK_UNSIGNED 64 #define TK_NK_DOT 64
#define TK_JSON 65 #define TK_NK_COMMA 65
#define TK_VARCHAR 66 #define TK_COMMENT 66
#define TK_MEDIUMBLOB 67 #define TK_BOOL 67
#define TK_BLOB 68 #define TK_TINYINT 68
#define TK_VARBINARY 69 #define TK_SMALLINT 69
#define TK_DECIMAL 70 #define TK_INT 70
#define TK_SMA 71 #define TK_INTEGER 71
#define TK_SHOW 72 #define TK_BIGINT 72
#define TK_DATABASES 73 #define TK_FLOAT 73
#define TK_TABLES 74 #define TK_DOUBLE 74
#define TK_NK_FLOAT 75 #define TK_BINARY 75
#define TK_NK_BOOL 76 #define TK_TIMESTAMP 76
#define TK_NK_VARIABLE 77 #define TK_NCHAR 77
#define TK_BETWEEN 78 #define TK_UNSIGNED 78
#define TK_IS 79 #define TK_JSON 79
#define TK_NULL 80 #define TK_VARCHAR 80
#define TK_NK_LT 81 #define TK_MEDIUMBLOB 81
#define TK_NK_GT 82 #define TK_BLOB 82
#define TK_NK_LE 83 #define TK_VARBINARY 83
#define TK_NK_GE 84 #define TK_DECIMAL 84
#define TK_NK_NE 85 #define TK_SMA 85
#define TK_NK_EQ 86 #define TK_NK_FLOAT 86
#define TK_LIKE 87 #define TK_NK_BOOL 87
#define TK_MATCH 88 #define TK_NK_VARIABLE 88
#define TK_NMATCH 89 #define TK_BETWEEN 89
#define TK_IN 90 #define TK_IS 90
#define TK_FROM 91 #define TK_NULL 91
#define TK_AS 92 #define TK_NK_LT 92
#define TK_JOIN 93 #define TK_NK_GT 93
#define TK_ON 94 #define TK_NK_LE 94
#define TK_INNER 95 #define TK_NK_GE 95
#define TK_SELECT 96 #define TK_NK_NE 96
#define TK_DISTINCT 97 #define TK_NK_EQ 97
#define TK_WHERE 98 #define TK_LIKE 98
#define TK_PARTITION 99 #define TK_MATCH 99
#define TK_BY 100 #define TK_NMATCH 100
#define TK_SESSION 101 #define TK_IN 101
#define TK_STATE_WINDOW 102 #define TK_FROM 102
#define TK_INTERVAL 103 #define TK_AS 103
#define TK_SLIDING 104 #define TK_JOIN 104
#define TK_FILL 105 #define TK_ON 105
#define TK_VALUE 106 #define TK_INNER 106
#define TK_NONE 107 #define TK_SELECT 107
#define TK_PREV 108 #define TK_DISTINCT 108
#define TK_LINEAR 109 #define TK_WHERE 109
#define TK_NEXT 110 #define TK_PARTITION 110
#define TK_GROUP 111 #define TK_BY 111
#define TK_HAVING 112 #define TK_SESSION 112
#define TK_ORDER 113 #define TK_STATE_WINDOW 113
#define TK_SLIMIT 114 #define TK_INTERVAL 114
#define TK_SOFFSET 115 #define TK_SLIDING 115
#define TK_LIMIT 116 #define TK_FILL 116
#define TK_OFFSET 117 #define TK_VALUE 117
#define TK_ASC 118 #define TK_NONE 118
#define TK_DESC 119 #define TK_PREV 119
#define TK_NULLS 120 #define TK_LINEAR 120
#define TK_FIRST 121 #define TK_NEXT 121
#define TK_LAST 122 #define TK_GROUP 122
#define TK_HAVING 123
#define TK_ORDER 124
#define TK_SLIMIT 125
#define TK_SOFFSET 126
#define TK_LIMIT 127
#define TK_OFFSET 128
#define TK_ASC 129
#define TK_DESC 130
#define TK_NULLS 131
#define TK_FIRST 132
#define TK_LAST 133
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301
@ -155,9 +166,8 @@
#define TK_NOW 504 #define TK_NOW 504
#define TK_VALUES 507 #define TK_VALUES 507
#define TK_IMPORT 507 #define TK_IMPORT 507
#define TK_SEMI 508 #define TK_NK_SEMI 508
#define TK_IPTOKEN 509
#define TK_NIL 65535 #define TK_NK_NIL 65535
#endif /*_TD_COMMON_TOKEN_H_*/ #endif /*_TD_COMMON_TOKEN_H_*/

5
include/dnode/snode/snode.h
View File

@ -16,6 +16,9 @@
#ifndef _TD_SNODE_H_ #ifndef _TD_SNODE_H_
#define _TD_SNODE_H_ #define _TD_SNODE_H_
#include "tmsg.h"
#include "trpc.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -88,4 +91,4 @@ void sndDestroy(const char *path);
} }
#endif #endif
#endif /*_TD_SNODE_H_*/ #endif /*_TD_SNODE_H_*/

19
include/libs/executor/executor.h
View File

@ -89,25 +89,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
*/ */
int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext); int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
/**
*
* Retrieve the actual results to fill the response message payload.
* Note that this function must be executed after qRetrieveQueryResultInfo is invoked.
*
* @param tinfo tinfo object
* @param pRsp response message
* @param contLen payload length
* @return
*/
//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
/**
* return the transporter context (RPC)
* @param tinfo
* @return
*/
void* qGetResultRetrieveMsg(qTaskInfo_t tinfo);
/** /**
* kill the ongoing query and free the query handle and corresponding resources automatically * kill the ongoing query and free the query handle and corresponding resources automatically
* @param tinfo qhandle * @param tinfo qhandle

112
include/libs/function/function.h
View File

@ -20,9 +20,30 @@
extern "C" { extern "C" {
#endif #endif
#include "tbuffer.h"
#include "tcommon.h" #include "tcommon.h"
#include "tvariant.h" #include "tvariant.h"
#include "tbuffer.h"
struct SqlFunctionCtx;
struct SResultRowEntryInfo;
typedef struct SFunctionNode SFunctionNode;
typedef struct SFuncExecEnv {
int32_t calcMemSize;
} SFuncExecEnv;
typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
typedef struct SFuncExecFuncs {
FExecGetEnv getEnv;
FExecInit init;
FExecProcess process;
FExecFinalize finalize;
} SFuncExecFuncs;
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
@ -118,57 +139,58 @@ typedef struct SExtTagsInfo {
} SExtTagsInfo; } SExtTagsInfo;
typedef struct SResultDataInfo { typedef struct SResultDataInfo {
int16_t precision;
int16_t scale;
int16_t type; int16_t type;
int16_t bytes; int16_t bytes;
int32_t intermediateBytes; int32_t interBufSize;
} SResultDataInfo; } SResultDataInfo;
#define GET_RES_INFO(ctx) ((ctx)->resultInfo) #define GET_RES_INFO(ctx) ((ctx)->resultInfo)
typedef struct SFunctionFpSet { typedef struct SInputColumnInfoData {
bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment int32_t totalRows; // total rows in current columnar data
void (*addInput)(struct SqlFunctionCtx *pCtx); int32_t startRowIndex; // handle started row index
int32_t numOfRows; // the number of rows needs to be handled
// finalizer must be called after all exec has been executed to generated final result. int32_t numOfInputCols; // PTS is not included
void (*finalize)(struct SqlFunctionCtx *pCtx); bool colDataAggIsSet;// if agg is set or not
void (*combine)(struct SqlFunctionCtx *pCtx); SColumnInfoData *pPTS; // primary timestamp column
} SFunctionFpSet; SColumnInfoData **pData;
SColumnDataAgg **pColumnDataAgg;
extern SFunctionFpSet fpSet[1]; } SInputColumnInfoData;
// sql function runtime context // sql function runtime context
typedef struct SqlFunctionCtx { typedef struct SqlFunctionCtx {
int32_t startRow; SInputColumnInfoData input;
int32_t size; // number of rows SResultDataInfo resDataInfo;
uint32_t order; // asc|desc
////////////////////////////////////////////////////////////////
int32_t startRow; // start row index
int32_t size; // handled processed row number
SColumnInfoData* pInput; SColumnInfoData* pInput;
SColumnDataAgg agg;
uint32_t order; // asc|desc int16_t inputType; // TODO remove it
int16_t inputType; int16_t inputBytes; // TODO remove it
int16_t inputBytes; bool hasNull; // null value exist in current block, TODO remove it
bool requireNull; // require null in some function, TODO remove it
SResultDataInfo resDataInfo; int32_t columnIndex; // TODO remove it
bool hasNull; // null value exist in current block uint8_t currentStage; // record current running step, default: 0
bool requireNull; // require null in some function bool isAggSet;
bool stableQuery; /////////////////////////////////////////////////////////////////
int16_t functionId; // function id bool stableQuery;
char * pOutput; // final result output buffer, point to sdata->data int16_t functionId; // function id
uint8_t currentStage; // record current running step, default: 0 char * pOutput; // final result output buffer, point to sdata->data
int64_t startTs; // timestamp range of current query when function is executed on a specific data block int64_t startTs; // timestamp range of current query when function is executed on a specific data block
int32_t numOfParams; int32_t numOfParams;
SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
int64_t *ptsList; // corresponding timestamp array list int64_t *ptsList; // corresponding timestamp array list
void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SVariant tag; SVariant tag;
bool isAggSet;
SColumnDataAgg agg;
struct SResultRowEntryInfo *resultInfo; struct SResultRowEntryInfo *resultInfo;
SExtTagsInfo tagInfo; SExtTagsInfo tagInfo;
SPoint1 start; SPoint1 start;
SPoint1 end; SPoint1 end;
SFuncExecFuncs fpSet;
int32_t columnIndex;
SFunctionFpSet* fpSet;
} SqlFunctionCtx; } SqlFunctionCtx;
enum { enum {
@ -194,9 +216,10 @@ typedef struct tExprNode {
struct SVariant *pVal; // value node struct SVariant *pVal; // value node
struct {// function node struct {// function node
char functionName[FUNCTIONS_NAME_MAX_LENGTH]; char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor
int32_t functionId;
int32_t num; int32_t num;
SFunctionNode *pFunctNode;
// Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the
// calculation instead. // calculation instead.
// E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes. // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes.
@ -207,7 +230,6 @@ typedef struct tExprNode {
}; };
} tExprNode; } tExprNode;
//TODO create?
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
@ -294,7 +316,7 @@ tExprNode* exprdup(tExprNode* pTree);
void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);
void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell);
int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock);
bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);

16
include/libs/function/functionMgt.h
View File

@ -102,22 +102,6 @@ struct SqlFunctionCtx;
struct SResultRowEntryInfo; struct SResultRowEntryInfo;
struct STimeWindow; struct STimeWindow;
typedef struct SFuncExecEnv {
int32_t calcMemSize;
} SFuncExecEnv;
typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
typedef struct SFuncExecFuncs {
FExecGetEnv getEnv;
FExecInit init;
FExecProcess process;
FExecFinalize finalize;
} SFuncExecFuncs;
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs { typedef struct SScalarFuncExecFuncs {

55
include/libs/monitor/monitor.h
View File

@ -18,6 +18,7 @@
#include "tarray.h" #include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlog.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -31,6 +32,8 @@ extern "C" {
typedef struct { typedef struct {
int32_t dnode_id; int32_t dnode_id;
char dnode_ep[TSDB_EP_LEN]; char dnode_ep[TSDB_EP_LEN];
int64_t cluster_id;
int32_t protocol;
} SMonBasicInfo; } SMonBasicInfo;
typedef struct { typedef struct {
@ -85,29 +88,26 @@ typedef struct {
typedef struct { typedef struct {
float uptime; // day float uptime; // day
float cpu_engine; double cpu_engine;
float cpu_system; double cpu_system;
float cpu_cores; float cpu_cores;
int64_t mem_engine; // KB int64_t mem_engine; // KB
int64_t mem_system; // KB int64_t mem_system; // KB
int64_t mem_total; // KB int64_t mem_total; // KB
float disk_engine; // GB int64_t disk_engine; // Byte
float disk_used; // GB int64_t disk_used; // Byte
float disk_total; // GB int64_t disk_total; // Byte
int64_t net_in; int64_t net_in; // bytes
int64_t net_out; int64_t net_out; // bytes
float io_read; int64_t io_read; // bytes
float io_write; int64_t io_write; // bytes
float io_read_disk; int64_t io_read_disk; // bytes
float io_write_disk; int64_t io_write_disk; // bytes
int32_t req_select; int64_t req_select;
float req_select_rate; int64_t req_insert;
int32_t req_insert; int64_t req_insert_success;
int32_t req_insert_success; int64_t req_insert_batch;
float req_insert_rate; int64_t req_insert_batch_success;
int32_t req_insert_batch;
int32_t req_insert_batch_success;
float req_insert_batch_rate;
int32_t errors; int32_t errors;
int32_t vnodes_num; int32_t vnodes_num;
int32_t masters; int32_t masters;
@ -116,7 +116,7 @@ typedef struct {
typedef struct { typedef struct {
char name[TSDB_FILENAME_LEN]; char name[TSDB_FILENAME_LEN];
int32_t level; int8_t level;
SDiskSize size; SDiskSize size;
} SMonDiskDesc; } SMonDiskDesc;
@ -126,23 +126,18 @@ typedef struct {
SMonDiskDesc tempdir; SMonDiskDesc tempdir;
} SMonDiskInfo; } SMonDiskInfo;
typedef struct {
int64_t ts;
int8_t level;
char content[MON_LOG_LEN];
} SMonLogItem;
typedef struct SMonInfo SMonInfo; typedef struct SMonInfo SMonInfo;
typedef struct { typedef struct {
const char *server; const char *server;
uint16_t port; uint16_t port;
int32_t maxLogs; int32_t maxLogs;
bool comp;
} SMonCfg; } SMonCfg;
int32_t monInit(const SMonCfg *pCfg); int32_t monInit(const SMonCfg *pCfg);
void monCleanup(); void monCleanup();
void monAddLogItem(SMonLogItem *pItem); void monRecordLog(int64_t ts, ELogLevel level, const char *content);
SMonInfo *monCreateMonitorInfo(); SMonInfo *monCreateMonitorInfo();
void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo); void monSetBasicInfo(SMonInfo *pMonitor, SMonBasicInfo *pInfo);

61
include/libs/nodes/cmdnodes.h
View File

@ -54,6 +54,12 @@ typedef struct SUseDatabaseStmt {
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
} SUseDatabaseStmt; } SUseDatabaseStmt;
typedef struct SDropDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
bool ignoreNotExists;
} SDropDatabaseStmt;
typedef struct STableOptions { typedef struct STableOptions {
int32_t keep; int32_t keep;
int32_t ttl; int32_t ttl;
@ -94,6 +100,61 @@ typedef struct SCreateMultiTableStmt {
SNodeList* pSubTables; SNodeList* pSubTables;
} SCreateMultiTableStmt; } SCreateMultiTableStmt;
typedef struct SDropTableClause {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
} SDropTableClause;
typedef struct SDropTableStmt {
ENodeType type;
SNodeList* pTables;
} SDropTableStmt;
typedef struct SDropSuperTableStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
} SDropSuperTableStmt;
typedef struct SCreateUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
} SCreateUserStmt;
typedef struct SAlterUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN];
int8_t alterType;
} SAlterUserStmt;
typedef struct SDropUserStmt {
ENodeType type;
char useName[TSDB_USER_LEN];
} SDropUserStmt;
typedef struct SCreateDnodeStmt {
ENodeType type;
char fqdn[TSDB_FQDN_LEN];
int32_t port;
} SCreateDnodeStmt;
typedef struct SDropDnodeStmt {
ENodeType type;
int32_t dnodeId;
char fqdn[TSDB_FQDN_LEN];
int32_t port;
} SDropDnodeStmt;
typedef struct SShowStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SShowStmt;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

18
include/libs/nodes/nodes.h
View File

@ -72,12 +72,25 @@ typedef enum ENodeType {
QUERY_NODE_SELECT_STMT, QUERY_NODE_SELECT_STMT,
QUERY_NODE_VNODE_MODIF_STMT, QUERY_NODE_VNODE_MODIF_STMT,
QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_SHOW_DATABASES_STMT, // temp
QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLE_STMT, QUERY_NODE_CREATE_MULTI_TABLE_STMT,
QUERY_NODE_USE_DATABASE_STMT, QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_SHOW_DATABASES_STMT, // temp QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT,
QUERY_NODE_SHOW_TABLES_STMT, // temp QUERY_NODE_SHOW_TABLES_STMT, // temp
QUERY_NODE_SHOW_STABLES_STMT,
QUERY_NODE_CREATE_USER_STMT,
QUERY_NODE_ALTER_USER_STMT,
QUERY_NODE_DROP_USER_STMT,
QUERY_NODE_SHOW_USERS_STMT,
QUERY_NODE_USE_DATABASE_STMT,
QUERY_NODE_CREATE_DNODE_STMT,
QUERY_NODE_DROP_DNODE_STMT,
QUERY_NODE_SHOW_DNODES_STMT,
QUERY_NODE_SHOW_VGROUPS_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_SCAN,
@ -93,6 +106,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN,
QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN,
QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT, QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_JOIN, QUERY_NODE_PHYSICAL_PLAN_JOIN,
QUERY_NODE_PHYSICAL_PLAN_AGG, QUERY_NODE_PHYSICAL_PLAN_AGG,

16
include/libs/qcom/query.h
View File

@ -173,20 +173,18 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) #define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus #ifdef __cplusplus
} }

5
include/libs/scalar/filter.h
View File

@ -19,10 +19,12 @@
extern "C" { extern "C" {
#endif #endif
#include "tcommon.h"
#include "nodes.h"
typedef struct SFilterInfo SFilterInfo; typedef struct SFilterInfo SFilterInfo;
typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
enum { enum {
FLT_OPTION_NO_REWRITE = 1, FLT_OPTION_NO_REWRITE = 1,
FLT_OPTION_TIMESTAMP = 2, FLT_OPTION_TIMESTAMP = 2,
@ -34,7 +36,6 @@ typedef struct SFilterColumnParam{
SArray* pDataBlock; SArray* pDataBlock;
} SFilterColumnParam; } SFilterColumnParam;
extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);

15
include/libs/sync/sync.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif #endif
#include <stdint.h> #include <stdint.h>
#include <tep.h> #include <tdatablock.h>
#include "taosdef.h" #include "taosdef.h"
#include "trpc.h" #include "trpc.h"
@ -138,6 +138,8 @@ typedef struct SSyncInfo {
void* rpcClient; void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
void* queue;
int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
} SSyncInfo; } SSyncInfo;
@ -147,13 +149,10 @@ typedef struct SSyncNode SSyncNode;
int32_t syncInit(); int32_t syncInit();
void syncCleanUp(); void syncCleanUp();
int64_t syncStart(const SSyncInfo* pSyncInfo); int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid); void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
// int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
ESyncState syncGetMyRole(int64_t rid); ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);

4
include/util/thttp.h → include/libs/transport/thttp.h
View File

@ -22,7 +22,9 @@
extern "C" { extern "C" {
#endif #endif
int32_t taosSendHttpReport(const char* server, uint16_t port, const char* pCont, int32_t contLen); typedef enum { HTTP_GZIP, HTTP_FLAT } EHttpCompFlag;
int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag);
#ifdef __cplusplus #ifdef __cplusplus
} }

62
include/libs/wal/wal.h
View File

@ -24,43 +24,41 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t wDebugFlag; #define wFatal(...) \
{ \
#define wFatal(...) \ if (wDebugFlag & DEBUG_FATAL) { \
{ \ taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
if (wDebugFlag & DEBUG_FATAL) { \ } \
taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
} \
} }
#define wError(...) \ #define wError(...) \
{ \ { \
if (wDebugFlag & DEBUG_ERROR) { \ if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} }
#define wWarn(...) \ #define wWarn(...) \
{ \ { \
if (wDebugFlag & DEBUG_WARN) { \ if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} }
#define wInfo(...) \ #define wInfo(...) \
{ \ { \
if (wDebugFlag & DEBUG_INFO) { \ if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} }
#define wDebug(...) \ #define wDebug(...) \
{ \ { \
if (wDebugFlag & DEBUG_DEBUG) { \ if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define wTrace(...) \ #define wTrace(...) \
{ \ { \
if (wDebugFlag & DEBUG_TRACE) { \ if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define WAL_HEAD_VER 0 #define WAL_HEAD_VER 0

1
include/os/os.h
View File

@ -44,6 +44,7 @@ extern "C" {
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/utsname.h> #include <sys/utsname.h>
#include <sys/param.h>
#include <unistd.h> #include <unistd.h>
#include <wchar.h> #include <wchar.h>
#include <wctype.h> #include <wctype.h>

9
include/os/osFile.h
View File

@ -22,6 +22,7 @@ extern "C" {
#include "osSocket.h" #include "osSocket.h"
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID #define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID
@ -31,6 +32,8 @@ extern "C" {
#define fstat FSTAT_FUNC_TAOS_FORBID #define fstat FSTAT_FUNC_TAOS_FORBID
#define close CLOSE_FUNC_TAOS_FORBID #define close CLOSE_FUNC_TAOS_FORBID
#define fclose FCLOSE_FUNC_TAOS_FORBID #define fclose FCLOSE_FUNC_TAOS_FORBID
#define fsync FSYNC_FUNC_TAOS_FORBID
// #define fflush FFLUSH_FUNC_TAOS_FORBID
#endif #endif
#ifndef PATH_MAX #ifndef PATH_MAX
@ -47,13 +50,13 @@ typedef struct TdFile *TdFilePtr;
#define TD_FILE_TEXT 0x0020 #define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080 #define TD_FILE_EXCL 0x0080
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
#define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_EXIST_OK 0x1
#define TD_FILE_ACCESS_READ_OK 0x2 #define TD_FILE_ACCESS_READ_OK 0x2
#define TD_FILE_ACCESS_WRITE_OK 0x4 #define TD_FILE_ACCESS_WRITE_OK 0x4
bool taosCheckAccessFile(const char *pathname, int mode); bool taosCheckAccessFile(const char *pathname, int mode);
int32_t taosLockFile(TdFilePtr pFile); int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile);
@ -80,10 +83,10 @@ int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName); int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to); int64_t taosCopyFile(const char *from, const char *to);
int32_t taosRemoveFile(const char *path);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);

16
include/os/osSocket.h
View File

@ -16,6 +16,14 @@
#ifndef _TD_OS_SOCKET_H_ #ifndef _TD_OS_SOCKET_H_
#define _TD_OS_SOCKET_H_ #define _TD_OS_SOCKET_H_
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID
// #define accept ACCEPT_FUNC_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#include "winsock2.h" #include "winsock2.h"
#include <WS2tcpip.h> #include <WS2tcpip.h>
@ -49,9 +57,6 @@ int32_t taosCloseSocket(SocketFd fd);
void taosShutDownSocketRD(SOCKET fd); void taosShutDownSocketRD(SOCKET fd);
void taosShutDownSocketWR(SOCKET fd); void taosShutDownSocketWR(SOCKET fd);
int32_t taosSetNonblocking(SOCKET sock, int32_t on); int32_t taosSetNonblocking(SOCKET sock, int32_t on);
void taosIgnSIGPIPE();
void taosBlockSIGPIPE();
void taosSetMaskSIGPIPE();
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
@ -81,10 +86,13 @@ SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosKeepTcpAlive(SOCKET sockFd);
int32_t taosGetFqdn(char *); void taosBlockSIGPIPE();
uint32_t taosGetIpv4FromFqdn(const char *); uint32_t taosGetIpv4FromFqdn(const char *);
int32_t taosGetFqdn(char *);
void tinet_ntoa(char *ipstr, uint32_t ip); void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr); uint32_t ip2uint(const char *const ip_addr);
void taosIgnSIGPIPE();
void taosSetMaskSIGPIPE();
#ifdef __cplusplus #ifdef __cplusplus
} }

8
include/os/osSysinfo.h
View File

@ -38,15 +38,13 @@ int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores); int32_t taosGetCpuCores(float *numOfCores);
int32_t taosGetCpuUsage(float *cpu_system, float *cpu_engine); int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine);
int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetTotalMemory(int64_t *totalKB);
int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetProcMemory(int64_t *usedKB);
int32_t taosGetSysMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB);
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosReadProcIO(int64_t *rchars, int64_t *wchars); int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetProcIO(float *readKB, float *writeKB); int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
int32_t taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes);
int32_t taosGetBandSpeed(float *bandSpeedKb);
int32_t taosSystem(const char *cmd); int32_t taosSystem(const char *cmd);
void taosKillSystem(); void taosKillSystem();

7
include/util/taoserror.h
View File

@ -84,6 +84,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x0120)
#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x0121) #define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x0121)
#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x0122) #define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x0122)
#define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0123)
//client //client
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
@ -457,6 +458,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502)
//parser //parser
#define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600)
#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) #define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601)
#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) #define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602)
#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603)
@ -471,6 +473,11 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) #define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C)
#define TSDB_CODE_PAR_TAGS_NOT_MATCHED TAOS_DEF_ERROR_CODE(0, 0x260D) #define TSDB_CODE_PAR_TAGS_NOT_MATCHED TAOS_DEF_ERROR_CODE(0, 0x260D)
#define TSDB_CODE_PAR_INVALID_TAG_NAME TAOS_DEF_ERROR_CODE(0, 0x260E) #define TSDB_CODE_PAR_INVALID_TAG_NAME TAOS_DEF_ERROR_CODE(0, 0x260E)
#define TSDB_CODE_PAR_INCOMPLETE_SQL TAOS_DEF_ERROR_CODE(0, 0x260F)
#define TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2610)
#define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611)
#define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612)
#define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613)
#ifdef __cplusplus #ifdef __cplusplus
} }

67
include/util/tcache.h
View File

@ -40,55 +40,9 @@ typedef struct SCacheStatis {
int64_t refreshCount; int64_t refreshCount;
} SCacheStatis; } SCacheStatis;
struct STrashElem; typedef struct SCacheObj SCacheObj;
typedef struct SCacheIter SCacheIter;
typedef struct SCacheDataNode { typedef struct STrashElem STrashElem;
uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // life duration when this element should be remove from cache
uint64_t expireTime; // expire time
uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keySize : 15; // max key size: 32kb
bool inTrashcan : 1; // denote if it is in trash or not
uint32_t size; // allocated size for current SCacheDataNode
T_REF_DECLARE()
char *key;
char data[];
} SCacheDataNode;
typedef struct STrashElem {
struct STrashElem *prev;
struct STrashElem *next;
SCacheDataNode *pData;
} STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
STrashElem *pTrash;
char *name;
SCacheStatis statistics;
SHashObj *pHashTable;
__cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
bool extendLifespan; // auto extend life span when one item is accessed.
int64_t checkTick; // tick used to record the check times of the refresh threads
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
} SCacheObj;
/** /**
* initialize the cache object * initialize the cache object
@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data);
* @param data * @param data
* @return * @return
*/ */
void *taosCacheTransfer(SCacheObj *pCacheObj, void **data); void *taosCacheTransferData(SCacheObj *pCacheObj, void **data);
/** /**
* remove data in cache, the data will not be removed immediately. * remove data in cache, the data will not be removed immediately.
@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
*/ */
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove); void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
/**
*
* @param pCacheObj
* @return
*/
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
/** /**
* move all data node into trash, clear node in trash can if it is not referenced by any clients * move all data node into trash, clear node in trash can if it is not referenced by any clients
* @param handle * @param handle
@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
*/ */
void taosStopCacheRefreshWorker(); void taosStopCacheRefreshWorker();
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
bool taosCacheIterNext(SCacheIter* pIter);
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
void taosCacheDestroyIter(SCacheIter* pIter);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

9
include/util/tdef.h
View File

@ -195,6 +195,7 @@ typedef enum ELogicConditionType {
#define TSDB_NODE_NAME_LEN 64 #define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string #define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_CGROUP_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65 #define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
@ -206,12 +207,12 @@ typedef enum ELogicConditionType {
#define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_FUNC_MAX_RETRIEVE 1024
#define TSDB_INDEX_NAME_LEN 32
#define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20)
#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_COL_NAME_LEN 65 #define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
@ -441,6 +442,8 @@ typedef struct {
int32_t primary; int32_t primary;
} SDiskCfg; } SDiskCfg;
#define TMQ_SEPARATOR ':'
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

105
include/util/thash.h
View File

@ -28,12 +28,8 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *);
#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_KEY_ALREADY_EXISTS (-2)
#define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NODE_EXIST(code) (code == -2)
/** /**
* murmur hash algorithm * murmur hash algorithm
@ -54,49 +50,15 @@ uint32_t taosIntHash_32(const char *key, uint32_t len);
uint32_t taosIntHash_64(const char *key, uint32_t len); uint32_t taosIntHash_64(const char *key, uint32_t len);
_hash_fn_t taosGetDefaultHashFunction(int32_t type); _hash_fn_t taosGetDefaultHashFunction(int32_t type);
_equal_fn_t taosGetDefaultEqualFunction(int32_t type); _equal_fn_t taosGetDefaultEqualFunction(int32_t type);
typedef struct SHashNode {
struct SHashNode *next;
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t count; // reference count
int8_t removed; // flag to indicate removed
char data[];
} SHashNode;
#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
typedef enum SHashLockTypeE { typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0, HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1, HASH_ENTRY_LOCK = 1,
} SHashLockTypeE; } SHashLockTypeE;
typedef struct SHashEntry { typedef struct SHashNode SHashNode;
int32_t num; // number of elements in current entry typedef struct SHashObj SHashObj;
SRWLatch latch; // entry latch
SHashNode *next;
} SHashEntry;
typedef struct SHashObj {
SHashEntry **hashList;
uint32_t capacity; // number of slots
uint32_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_hash_free_fn_t freeFp; // hash node free callback function
_equal_fn_t equalFp; // equal function
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry
} SHashObj;
/** /**
* init the hash table * init the hash table
@ -126,8 +88,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
*/ */
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
/** /**
* return the payload data with the specified key * return the payload data with the specified key
* *
@ -146,17 +106,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
* @param destBuf * @param destBuf
* @return * @return
*/ */
void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
/** /**
* Clone the result to interval allocated buffer *
* @param pHashObj * @param pHashObj
* @param key * @param key
* @param keyLen * @param keyLen
* @param destBuf * @param destBuf
* @param size
* @return * @return
*/ */
void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size);
/** /**
* remove item with the specified key * remove item with the specified key
@ -207,37 +168,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
*/ */
void taosHashCancelIterate(SHashObj *pHashObj, void *p); void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/** /**
* Get the corresponding key information for a given data in hash table * Get the corresponding key information for a given data in hash table
* @param data * @param data
* @return * @param keyLen
*/ * @return
int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); */
void *taosHashGetKey(void *data, size_t* keyLen);
/**
* Get the corresponding key information for a given data in hash table, using memcpy
* @param data
* @param dst
* @return
*/
static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) {
if (NULL == data || NULL == dst) {
return -1;
}
SHashNode *node = GET_HASH_PNODE(data);
void *key = GET_HASH_NODE_KEY(node);
memcpy(dst, key, node->keyLen);
return 0;
}
/**
* Get the corresponding data length for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetDataLen(void *data);
/** /**
* return the payload data with the specified key(reference number added) * return the payload data with the specified key(reference number added)
@ -258,8 +195,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
*/ */
void taosHashRelease(SHashObj *pHashObj, void *p); void taosHashRelease(SHashObj *pHashObj, void *p);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

56
include/util/tlog.h
View File

@ -22,9 +22,29 @@
extern "C" { extern "C" {
#endif #endif
typedef enum {
DEBUG_FATAL = 1,
DEBUG_ERROR = 1,
DEBUG_WARN = 2,
DEBUG_INFO = 2,
DEBUG_DEBUG = 4,
DEBUG_TRACE = 8,
DEBUG_DUMP = 16,
DEBUG_SCREEN = 64,
DEBUG_FILE = 128
} ELogLevel;
typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content);
extern bool tsLogEmbedded;
extern bool tsAsyncLog; extern bool tsAsyncLog;
extern int32_t tsNumOfLogLines; extern int32_t tsNumOfLogLines;
extern int32_t tsLogKeepDays; extern int32_t tsLogKeepDays;
extern LogFp tsLogFp;
extern int64_t tsNumOfErrorLogs;
extern int64_t tsNumOfInfoLogs;
extern int64_t tsNumOfDebugLogs;
extern int64_t tsNumOfTraceLogs;
extern int32_t dDebugFlag; extern int32_t dDebugFlag;
extern int32_t vDebugFlag; extern int32_t vDebugFlag;
extern int32_t mDebugFlag; extern int32_t mDebugFlag;
@ -40,45 +60,33 @@ extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag; extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag; extern int32_t fsDebugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U
#define DEBUG_INFO DEBUG_WARN
#define DEBUG_DEBUG 4U
#define DEBUG_TRACE 8U
#define DEBUG_DUMP 16U
#define DEBUG_SCREEN 64U
#define DEBUG_FILE 128U
int32_t taosInitLog(const char *logName, int32_t maxFiles); int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog(); void taosCloseLog();
void taosResetLog(); void taosResetLog();
void taosSetAllDebugFlag(int32_t flag); void taosSetAllDebugFlag(int32_t flag);
void taosDumpData(uint8_t *msg, int32_t len); void taosDumpData(uint8_t *msg, int32_t len);
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__ #ifdef __GNUC__
__attribute__((format(printf, 3, 4))) __attribute__((format(printf, 4, 5)))
#endif #endif
; ;
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...)
#ifdef __GNUC__ #ifdef __GNUC__
__attribute__((format(printf, 3, 4))) __attribute__((format(printf, 4, 5)))
#endif #endif
; ;
extern int8_t tscEmbeddedInUtil; #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", DEBUG_INFO, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", DEBUG_DEBUG, uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", DEBUG_TRACE, uDebugFlag, __VA_ARGS__); }}
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); }
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/util/tpagedbuf.h
View File

@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis {
* @param handle * @param handle
* @return * @return
*/ */
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir);
/** /**
* *

1
include/util/version.h
View File

@ -23,7 +23,6 @@ extern "C" {
extern char version[]; extern char version[];
extern char compatible_version[]; extern char compatible_version[];
extern char gitinfo[]; extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[]; extern char buildinfo[];
#ifdef __cplusplus #ifdef __cplusplus

30
source/client/inc/clientInt.h
View File

@ -20,12 +20,12 @@
extern "C" { extern "C" {
#endif #endif
#include "tcommon.h"
#include "parser.h" #include "parser.h"
#include "query.h" #include "query.h"
#include "taos.h" #include "taos.h"
#include "tcommon.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
#include "tmsg.h" #include "tmsg.h"
@ -47,12 +47,12 @@ extern "C" {
typedef struct SAppInstInfo SAppInstInfo; typedef struct SAppInstInfo SAppInstInfo;
typedef struct SHbConnInfo { typedef struct {
void* param; void* param;
SClientHbReq* req; SClientHbReq* req;
} SHbConnInfo; } SHbConnInfo;
typedef struct SAppHbMgr { typedef struct {
char* key; char* key;
// statistics // statistics
int32_t reportCnt; int32_t reportCnt;
@ -68,11 +68,11 @@ typedef struct SAppHbMgr {
SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo> SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo>
} SAppHbMgr; } SAppHbMgr;
typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
typedef struct SClientHbMgr { typedef struct {
int8_t inited; int8_t inited;
// ctl // ctl
int8_t threadStop; int8_t threadStop;
@ -108,13 +108,13 @@ typedef struct SHeartBeatInfo {
} SHeartBeatInfo; } SHeartBeatInfo;
struct SAppInstInfo { struct SAppInstInfo {
int64_t numOfConns; int64_t numOfConns;
SCorEpSet mgmtEp; SCorEpSet mgmtEp;
SInstanceSummary summary; SInstanceSummary summary;
SList* pConnList; // STscObj linked list SList* pConnList; // STscObj linked list
int64_t clusterId; int64_t clusterId;
void* pTransporter; void* pTransporter;
struct SAppHbMgr* pAppHbMgr; SAppHbMgr* pAppHbMgr;
}; };
typedef struct SAppInfo { typedef struct SAppInfo {
@ -141,10 +141,6 @@ typedef struct STscObj {
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
} STscObj; } STscObj;
typedef struct SMqConsumer {
STscObj* pTscObj;
} SMqConsumer;
typedef struct SReqResultInfo { typedef struct SReqResultInfo {
const char* pRspMsg; const char* pRspMsg;
const char* pData; const char* pData;

14
source/client/inc/clientLog.h
View File

@ -22,13 +22,13 @@ extern "C" {
#include "tlog.h" #include "tlog.h"
#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0)
#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus #ifdef __cplusplus
} }

50
source/client/src/clientHb.c
View File

@ -23,7 +23,7 @@ static SClientHbMgr clientHbMgr = {0};
static int32_t hbCreateThread(); static int32_t hbCreateThread();
static void hbStopThread(); static void hbStopThread();
static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; }
static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0; int32_t code = 0;
@ -104,7 +104,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == info) { if (NULL == info) {
tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType);
@ -163,7 +163,7 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp *pRs
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbMqAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) { static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) {
static int32_t emptyRspNum = 0; static int32_t emptyRspNum = 0;
if (code != 0) { if (code != 0) {
tfree(param); tfree(param);
@ -227,7 +227,11 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
db->numOfTable = htonl(db->numOfTable); db->numOfTable = htonl(db->numOfTable);
} }
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs}; SKv kv = {
.key = HEARTBEAT_KEY_DBINFO,
.valueLen = sizeof(SDbVgVersion) * dbNum,
.value = dbs,
};
tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen);
@ -257,7 +261,11 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
stb->tversion = htons(stb->tversion); stb->tversion = htons(stb->tversion);
} }
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; SKv kv = {
.key = HEARTBEAT_KEY_STBINFO,
.valueLen = sizeof(SSTableMetaVersion) * stbNum,
.value = stbs,
};
tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen);
@ -289,7 +297,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) {} int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; }
void hbMgrInitMqHbHandle() { void hbMgrInitMqHbHandle() {
clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle;
@ -397,7 +405,7 @@ static void *hbThreadFunc(void *param) {
free(buf); free(buf);
break; break;
} }
pInfo->fp = hbMqAsyncCallBack; pInfo->fp = hbAsyncCallBack;
pInfo->msgInfo.pData = buf; pInfo->msgInfo.pData = buf;
pInfo->msgInfo.len = tlen; pInfo->msgInfo.len = tlen;
pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->msgType = TDMT_MND_HEARTBEAT;
@ -427,11 +435,11 @@ static int32_t hbCreateThread() {
pthread_attr_init(&thAttr); pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) { // if (pthread_create(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); // terrno = TAOS_SYSTEM_ERROR(errno);
return -1; // return -1;
} // }
pthread_attr_destroy(&thAttr); // pthread_attr_destroy(&thAttr);
return 0; return 0;
} }
@ -449,7 +457,6 @@ static void hbStopThread() {
} }
SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
/*return NULL;*/
hbMgrInit(); hbMgrInit();
SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr)); SAppHbMgr *pAppHbMgr = malloc(sizeof(SAppHbMgr));
if (pAppHbMgr == NULL) { if (pAppHbMgr == NULL) {
@ -474,7 +481,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
free(pAppHbMgr); free(pAppHbMgr);
return NULL; return NULL;
} }
pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq);
// init getInfoFunc // init getInfoFunc
pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
@ -507,7 +515,6 @@ void appHbMgrCleanup(void) {
} }
int hbMgrInit() { int hbMgrInit() {
/*return 0;*/
// init once // init once
int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1);
if (old == 1) return 0; if (old == 1) return 0;
@ -525,7 +532,7 @@ int hbMgrInit() {
} }
void hbMgrCleanUp() { void hbMgrCleanUp() {
return; #if 0
hbStopThread(); hbStopThread();
// destroy all appHbMgr // destroy all appHbMgr
@ -538,6 +545,7 @@ void hbMgrCleanUp() {
pthread_mutex_unlock(&clientHbMgr.lock); pthread_mutex_unlock(&clientHbMgr.lock);
clientHbMgr.appHbMgrs = NULL; clientHbMgr.appHbMgrs = NULL;
#endif
} }
int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {
@ -564,9 +572,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *
} }
int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) {
/*return 0;*/ SClientHbKey connKey = {
SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; .connId = connId,
SHbConnInfo info = {0}; .hbType = HEARTBEAT_TYPE_QUERY,
};
SHbConnInfo info = {0};
switch (hbType) { switch (hbType) {
case HEARTBEAT_TYPE_QUERY: { case HEARTBEAT_TYPE_QUERY: {
@ -587,7 +597,6 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int32_t connId, int64_t clusterId, int3
} }
void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
/*return;*/
int32_t code = 0; int32_t code = 0;
code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey));
@ -599,7 +608,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen,
int32_t valueLen) { int32_t valueLen) {
return 0;
// find req by connection id // find req by connection id
SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
ASSERT(pReq != NULL); ASSERT(pReq != NULL);

2
source/client/src/clientImpl.c
View File

@ -4,8 +4,8 @@
#include "parser.h" #include "parser.h"
#include "planner.h" #include "planner.h"
#include "scheduler.h" #include "scheduler.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"

382
source/client/src/tmq.c
View File

@ -20,8 +20,8 @@
#include "parser.h" #include "parser.h"
#include "planner.h" #include "planner.h"
#include "scheduler.h" #include "scheduler.h"
#include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
@ -46,7 +46,7 @@ struct tmq_topic_vgroup_list_t {
struct tmq_conf_t { struct tmq_conf_t {
char clientId[256]; char clientId[256];
char groupId[256]; char groupId[TSDB_CGROUP_LEN];
int8_t auto_commit; int8_t auto_commit;
int8_t resetOffset; int8_t resetOffset;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
@ -56,9 +56,10 @@ struct tmq_conf_t {
struct tmq_t { struct tmq_t {
// conf // conf
char groupId[256]; char groupId[TSDB_CGROUP_LEN];
char clientId[256]; char clientId[256];
int8_t autoCommit; int8_t autoCommit;
int8_t inWaiting;
int64_t consumerId; int64_t consumerId;
int32_t epoch; int32_t epoch;
int32_t resetOffsetCfg; int32_t resetOffsetCfg;
@ -66,9 +67,12 @@ struct tmq_t {
STscObj* pTscObj; STscObj* pTscObj;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
int32_t nextTopicIdx; int32_t nextTopicIdx;
int32_t waitingRequest;
int32_t readyRequest;
SArray* clientTopics; // SArray<SMqClientTopic> SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of tmq_message_t STaosQueue* mqueue; // queue of tmq_message_t
STaosQall* qall; STaosQall* qall;
tsem_t rspSem;
// stat // stat
int64_t pollCnt; int64_t pollCnt;
}; };
@ -117,10 +121,12 @@ typedef struct {
} SMqAskEpCbParam; } SMqAskEpCbParam;
typedef struct { typedef struct {
tmq_t* tmq; tmq_t* tmq;
SMqClientVg* pVg; SMqClientVg* pVg;
int32_t epoch; int32_t epoch;
tsem_t rspSem; tsem_t rspSem;
tmq_message_t** msg;
int32_t sync;
} SMqPollCbParam; } SMqPollCbParam;
typedef struct { typedef struct {
@ -196,6 +202,26 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) {
return 0; return 0;
} }
void tmqClearUnhandleMsg(tmq_t* tmq) {
tmq_message_t* msg;
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
taosReadAllQitems(tmq->mqueue, tmq->qall);
while (1) {
taosGetQitem(tmq->qall, (void**)&msg);
if (msg)
taosFreeQitem(msg);
else
break;
}
}
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code; pParam->rspErr = code;
@ -205,16 +231,11 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
if (pParam->tmq->commit_cb) { if (pParam->tmq->commit_cb) {
pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); pParam->tmq->commit_cb(pParam->tmq, pParam->rspErr, NULL, NULL);
}
if (!pParam->async)
tsem_post(&pParam->rspSem);
else {
tsem_destroy(&pParam->rspSem);
free(param);
} }
if (!pParam->async) tsem_post(&pParam->rspSem);
return 0; return 0;
} }
@ -240,9 +261,12 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
return NULL; return NULL;
} }
pTmq->pTscObj = (STscObj*)conn; pTmq->pTscObj = (STscObj*)conn;
pTmq->inWaiting = 0;
pTmq->status = 0; pTmq->status = 0;
pTmq->pollCnt = 0; pTmq->pollCnt = 0;
pTmq->epoch = 0; pTmq->epoch = 0;
pTmq->waitingRequest = 0;
pTmq->readyRequest = 0;
// set conf // set conf
strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId); strcpy(pTmq->groupId, conf->groupId);
@ -250,6 +274,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
pTmq->commit_cb = conf->commit_cb; pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset; pTmq->resetOffsetCfg = conf->resetOffset;
tsem_init(&pTmq->rspSem, 0, 0);
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
@ -315,6 +341,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
} }
pParam->tmq = tmq; pParam->tmq = tmq;
tsem_init(&pParam->rspSem, 0, 0); tsem_init(&pParam->rspSem, 0, 0);
pParam->async = async;
pRequest->body.requestMsg = (SDataBuf){ pRequest->body.requestMsg = (SDataBuf){
.pData = buf, .pData = buf,
@ -335,6 +362,9 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
resp = pParam->rspErr; resp = pParam->rspErr;
} }
tsem_destroy(&pParam->rspSem);
free(pParam);
if (pArray) { if (pArray) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
} }
@ -576,7 +606,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0; if (tmq_message == NULL) return 0;
SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
return pRsp->skipLogNum; return pRsp->skipLogNum;
} }
@ -625,56 +655,74 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
} }
int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
printf("recv poll\n"); /*printf("recv poll\n");*/
SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqPollCbParam* pParam = (SMqPollCbParam*)param;
SMqClientVg* pVg = pParam->pVg; SMqClientVg* pVg = pParam->pVg;
tmq_t* tmq = pParam->tmq; tmq_t* tmq = pParam->tmq;
if (code != 0) { if (code != 0) {
printf("msg discard\n"); printf("msg discard\n");
if (pParam->epoch == tmq->epoch) { goto WRITE_QUEUE_FAIL;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
return 0;
} }
int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
int32_t tmqEpoch = atomic_load_32(&tmq->epoch); int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) { if (msgEpoch < tmqEpoch) {
tsem_post(&tmq->rspSem);
printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
return 0; return 0;
} }
if (msgEpoch != tmqEpoch) { if (msgEpoch != tmqEpoch) {
printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch); printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
} else {
atomic_sub_fetch_32(&tmq->waitingRequest, 1);
} }
#if 0
if (pParam->sync == 1) {
/**pParam->msg = malloc(sizeof(tmq_message_t));*/
*pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
if (*pParam->msg) {
memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
}
taosWriteQitem(tmq->mqueue, *pParam->msg);
tsem_post(&pParam->rspSem);
return 0;
}
tsem_post(&pParam->rspSem);
return -1;
}
#endif
/*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/ /*SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp));*/
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) { if (pRsp == NULL) {
printf("fail\n"); goto WRITE_QUEUE_FAIL;
return -1;
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->consumeRsp.numOfTopics == 0) { if (pRsp->consumeRsp.numOfTopics == 0) {
printf("no data\n"); /*printf("no data\n");*/
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
taosFreeQitem(pRsp); taosFreeQitem(pRsp);
return 0; goto WRITE_QUEUE_FAIL;
} }
pRsp->extra = pParam->pVg; pRsp->extra = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp); taosWriteQitem(tmq->mqueue, pRsp);
printf("poll in queue\n"); atomic_add_fetch_32(&tmq->readyRequest, 1);
/*pParam->rspMsg = (tmq_message_t*)pRsp;*/ tsem_post(&tmq->rspSem);
/*pVg->currentOffset = pRsp->consumeRsp.rspOffset;*/
/*printf("rsp offset: %ld\n", rsp.rspOffset);*/
/*printf("-----msg begin----\n");*/
/*printf("\n-----msg end------\n");*/
return 0; return 0;
WRITE_QUEUE_FAIL:
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
tsem_post(&tmq->rspSem);
return code;
} }
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
@ -711,81 +759,94 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = pParam->tmq; tmq_t* tmq = pParam->tmq;
if (code != 0) { if (code != 0) {
printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync); printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
if (pParam->sync) { goto END;
tsem_post(&pParam->rspSem);
}
return 0;
} }
tscDebug("tmq ask ep cb called");
// tmq's epoch is monotomically increase,
// so it's safe to discard any old epoch msg.
// epoch will only increase when received newer epoch ep msg
SMqRspHead* head = pMsg->pData;
int32_t epoch = atomic_load_32(&tmq->epoch);
if (head->epoch <= epoch) {
goto END;
}
if (pParam->sync) { if (pParam->sync) {
SMqRspHead* head = pMsg->pData;
SMqCMGetSubEpRsp rsp; SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
int32_t epoch = atomic_load_32(&tmq->epoch); if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
if (head->epoch > epoch && tmqUpdateEp(tmq, head->epoch, &rsp)) {
atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY);
} }
tsem_post(&pParam->rspSem);
tDeleteSMqCMGetSubEpRsp(&rsp); tDeleteSMqCMGetSubEpRsp(&rsp);
} else { } else {
tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; code = -1;
goto END;
} }
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp);
taosWriteQitem(tmq->mqueue, pRsp); taosWriteQitem(tmq->mqueue, pRsp);
tsem_post(&tmq->rspSem);
} }
return 0;
END:
if (pParam->sync) {
tsem_post(&pParam->rspSem);
}
return code;
} }
int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tmqAskEp(tmq_t* tmq, bool sync) {
printf("ask ep sync %d\n", sync);
int32_t tlen = sizeof(SMqCMGetSubEpReq); int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen); SMqCMGetSubEpReq* req = malloc(tlen);
if (buf == NULL) { if (req == NULL) {
tscError("failed to malloc get subscribe ep buf"); tscError("failed to malloc get subscribe ep buf");
goto END; return -1;
} }
buf->consumerId = htobe64(tmq->consumerId); req->consumerId = htobe64(tmq->consumerId);
buf->epoch = htonl(tmq->epoch); req->epoch = htonl(tmq->epoch);
strcpy(buf->cgroup, tmq->groupId); strcpy(req->cgroup, tmq->groupId);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
if (pRequest == NULL) {
tscError("failed to malloc subscribe ep request");
goto END;
}
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam)); SMqAskEpCbParam* pParam = malloc(sizeof(SMqAskEpCbParam));
if (pParam == NULL) { if (pParam == NULL) {
tscError("failed to malloc subscribe param"); tscError("failed to malloc subscribe param");
goto END; free(req);
return -1;
} }
pParam->tmq = tmq; pParam->tmq = tmq;
pParam->sync = sync; pParam->sync = sync;
tsem_init(&pParam->rspSem, 0, 0); tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
tsem_destroy(&pParam->rspSem);
free(pParam);
free(req);
return -1;
}
sendInfo->msgInfo = (SDataBuf){
.pData = req,
.len = tlen,
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0; sendInfo->requestObjRefId = 0;
sendInfo->param = pParam; sendInfo->param = pParam;
sendInfo->fp = tmqAskEpCb; sendInfo->fp = tmqAskEpCb;
sendInfo->msgType = TDMT_MND_GET_SUB_EP;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0; int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
END:
if (sync) tsem_wait(&pParam->rspSem); if (sync) tsem_wait(&pParam->rspSem);
return 0; return 0;
} }
@ -804,6 +865,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
if (pVg->vgId == pOffset->vgId) { if (pVg->vgId == pOffset->vgId) {
pVg->currentOffset = pOffset->offset; pVg->currentOffset = pOffset->offset;
tmqClearUnhandleMsg(tmq);
return TMQ_RESP_ERR__SUCCESS; return TMQ_RESP_ERR__SUCCESS;
} }
} }
@ -812,7 +874,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
return TMQ_RESP_ERR__FAIL; return TMQ_RESP_ERR__FAIL;
} }
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset; int64_t reqOffset;
if (pVg->currentOffset >= 0) { if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset; reqOffset = pVg->currentOffset;
@ -832,7 +894,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
strcpy(pReq->topic, pTopic->topicName); strcpy(pReq->topic, pTopic->topicName);
strcpy(pReq->cgroup, tmq->groupId); strcpy(pReq->cgroup, tmq->groupId);
pReq->blockingTime = blocking_time; pReq->blockingTime = blockingTime;
pReq->consumerId = tmq->consumerId; pReq->consumerId = tmq->consumerId;
pReq->epoch = tmq->epoch; pReq->epoch = tmq->epoch;
pReq->currentOffset = reqOffset; pReq->currentOffset = reqOffset;
@ -842,28 +904,76 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClie
return pReq; return pReq;
} }
void tmqClearUnhandleMsg(tmq_t* tmq) { tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
tmq_message_t* msg; tmq_message_t* msg = NULL;
while (1) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
taosGetQitem(tmq->qall, (void**)&msg); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (msg) for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
taosFreeQitem(msg); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
else int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
break; /*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
} /*continue;*/
/*}*/
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
return NULL;
}
taosReadAllQitems(tmq->mqueue, tmq->qall); SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
while (1) { if (pParam == NULL) {
taosGetQitem(tmq->qall, (void**)&msg); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
if (msg) // TODO: out of mem
taosFreeQitem(msg); return NULL;
else }
break; pParam->tmq = tmq;
pParam->pVg = pVg;
pParam->epoch = tmq->epoch;
pParam->sync = 1;
pParam->msg = &msg;
tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
return NULL;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = pParam;
sendInfo->fp = tmqPollCb;
sendInfo->msgType = TDMT_VND_CONSUME;
int64_t transporterId = 0;
/*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
tmq->pollCnt++;
tsem_wait(&pParam->rspSem);
tmq_message_t* nmsg = NULL;
while (1) {
taosReadQitem(tmq->mqueue, (void**)&nmsg);
if (nmsg == NULL) continue;
while (nmsg->head.mqMsgType != TMQ_MSG_TYPE__POLL_RSP) {
taosReadQitem(tmq->mqueue, (void**)&nmsg);
}
return nmsg;
}
}
} }
return NULL;
} }
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
printf("call poll\n"); /*printf("call poll\n");*/
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
@ -875,32 +985,44 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) { if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem tsem_post(&tmq->rspSem);
return -1; return -1;
} }
SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); SMqPollCbParam* pParam = malloc(sizeof(SMqPollCbParam));
if (param == NULL) { if (pParam == NULL) {
free(pReq);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem tsem_post(&tmq->rspSem);
return -1; return -1;
} }
param->tmq = tmq; pParam->tmq = tmq;
param->pVg = pVg; pParam->pVg = pVg;
param->epoch = tmq->epoch; pParam->epoch = tmq->epoch;
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pParam->sync = 0;
pRequest->body.requestMsg = (SDataBuf){
SMsgSendInfo* sendInfo = malloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) {
free(pReq);
free(pParam);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
return -1;
}
sendInfo->msgInfo = (SDataBuf){
.pData = pReq, .pData = pReq,
.len = sizeof(SMqConsumeReq), .len = sizeof(SMqConsumeReq),
.handle = NULL, .handle = NULL,
}; };
sendInfo->requestId = generateRequestId();
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0; sendInfo->requestObjRefId = 0;
sendInfo->param = param; sendInfo->param = pParam;
sendInfo->fp = tmqPollCb; sendInfo->fp = tmqPollCb;
sendInfo->msgType = TDMT_VND_CONSUME;
int64_t transporterId = 0; int64_t transporterId = 0;
printf("send poll\n"); /*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++; pVg->pollCnt++;
tmq->pollCnt++; tmq->pollCnt++;
@ -912,7 +1034,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
// return // return
int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) {
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch); /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) {
tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp);
tmqClearUnhandleMsg(tmq); tmqClearUnhandleMsg(tmq);
@ -931,23 +1053,26 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
tmq_message_t* rspMsg = NULL; tmq_message_t* rspMsg = NULL;
taosGetQitem(tmq->qall, (void**)&rspMsg); taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) { if (rspMsg == NULL) {
break; taosReadAllQitems(tmq->mqueue, tmq->qall);
taosGetQitem(tmq->qall, (void**)&rspMsg);
if (rspMsg == NULL) return NULL;
} }
if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) {
printf("handle poll rsp %d\n", rspMsg->head.mqMsgType); atomic_sub_fetch_32(&tmq->readyRequest, 1);
/*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/
if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) {
printf("epoch match\n"); /*printf("epoch match\n");*/
SMqClientVg* pVg = rspMsg->extra; SMqClientVg* pVg = rspMsg->extra;
pVg->currentOffset = rspMsg->consumeRsp.rspOffset; pVg->currentOffset = rspMsg->consumeRsp.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
return rspMsg; return rspMsg;
} else { } else {
printf("epoch mismatch\n"); /*printf("epoch mismatch\n");*/
taosFreeQitem(rspMsg); taosFreeQitem(rspMsg);
} }
} else { } else {
printf("handle ep rsp %d\n", rspMsg->head.mqMsgType); /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
bool reset = false; bool reset = false;
tmqHandleRes(tmq, rspMsg, &reset); tmqHandleRes(tmq, rspMsg, &reset);
taosFreeQitem(rspMsg); taosFreeQitem(rspMsg);
@ -957,36 +1082,57 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
} }
} }
} }
return NULL; }
tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL;
int64_t startTime = taosGetTimestampMs();
int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
while (1) {
rspMsg = tmqSyncPollImpl(tmq, blocking_time);
if (rspMsg && rspMsg->consumeRsp.numOfTopics) {
return rspMsg;
}
if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
return NULL;
}
} else
return NULL;
}
} }
tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL; tmq_message_t* rspMsg;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();
// TODO: put into another thread or delayed queue // TODO: put into another thread or delayed queue
int64_t status = atomic_load_64(&tmq->status); int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT); tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
taosGetQitem(tmq->qall, (void**)&rspMsg); rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspMsg == NULL) { if (rspMsg) {
taosReadAllQitems(tmq->mqueue, tmq->qall); return rspMsg;
} }
tmqHandleAllRsp(tmq, blocking_time, false);
tmqPollImpl(tmq, blocking_time);
while (1) { while (1) {
/*printf("cycle\n");*/ /*printf("cycle\n");*/
taosReadAllQitems(tmq->mqueue, tmq->qall); tmqPollImpl(tmq, blocking_time);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, true);
tsem_wait(&tmq->rspSem);
rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspMsg) { if (rspMsg) {
return rspMsg; return rspMsg;
} }
if (blocking_time != 0) { if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs(); int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) { if (endTime - startTime > blocking_time) {
printf("normal exit\n");
return NULL; return NULL;
} }
} }
@ -1127,6 +1273,7 @@ void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return; if (tmq_message == NULL) return;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp; SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
tDeleteSMqConsumeRsp(pRsp); tDeleteSMqConsumeRsp(pRsp);
/*free(tmq_message);*/
taosFreeQitem(tmq_message); taosFreeQitem(tmq_message);
} }
@ -1138,6 +1285,7 @@ const char* tmq_err2str(tmq_resp_err_t err) {
} }
return "fail"; return "fail";
} }
#if 0 #if 0
tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
tmq_t* pTmq = malloc(sizeof(tmq_t)); tmq_t* pTmq = malloc(sizeof(tmq_t));

23
source/client/test/clientTests.cpp
View File

@ -53,7 +53,7 @@ TEST(testCase, driverInit_Test) {
// taos_init(); // taos_init();
} }
#if 1 #if 0
TEST(testCase, connect_Test) { TEST(testCase, connect_Test) {
// taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); // taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg");
@ -571,15 +571,15 @@ TEST(testCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); // TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes)); // printf("error in create db, reason:%s\n", taos_errstr(pRes));
} // }
taos_free_result(pRes); // taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes); taos_free_result(pRes);
#if 0
pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
@ -592,7 +592,7 @@ TEST(testCase, projection_query_tables) {
} }
taos_free_result(pRes); taos_free_result(pRes);
for(int32_t i = 0; i < 10000000; ++i) { for(int32_t i = 0; i < 10000; ++i) {
char sql[512] = {0}; char sql[512] = {0};
sprintf(sql, "insert into tu values(now+%da, %d)", i, i); sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
TAOS_RES* p = taos_query(pConn, sql); TAOS_RES* p = taos_query(pConn, sql);
@ -602,8 +602,9 @@ TEST(testCase, projection_query_tables) {
taos_free_result(p); taos_free_result(p);
} }
#endif
pRes = taos_query(pConn, "select * from tu"); pRes = taos_query(pConn, "select count(ts) from tu");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); taos_free_result(pRes);
@ -623,8 +624,8 @@ TEST(testCase, projection_query_tables) {
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
} }
#if 0
#if 0
TEST(testCase, projection_query_stables) { TEST(testCase, projection_query_stables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);

6
source/common/src/tep.c → source/common/src/tdatablock.c
View File

@ -14,7 +14,7 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tep.h" #include "tdatablock.h"
#include "tcompare.h" #include "tcompare.h"
#include "tglobal.h" #include "tglobal.h"
@ -239,7 +239,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
return numOfRow1 + numOfRow2; return numOfRow1 + numOfRow2;
} }
size_t colDataGetNumOfCols(const SSDataBlock* pBlock) { size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
ASSERT(pBlock); ASSERT(pBlock);
size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0; size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0;
@ -247,7 +247,7 @@ size_t colDataGetNumOfCols(const SSDataBlock* pBlock) {
return pBlock->info.numOfCols; return pBlock->info.numOfCols;
} }
size_t colDataGetNumOfRows(const SSDataBlock* pBlock) { size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) {
return pBlock->info.rows; return pBlock->info.rows;
} }

76
source/common/src/tglobal.c
View File

@ -17,7 +17,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tcompare.h" #include "tcompare.h"
#include "tconfig.h" #include "tconfig.h"
#include "tep.h" #include "tdatablock.h"
#include "tlog.h" #include "tlog.h"
SConfig *tsCfg = NULL; SConfig *tsCfg = NULL;
@ -48,10 +48,11 @@ bool tsPrintAuth = 0;
// monitor // monitor
bool tsEnableMonitor = 1; bool tsEnableMonitor = 1;
int32_t tsMonitorInterval = 5; int32_t tsMonitorInterval = 30;
char tsMonitorFqdn[TSDB_FQDN_LEN] = {0}; char tsMonitorFqdn[TSDB_FQDN_LEN] = {0};
uint16_t tsMonitorPort = 6043; uint16_t tsMonitorPort = 6043;
int32_t tsMonitorMaxLogs = 100; int32_t tsMonitorMaxLogs = 100;
bool tsMonitorComp = false;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -155,21 +156,40 @@ static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t prima
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
} }
static void taosSetTfsCfg(SConfig *pCfg) { static int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
if (pItem == NULL) return; memset(tsDataDir, 0, PATH_MAX);
int32_t size = taosArrayGetSize(pItem->array); int32_t size = taosArrayGetSize(pItem->array);
if (size <= 0) { if (size <= 0) {
tsDiskCfgNum = 1; tsDiskCfgNum = 1;
taosAddDataDir(0, pItem->str, 0, 1); taosAddDataDir(0, pItem->str, 0, 1);
tstrncpy(tsDataDir, pItem->str, PATH_MAX);
if (taosMkDir(tsDataDir) != 0) {
uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} else { } else {
tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS;
for (int32_t index = 0; index < tsDiskCfgNum; ++index) { for (int32_t index = 0; index < tsDiskCfgNum; ++index) {
SDiskCfg *pCfg = taosArrayGet(pItem->array, index); SDiskCfg *pCfg = taosArrayGet(pItem->array, index);
memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg));
if (pCfg->level == 0 && pCfg->primary == 1) {
tstrncpy(tsDataDir, pCfg->dir, PATH_MAX);
}
if (taosMkDir(pCfg->dir) != 0) {
uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr());
return -1;
}
} }
} }
if (tsDataDir[0] == 0) {
uError("datadir not set");
return -1;
}
return 0;
} }
struct SConfig *taosGetCfg() { struct SConfig *taosGetCfg() {
@ -294,7 +314,6 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; if (cfgAddString(pCfg, "version", version, 1) != 0) return -1;
if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1;
if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1; if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1;
if (cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1) != 0) return -1;
if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1; if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1;
return 0; return 0;
} }
@ -327,6 +346,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, 0) != 0) return -1;
return 0; return 0;
} }
@ -358,7 +378,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
} }
static void taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN);
tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
@ -375,9 +395,13 @@ static void taosSetClientCfg(SConfig *pCfg) {
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); taosExpandDir(tsTempDir, tsTempDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create tempDir:%s since %s", tsTempDir, terrstr());
return -1;
}
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
@ -392,6 +416,8 @@ static void taosSetClientCfg(SConfig *pCfg) {
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
return 0;
} }
static void taosSetSystemCfg(SConfig *pCfg) { static void taosSetSystemCfg(SConfig *pCfg) {
@ -411,11 +437,8 @@ static void taosSetSystemCfg(SConfig *pCfg) {
tsVersion = 30000000; tsVersion = 30000000;
} }
static void taosSetServerCfg(SConfig *pCfg) { static int32_t taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
taosExpandDir(tsDataDir, tsDataDir, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
@ -440,10 +463,13 @@ static void taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);
tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32;
tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32;
tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval;
if (tsQueryBufferSize >= 0) { if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
} }
return 0;
} }
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
@ -454,10 +480,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (pCfg == NULL) return -1; if (pCfg == NULL) return -1;
if (tsc) { if (tsc) {
tscEmbeddedInUtil = 0; tsLogEmbedded = 0;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) return -1;
} else { } else {
tscEmbeddedInUtil = 1; tsLogEmbedded = 1;
if (taosAddClientLogCfg(pCfg) != 0) return -1; if (taosAddClientLogCfg(pCfg) != 0) return -1;
if (taosAddServerLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1;
} }
@ -504,8 +530,8 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
tsCfg = cfgInit(); tsCfg = cfgInit();
if (tsc) { if (tsc) {
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
} else { } else {
if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddServerCfg(tsCfg) != 0) return -1; if (taosAddServerCfg(tsCfg) != 0) return -1;
@ -528,24 +554,14 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
} }
if (tsc) { if (tsc) {
taosSetClientCfg(tsCfg); if (taosSetClientCfg(tsCfg)) return -1;
} else { } else {
taosSetClientCfg(tsCfg); if (taosSetClientCfg(tsCfg)) return -1;
taosSetServerCfg(tsCfg); if (taosSetServerCfg(tsCfg)) return -1;
taosSetTfsCfg(tsCfg); if (taosSetTfsCfg(tsCfg) != 0) return -1;
} }
taosSetSystemCfg(tsCfg); taosSetSystemCfg(tsCfg);
if (taosMkDir(tsTempDir) != 0) {
uError("failed to create dir:%s since %s", tsTempDir, terrstr());
return -1;
}
if (!tsc && taosMkDir(tsDataDir) != 0) {
uError("failed to create dir:%s since %s", tsDataDir, terrstr());
return -1;
}
cfgDumpCfg(tsCfg, tsc, false); cfgDumpCfg(tsCfg, tsc, false);
return 0; return 0;
} }

55
source/common/src/tmsg.c
View File

@ -1007,11 +1007,32 @@ int32_t tDeserializeSMCreateDropQSBNodeReq(void *buf, int32_t bufLen, SMCreateQn
} }
int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) {
return tSerializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->fqdn) < 0) return -1;
if (tEncodeI32(&encoder, pReq->port) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
} }
int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) { int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) {
return tDeserializeSMCreateDropQSBNodeReq(buf, bufLen, (SMCreateQnodeReq *)pReq); SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->fqdn) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->port) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
} }
int32_t tSerializeSMCreateDropMnodeReq(void *buf, int32_t bufLen, SMCreateMnodeReq *pReq) { int32_t tSerializeSMCreateDropMnodeReq(void *buf, int32_t bufLen, SMCreateMnodeReq *pReq) {
@ -2602,5 +2623,35 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); } void tFreeSSchedulerHbRsp(SSchedulerHbRsp *pRsp) { taosArrayDestroy(pRsp->taskStatus); }
int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += tEncodeTSma(buf, &pReq->tSma);
return tlen;
}
void *tDeserializeSVCreateTSmaReq(void *buf, SVCreateTSmaReq *pReq) {
buf = taosDecodeFixedI64(buf, &(pReq->ver));
if ((buf = tDecodeTSma(buf, &pReq->tSma)) == NULL) {
tdDestroyTSma(&pReq->tSma);
}
return buf;
}
int32_t tSerializeSVDropTSmaReq(void **buf, SVDropTSmaReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeString(buf, pReq->indexName);
return tlen;
}
void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
buf = taosDecodeFixedI64(buf, &(pReq->ver));
buf = taosDecodeStringTo(buf, pReq->indexName);
return buf;
}

4
source/common/src/ttszip.c
View File

@ -46,7 +46,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
} }
if (!autoDelete) { if (!autoDelete) {
remove(pTSBuf->path); taosRemoveFile(pTSBuf->path);
} }
if (NULL == allocResForTSBuf(pTSBuf)) { if (NULL == allocResForTSBuf(pTSBuf)) {
@ -178,7 +178,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
if (pTSBuf->autoDelete) { if (pTSBuf->autoDelete) {
// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path);
remove(pTSBuf->path); taosRemoveFile(pTSBuf->path);
} else { } else {
// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
} }

7
source/common/test/commonTests.cpp
View File

@ -8,7 +8,8 @@
#pragma GCC diagnostic ignored "-Wsign-compare" #pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h" #include "os.h"
#include "tep.h" #include "tcommon.h"
#include "tdatablock.h"
#include "tcommon.h" #include "tcommon.h"
#include "taos.h" #include "taos.h"
#include "tvariant.h" #include "tvariant.h"
@ -159,8 +160,8 @@ TEST(testCase, Datablock_test) {
printf("binary column length:%d\n", *(int32_t*) p1->pData); printf("binary column length:%d\n", *(int32_t*) p1->pData);
ASSERT_EQ(colDataGetNumOfCols(b), 2); ASSERT_EQ(blockDataGetNumOfCols(b), 2);
ASSERT_EQ(colDataGetNumOfRows(b), 40); ASSERT_EQ(blockDataGetNumOfRows(b), 40);
char* pData = colDataGetData(p1, 3); char* pData = colDataGetData(p1, 3);
printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));

1
source/dnode/mgmt/daemon/src/dmnUtil.c
View File

@ -30,6 +30,5 @@ void dmnPrintVersion() {
#endif #endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo); printf("gitinfo: %s\n", gitinfo);
printf("gitinfoI: %s\n", gitinfoOfInternal);
printf("builuInfo: %s\n", buildinfo); printf("builuInfo: %s\n", buildinfo);
} }

14
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -105,9 +105,19 @@ typedef struct {
} SBnodeMgmt; } SBnodeMgmt;
typedef struct { typedef struct {
int32_t openVnodes;
int32_t totalVnodes;
int32_t masterNum;
int64_t numOfSelectReqs;
int64_t numOfInsertReqs;
int64_t numOfInsertSuccessReqs;
int64_t numOfBatchInsertReqs;
int64_t numOfBatchInsertSuccessReqs;
} SVnodesStat;
typedef struct {
SVnodesStat stat;
SHashObj *hash; SHashObj *hash;
int32_t openVnodes;
int32_t totalVnodes;
SRWLatch latch; SRWLatch latch;
SQWorkerPool queryPool; SQWorkerPool queryPool;
SFWorkerPool fetchPool; SFWorkerPool fetchPool;

20
source/dnode/mgmt/impl/inc/dndInt.h
View File

@ -23,9 +23,11 @@ extern "C" {
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "monitor.h"
#include "tcache.h" #include "tcache.h"
#include "tcrc32c.h" #include "tcrc32c.h"
#include "tep.h" #include "tdatablock.h"
#include "tglobal.h"
#include "thash.h" #include "thash.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "tlog.h" #include "tlog.h"
@ -35,8 +37,6 @@ extern "C" {
#include "tthread.h" #include "tthread.h"
#include "ttime.h" #include "ttime.h"
#include "tworker.h" #include "tworker.h"
#include "tglobal.h"
#include "monitor.h"
#include "dnode.h" #include "dnode.h"
@ -47,14 +47,12 @@ extern "C" {
#include "vnode.h" #include "vnode.h"
#include "tfs.h" #include "tfs.h"
extern int32_t dDebugFlag; #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }} #define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", 255, __VA_ARGS__); }} #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", 255, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;

1
source/dnode/mgmt/impl/inc/dndMnode.h
View File

@ -34,7 +34,6 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo); SMonGrantInfo *pGrantInfo);
int8_t dndIsMnode(SDnode *pDnode);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -298,7 +298,7 @@ int32_t dndInit() {
return -1; return -1;
} }
SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn}; SMonCfg monCfg = {.maxLogs = tsMonitorMaxLogs, .port = tsMonitorPort, .server = tsMonitorFqdn, .comp = tsMonitorComp};
if (monInit(&monCfg) != 0) { if (monInit(&monCfg) != 0) {
dError("failed to init monitor since %s", terrstr()); dError("failed to init monitor since %s", terrstr());
dndCleanup(); dndCleanup();

35
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -477,6 +477,8 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
pInfo->dnode_id = dndGetDnodeId(pDnode); pInfo->dnode_id = dndGetDnodeId(pDnode);
tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN); tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
pInfo->cluster_id = dndGetClusterId(pDnode);
pInfo->protocol = 1;
} }
static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
@ -487,24 +489,21 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
taosGetSysMemory(&pInfo->mem_system); taosGetSysMemory(&pInfo->mem_system);
pInfo->mem_total = tsTotalMemoryKB; pInfo->mem_total = tsTotalMemoryKB;
pInfo->disk_engine = 0; pInfo->disk_engine = 0;
pInfo->disk_used = tsDataSpace.size.used / (1024 * 1024 * 1024.0); pInfo->disk_used = tsDataSpace.size.used;
pInfo->disk_total = tsDataSpace.size.avail / (1024 * 1024 * 1024.0); pInfo->disk_total = tsDataSpace.size.total;
taosGetCardInfo(NULL, &pInfo->net_in, &pInfo->net_out); taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
taosGetProcIO(&pInfo->io_read, &pInfo->io_write); taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
pInfo->io_read_disk = 0;
pInfo->io_write_disk = 0; SVnodesStat *pStat = &pDnode->vmgmt.stat;
pInfo->req_select = 0; pInfo->req_select = pStat->numOfSelectReqs;
pInfo->req_select_rate = 0; pInfo->req_insert = pStat->numOfInsertReqs;
pInfo->req_insert = 0; pInfo->req_insert_success = pStat->numOfInsertSuccessReqs;
pInfo->req_insert_success = 0; pInfo->req_insert_batch = pStat->numOfBatchInsertReqs;
pInfo->req_insert_rate = 0; pInfo->req_insert_batch_success = pStat->numOfBatchInsertSuccessReqs;
pInfo->req_insert_batch = 0; pInfo->errors = tsNumOfErrorLogs;
pInfo->req_insert_batch_success = 0; pInfo->vnodes_num = pStat->totalVnodes;
pInfo->req_insert_batch_rate = 0; pInfo->masters = pStat->masterNum;
pInfo->errors = 0; pInfo->has_mnode = pDnode->mmgmt.deployed;
pInfo->vnodes_num = 0;
pInfo->masters = 0;
pInfo->has_mnode = dndIsMnode(pDnode);
} }
static void dndSendMonitorReport(SDnode *pDnode) { static void dndSendMonitorReport(SDnode *pDnode) {

7
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -640,10 +640,3 @@ int32_t dndGetMnodeMonitorInfo(SDnode *pDnode, SMonClusterInfo *pClusterInfo, SM
dndReleaseMnode(pDnode, pMnode); dndReleaseMnode(pDnode, pMnode);
return code; return code;
} }
int8_t dndIsMnode(SDnode *pDnode) {
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL) return 0;
dndReleaseMnode(pDnode, pMnode);
return 1;
}

42
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -17,6 +17,7 @@
#include "dndVnodes.h" #include "dndVnodes.h"
#include "dndMgmt.h" #include "dndMgmt.h"
#include "dndTransport.h" #include "dndTransport.h"
#include "sync.h"
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
@ -85,7 +86,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
int32_t refCount = 0; int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch); taosRLockLatch(&pMgmt->latch);
taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
if (pVnode == NULL) { if (pVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else { } else {
@ -381,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
char stepDesc[TSDB_STEP_DESC_LEN] = {0}; char stepDesc[TSDB_STEP_DESC_LEN] = {0};
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
pMgmt->openVnodes, pMgmt->totalVnodes); pMgmt->stat.openVnodes, pMgmt->stat.totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc); dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
@ -395,7 +396,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
pThread->opened++; pThread->opened++;
} }
atomic_add_fetch_32(&pMgmt->openVnodes, 1); atomic_add_fetch_32(&pMgmt->stat.openVnodes, 1);
} }
dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
@ -421,7 +422,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
return -1; return -1;
} }
pMgmt->totalVnodes = numOfVnodes; pMgmt->stat.totalVnodes = numOfVnodes;
int32_t threadNum = tsNumOfCores; int32_t threadNum = tsNumOfCores;
#if 1 #if 1
@ -469,11 +470,11 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
free(threads); free(threads);
free(pCfgs); free(pCfgs);
if (pMgmt->openVnodes != pMgmt->totalVnodes) { if (pMgmt->stat.openVnodes != pMgmt->stat.totalVnodes) {
dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); dError("there are total vnodes:%d, opened:%d", pMgmt->stat.totalVnodes, pMgmt->stat.openVnodes);
return -1; return -1;
} else { } else {
dInfo("total vnodes:%d open successfully", pMgmt->totalVnodes); dInfo("total vnodes:%d open successfully", pMgmt->stat.totalVnodes);
return 0; return 0;
} }
} }
@ -979,11 +980,18 @@ void dndCleanupVnodes(SDnode *pDnode) {
void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) { void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SVnodesStat *pStat = &pMgmt->stat;
int32_t totalVnodes = 0;
int32_t masterNum = 0;
int64_t numOfSelectReqs = 0;
int64_t numOfInsertReqs = 0;
int64_t numOfInsertSuccessReqs = 0;
int64_t numOfBatchInsertReqs = 0;
int64_t numOfBatchInsertSuccessReqs = 0;
taosRLockLatch(&pMgmt->latch); taosRLockLatch(&pMgmt->latch);
int32_t v = 0; void *pIter = taosHashIterate(pMgmt->hash, NULL);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue; if (ppVnode == NULL || *ppVnode == NULL) continue;
@ -993,8 +1001,24 @@ void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
vnodeGetLoad(pVnode->pImpl, &vload); vnodeGetLoad(pVnode->pImpl, &vload);
taosArrayPush(pLoads, &vload); taosArrayPush(pLoads, &vload);
numOfSelectReqs += vload.numOfSelectReqs;
numOfInsertReqs += vload.numOfInsertReqs;
numOfInsertSuccessReqs += vload.numOfInsertSuccessReqs;
numOfBatchInsertReqs += vload.numOfBatchInsertReqs;
numOfBatchInsertSuccessReqs += vload.numOfBatchInsertSuccessReqs;
totalVnodes++;
if (vload.role == TAOS_SYNC_STATE_LEADER) masterNum++;
pIter = taosHashIterate(pMgmt->hash, pIter); pIter = taosHashIterate(pMgmt->hash, pIter);
} }
taosRUnLockLatch(&pMgmt->latch); taosRUnLockLatch(&pMgmt->latch);
pStat->totalVnodes = totalVnodes;
pStat->masterNum = masterNum;
pStat->numOfSelectReqs = numOfSelectReqs;
pStat->numOfInsertReqs = numOfInsertReqs;
pStat->numOfInsertSuccessReqs = numOfInsertSuccessReqs;
pStat->numOfBatchInsertReqs = numOfBatchInsertReqs;
pStat->numOfBatchInsertSuccessReqs = numOfBatchInsertSuccessReqs;
} }

2
source/dnode/mgmt/impl/test/sut/src/client.cpp
View File

@ -14,7 +14,7 @@
*/ */
#include "sut.h" #include "sut.h"
#include "tep.h" #include "tdatablock.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent; TestClient* client = (TestClient*)parent;

2
source/dnode/mgmt/impl/test/sut/src/sut.cpp
View File

@ -28,7 +28,7 @@ void Testbase::InitLog(const char* path) {
wDebugFlag = 0; wDebugFlag = 0;
sDebugFlag = 0; sDebugFlag = 0;
tsdbDebugFlag = 0; tsdbDebugFlag = 0;
tscEmbeddedInUtil = 1; tsLogEmbedded = 1;
tsAsyncLog = 0; tsAsyncLog = 0;
taosRemoveDir(path); taosRemoveDir(path);

62
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -34,46 +34,6 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t mDebugFlag;
// mnode log function
#define mFatal(...) \
{ \
if (mDebugFlag & DEBUG_FATAL) { \
taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \
} \
}
#define mError(...) \
{ \
if (mDebugFlag & DEBUG_ERROR) { \
taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \
} \
}
#define mWarn(...) \
{ \
if (mDebugFlag & DEBUG_WARN) { \
taosPrintLog("MND WARN ", 255, __VA_ARGS__); \
} \
}
#define mInfo(...) \
{ \
if (mDebugFlag & DEBUG_INFO) { \
taosPrintLog("MND ", 255, __VA_ARGS__); \
} \
}
#define mDebug(...) \
{ \
if (mDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
#define mTrace(...) \
{ \
if (mDebugFlag & DEBUG_TRACE) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
typedef enum { typedef enum {
MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_START = 0,
MND_AUTH_ACCT_USER, MND_AUTH_ACCT_USER,
@ -393,6 +353,23 @@ typedef struct {
char payload[]; char payload[];
} SShowObj; } SShowObj;
typedef struct {
int64_t id;
int8_t type;
int8_t replica;
int16_t numOfColumns;
int32_t rowSize;
int32_t numOfRows;
int32_t numOfReads;
int32_t payloadLen;
void* pIter;
SMnode* pMnode;
char db[TSDB_DB_FNAME_LEN];
int16_t offset[TSDB_MAX_COLUMNS];
int32_t bytes[TSDB_MAX_COLUMNS];
char payload[];
} SSysTableRetrieveObj;
typedef struct { typedef struct {
int32_t vgId; // -1 for unassigned int32_t vgId; // -1 for unassigned
int32_t status; int32_t status;
@ -634,7 +611,7 @@ typedef struct {
int64_t consumerId; int64_t consumerId;
int64_t connId; int64_t connId;
SRWLatch lock; SRWLatch lock;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
SArray* currentTopics; // SArray<char*> SArray* currentTopics; // SArray<char*>
SArray* recentRemovedTopics; // SArray<char*> SArray* recentRemovedTopics; // SArray<char*>
int32_t epoch; int32_t epoch;
@ -701,6 +678,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
return buf; return buf;
} }
typedef struct {
} SStreamScheduler;
typedef struct SMnodeMsg { typedef struct SMnodeMsg {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];

17
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -20,7 +20,7 @@
#include "sdb.h" #include "sdb.h"
#include "tcache.h" #include "tcache.h"
#include "tep.h" #include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tqueue.h" #include "tqueue.h"
#include "ttime.h" #include "ttime.h"
@ -31,6 +31,13 @@
extern "C" { extern "C" {
#endif #endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode);
@ -85,6 +92,11 @@ typedef struct {
ESyncState state; ESyncState state;
} SSyncMgmt; } SSyncMgmt;
typedef struct {
int64_t expireTimeMS;
int64_t timeseriesAllowed;
} SGrantInfo;
typedef struct SMnode { typedef struct SMnode {
int32_t dnodeId; int32_t dnodeId;
int64_t clusterId; int64_t clusterId;
@ -105,6 +117,7 @@ typedef struct SMnode {
STelemMgmt telemMgmt; STelemMgmt telemMgmt;
SSyncMgmt syncMgmt; SSyncMgmt syncMgmt;
SHashObj *infosMeta; SHashObj *infosMeta;
SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX]; MndMsgFp msgFp[TDMT_MAX];
SendReqToDnodeFp sendReqToDnodeFp; SendReqToDnodeFp sendReqToDnodeFp;
SendReqToMnodeFp sendReqToMnodeFp; SendReqToMnodeFp sendReqToMnodeFp;
@ -120,7 +133,7 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
uint64_t mndGenerateUid(char *name, int32_t len); uint64_t mndGenerateUid(char *name, int32_t len);
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/dnode/mnode/impl/inc/mndProfile.h
View File

@ -24,6 +24,7 @@ extern "C" {
int32_t mndInitProfile(SMnode *pMnode); int32_t mndInitProfile(SMnode *pMnode);
void mndCleanupProfile(SMnode *pMnode); void mndCleanupProfile(SMnode *pMnode);
int32_t mndGetNumOfConnections(SMnode *pMnode);
#ifdef __cplusplus #ifdef __cplusplus
} }

34
source/dnode/mnode/impl/inc/mndScheduler.h Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_SCHEDULER_H_
#define _TD_MND_SCHEDULER_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitScheduler(SMnode* pMnode);
void mndCleanupScheduler(SMnode* pMnode);
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_SCHEDULER_H_ */

13
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -340,9 +340,16 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
pVgroup->compStorage = pVload->compStorage; pVgroup->compStorage = pVload->compStorage;
pVgroup->pointsWritten = pVload->pointsWritten; pVgroup->pointsWritten = pVload->pointsWritten;
} }
bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
if (pVgroup->vnodeGid[vg].role != pVload->role) {
roleChanged = true;
}
pVgroup->vnodeGid[vg].role = pVload->role; pVgroup->vnodeGid[vg].role = pVload->role;
} }
if (roleChanged) {
// notify scheduler role has changed
}
} }
mndReleaseVgroup(pMnode, pVgroup); mndReleaseVgroup(pMnode, pVgroup);
@ -631,13 +638,13 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp
pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name"); strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = pShow->bytes[cols]; pSchema[cols].bytes = pShow->bytes[cols];
cols++; cols++;
pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY; pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "value"); strcpy(pSchema[cols].name, "value");
pSchema[cols].bytes = pShow->bytes[cols]; pSchema[cols].bytes = pShow->bytes[cols];
cols++; cols++;
@ -823,4 +830,4 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, i
static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
} }

4
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -99,11 +99,15 @@ void mndUpdateMnodeRole(SMnode *pMnode) {
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break; if (pIter == NULL) break;
ESyncState lastRole = pObj->role;
if (pObj->id == 1) { if (pObj->id == 1) {
pObj->role = TAOS_SYNC_STATE_LEADER; pObj->role = TAOS_SYNC_STATE_LEADER;
} else { } else {
pObj->role = TAOS_SYNC_STATE_CANDIDATE; pObj->role = TAOS_SYNC_STATE_CANDIDATE;
} }
if (pObj->role != lastRole) {
pObj->roleTime = taosGetTimestampMs();
}
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
} }

70
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
static void mndFreeConn(SConnObj *pConn); static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter);
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq);
static int32_t mndProcessConnectReq(SMnodeMsg *pReq); static int32_t mndProcessConnectReq(SMnodeMsg *pReq);
@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
taosCacheRelease(pMgmt->cache, (void **)&pConn, false); taosCacheRelease(pMgmt->cache, (void **)&pConn, false);
} }
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SConnObj* pConn = NULL;
bool hasNext = taosCacheIterNext(pIter);
*pConn = NULL; if (hasNext) {
size_t dataLen = 0;
pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); pConn = taosCacheIterGetData(pIter, &dataLen);
if (pIter == NULL) return NULL; } else {
taosCacheDestroyIter(pIter);
SCacheDataNode **pNode = pIter;
if (pNode == NULL || *pNode == NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
return NULL;
} }
*pConn = (SConnObj *)((*pNode)->data); return pConn;
return pIter;
} }
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; if (pIter != NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); taosCacheDestroyIter(pIter);
}
} }
static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0; int32_t rspLen = 0;
mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) { if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv); taosArrayPush(hbRsp.info, &kv1);
} }
break; break;
} }
@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0; int32_t rspLen = 0;
mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) { if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv); taosArrayPush(hbRsp.info, &kv1);
} }
break; break;
} }
@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
} }
pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbName, mndShowStr(pShow->type)); strcpy(pMeta->tbName, mndShowStr(pShow->type));
@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
char *pWrite; char *pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 6]; char ipStr[TSDB_IPv4ADDR_LEN + 6];
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) break; if (pConn == NULL) break;
cols = 0; cols = 0;
@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
void *pIter; void *pIter;
char str[TSDB_IPv4ADDR_LEN + 6] = {0}; char str[TSDB_IPv4ADDR_LEN + 6] = {0};
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) { while (numOfRows < rows) {
pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) { if (pConn == NULL) {
pShow->pIter = pIter; pShow->pIter = NULL;
break; break;
} }
if (numOfRows + pConn->numOfQueries >= rows) { if (numOfRows + pConn->numOfQueries >= rows) {
mndCancelGetNextConn(pMnode, pIter); taosCacheDestroyIter(pShow->pIter);
pShow->pIter = NULL;
break; break;
} }
pShow->pIter = pIter;
for (int32_t i = 0; i < pConn->numOfQueries; ++i) { for (int32_t i = 0; i < pConn->numOfQueries; ++i) {
SQueryDesc *pDesc = pConn->pQueries + i; SQueryDesc *pDesc = pConn->pQueries + i;
cols = 0; cols = 0;
@ -913,6 +919,12 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
} }
static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; if (pIter != NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); taosCacheDestroyIter(pIter);
}
} }
int32_t mndGetNumOfConnections(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
return taosCacheGetNumOfObj(pMgmt->cache);
}

78
source/dnode/mnode/impl/src/mndScheduler.c Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndScheduler.h"
#include "mndConsumer.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndSubscribe.h"
#include "mndTopic.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tcompare.h"
#include "tname.h"
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan);
SArray* pAray = NULL;
SArray* unassignedVg = pSub->unassignedVg;
ASSERT(pSub->vgNum == 0);
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
return -1;
}
SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0);
int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) {
return -1;
}
SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
void* pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pTopic->dbUid) {
sdbRelease(pSdb, pVgroup);
continue;
}
pSub->vgNum++;
plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
consumerEp.consumerId = -1;
consumerEp.epSet = plan->execNode.epset;
consumerEp.vgId = plan->execNode.nodeId;
int32_t msgLen;
int32_t code = qSubPlanToString(plan, &consumerEp.qmsg, &msgLen);
taosArrayPush(unassignedVg, &consumerEp);
}
return 0;
}

102
source/dnode/mnode/impl/src/mndShow.c
View File

@ -25,6 +25,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
static int32_t mndProcessShowReq(SMnodeMsg *pReq); static int32_t mndProcessShowReq(SMnodeMsg *pReq);
static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq); static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq);
static bool mndCheckRetrieveFinished(SShowObj *pShow); static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) { int32_t mndInitShow(SMnode *pMnode) {
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
@ -38,6 +39,7 @@ int32_t mndInitShow(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowReq);
mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveReq);
mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
return 0; return 0;
} }
@ -261,6 +263,106 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t mndProcessRetrieveSysTableReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
SShowMgmt *pMgmt = &pMnode->showMgmt;
int32_t rowsToRead = 0;
int32_t size = 0;
int32_t rowsRead = 0;
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
SShowObj* pShow = NULL;
if (retrieveReq.showId == 0) {
SShowReq req = {0};
req.type = retrieveReq.type;
strncpy(req.db, retrieveReq.db, tListLen(req.db));
pShow = mndCreateShowObj(pMnode, &req);
if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr());
return -1;
}
} else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1;
}
}
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) {
mndReleaseShowObj((SShowObj*) pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
mDebug("show:0x%" PRIx64 ", start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads,
pShow->numOfRows, mndShowStr(pShow->type));
if (mndCheckRetrieveFinished((SShowObj*) pShow)) {
mDebug("show:0x%" PRIx64 ", read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads,
pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows;
}
if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsToRead = pShow->numOfRows - pShow->numOfReads;
}
/* return no more than 100 tables in one round trip */
if (rowsToRead > SHOW_STEP_SIZE) rowsToRead = SHOW_STEP_SIZE;
/*
* the actual number of table may be larger than the value of pShow->numOfRows, if a query is
* issued during a continuous create table operation. Therefore, rowToRead may be less than 0.
*/
if (rowsToRead < 0) rowsToRead = 0;
size = pShow->rowSize * rowsToRead;
size += SHOW_STEP_SIZE;
SRetrieveTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
mndReleaseShowObj((SShowObj*) pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
// if free flag is set, client wants to clean the resources
if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
rowsRead = (*retrieveFp)(pReq, (SShowObj*) pShow, pRsp->data, rowsToRead);
}
mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead);
pRsp->numOfRows = htonl(rowsRead);
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision
pReq->pCont = pRsp;
pReq->contLen = size;
if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj((SShowObj*) pShow, true);
} else {
mDebug("show:0x%" PRIx64 ", retrieve not completed yet", pShow->id);
mndReleaseShowObj((SShowObj*) pShow, false);
}
return TSDB_CODE_SUCCESS;
}
char *mndShowStr(int32_t showType) { char *mndShowStr(int32_t showType) {
switch (showType) { switch (showType) {
case TSDB_MGMT_TABLE_ACCT: case TSDB_MGMT_TABLE_ACCT:

63
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -20,6 +20,7 @@
#include "mndDnode.h" #include "mndDnode.h"
#include "mndMnode.h" #include "mndMnode.h"
#include "mndOffset.h" #include "mndOffset.h"
#include "mndScheduler.h"
#include "mndShow.h" #include "mndShow.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndTopic.h" #include "mndTopic.h"
@ -39,7 +40,7 @@ enum {
MQ_SUBSCRIBE_STATUS__DELETED, MQ_SUBSCRIBE_STATUS__DELETED,
}; };
static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName); static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
@ -87,22 +88,25 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
char *key = mndMakeSubscribeKey(cgroup, pTopic->name); char key[TSDB_SUBSCRIBE_KEY_LEN];
if (key == NULL) { mndMakeSubscribeKey(key, cgroup, pTopic->name);
terrno = TSDB_CODE_OUT_OF_MEMORY; strcpy(pSub->key, key);
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
tDeleteSMqSubscribeObj(pSub); tDeleteSMqSubscribeObj(pSub);
free(pSub); free(pSub);
return NULL; return NULL;
} }
strcpy(pSub->key, key);
free(key);
#if 0
if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) { if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSMqSubscribeObj(pSub); tDeleteSMqSubscribeObj(pSub);
free(pSub); free(pSub);
return NULL; return NULL;
} }
#endif
// TODO: disable alter subscribed table // TODO: disable alter subscribed table
return pSub; return pSub;
} }
@ -325,15 +329,14 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
return 0; return 0;
} }
static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) {
int32_t i = 0; int32_t i = 0;
while (key[i] != ':') { while (key[i] != TMQ_SEPARATOR) {
i++; i++;
} }
key[i] = 0; memcpy(topic, key, i - 1);
*cgroup = strdup(key); topic[i] = 0;
key[i] = ':'; strcpy(cgroup, &key[i + 1]);
*topic = strdup(&key[i + 1]);
return 0; return 0;
} }
@ -369,8 +372,9 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
// get all topics of that topic // get all topics of that topic
int32_t sz = taosArrayGetSize(pConsumer->currentTopics); int32_t sz = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
char *key = mndMakeSubscribeKey(pConsumer->cgroup, topic); char key[TSDB_SUBSCRIBE_KEY_LEN];
mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId);
} }
@ -386,8 +390,9 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
} }
int32_t sz = taosArrayGetSize(rebSubs); int32_t sz = taosArrayGetSize(rebSubs);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = taosArrayGetP(rebSubs, i); char *topic = taosArrayGetP(rebSubs, i);
char *key = mndMakeSubscribeKey(pConsumer->cgroup, topic); char key[TSDB_SUBSCRIBE_KEY_LEN];
mndMakeSubscribeKey(key, pConsumer->cgroup, topic);
SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key);
if (status == MQ_CONSUMER_STATUS__INIT) { if (status == MQ_CONSUMER_STATUS__INIT) {
taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId);
@ -520,9 +525,9 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
if (pConsumerEp->oldConsumerId == -1) { if (pConsumerEp->oldConsumerId == -1) {
char *topic; char topic[TSDB_TOPIC_FNAME_LEN];
char *cgroup; char cgroup[TSDB_CGROUP_LEN];
mndSplitSubscribeKey(pSub->key, &topic, &cgroup); mndSplitSubscribeKey(pSub->key, topic, cgroup);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic, mInfo("mq set conn: assign vgroup %d of topic %s to consumer %ld", pConsumerEp->vgId, topic,
@ -530,8 +535,6 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
free(topic);
free(cgroup);
} else { } else {
mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId, mInfo("mq rebalance: assign vgroup %d, from consumer %ld to consumer %ld", pConsumerEp->vgId,
pConsumerEp->oldConsumerId, pConsumerEp->consumerId); pConsumerEp->oldConsumerId, pConsumerEp->consumerId);
@ -759,6 +762,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) {
} }
#endif #endif
#if 0
static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) { static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
@ -804,6 +808,7 @@ static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SM
/*qDestroyQueryDag(pDag);*/ /*qDestroyQueryDag(pDag);*/
return 0; return 0;
} }
#endif
static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
const SMqConsumerEp *pConsumerEp) { const SMqConsumerEp *pConsumerEp) {
@ -949,23 +954,19 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc
return 0; return 0;
} }
static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName) { static int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName) {
char *key = malloc(TSDB_SHOW_SUBQUERY_LEN);
if (key == NULL) {
return NULL;
}
int32_t tlen = strlen(cgroup); int32_t tlen = strlen(cgroup);
memcpy(key, cgroup, tlen); memcpy(key, cgroup, tlen);
key[tlen] = ':'; key[tlen] = TMQ_SEPARATOR;
strcpy(key + tlen + 1, topicName); strcpy(key + tlen + 1, topicName);
return key; return 0;
} }
SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const char *topicName) { SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *cgroup, const char *topicName) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char *key = mndMakeSubscribeKey(cgroup, topicName); char key[TSDB_SUBSCRIBE_KEY_LEN];
mndMakeSubscribeKey(key, cgroup, topicName);
SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); SMqSubscribeObj *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key);
free(key);
if (pSub == NULL) { if (pSub == NULL) {
terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; terrno = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
} }

6
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -18,15 +18,15 @@
#include "mndCluster.h" #include "mndCluster.h"
#include "mndSync.h" #include "mndSync.h"
#include "tbuffer.h" #include "tbuffer.h"
#include "tjson.h"
#include "thttp.h" #include "thttp.h"
#include "tjson.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_PORT 80 #define TELEMETRY_PORT 80
static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) { static void mndBuildRuntimeInfo(SMnode* pMnode, SJson* pJson) {
SMnodeLoad load = {0}; SMnodeLoad load = {0};
if (mndGetLoad(pMnode, &load) != 0) return; mndGetLoad(pMnode, &load);
tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode); tjsonAddDoubleToObject(pJson, "numOfDnode", load.numOfDnode);
tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode); tjsonAddDoubleToObject(pJson, "numOfMnode", load.numOfMnode);
@ -87,7 +87,7 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
taosWLockLatch(&pMgmt->lock); taosWLockLatch(&pMgmt->lock);
char* pCont = mndBuildTelemetryReport(pMnode); char* pCont = mndBuildTelemetryReport(pMnode);
if (pCont != NULL) { if (pCont != NULL) {
taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont)); taosSendHttpReport(TELEMETRY_SERVER, TELEMETRY_PORT, pCont, strlen(pCont), HTTP_FLAT);
free(pCont); free(pCont);
} }
taosWUnLockLatch(&pMgmt->lock); taosWUnLockLatch(&pMgmt->lock);

149
source/dnode/mnode/impl/src/mnode.c
View File

@ -359,6 +359,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL; return NULL;
} }
mndUpdateMnodeRole(pMnode);
mDebug("mnode open successfully "); mDebug("mnode open successfully ");
return pMnode; return pMnode;
} }
@ -385,26 +386,6 @@ void mndDestroy(const char *path) {
mDebug("mnode is destroyed"); mDebug("mnode is destroyed");
} }
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
pLoad->numOfDnode = 0;
pLoad->numOfMnode = 0;
pLoad->numOfVgroup = 0;
pLoad->numOfDatabase = 0;
pLoad->numOfSuperTable = 0;
pLoad->numOfChildTable = 0;
pLoad->numOfColumn = 0;
pLoad->totalPoints = 0;
pLoad->totalStorage = 0;
pLoad->compStorage = 0;
return 0;
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
return 0;
}
SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg));
if (pMsg == NULL) { if (pMsg == NULL) {
@ -523,3 +504,131 @@ uint64_t mndGenerateUid(char *name, int32_t len) {
} }
} while (true); } while (true);
} }
void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
memset(pLoad, 0, sizeof(SMnodeLoad));
SSdb *pSdb = pMnode->pSdb;
pLoad->numOfDnode = sdbGetSize(pSdb, SDB_DNODE);
pLoad->numOfMnode = sdbGetSize(pSdb, SDB_MNODE);
pLoad->numOfVgroup = sdbGetSize(pSdb, SDB_VGROUP);
pLoad->numOfDatabase = sdbGetSize(pSdb, SDB_DB);
pLoad->numOfSuperTable = sdbGetSize(pSdb, SDB_STB);
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pLoad->numOfChildTable += pVgroup->numOfTables;
pLoad->numOfColumn += pVgroup->numOfTimeSeries;
pLoad->totalPoints += pVgroup->pointsWritten;
pLoad->totalStorage += pVgroup->totalStorage;
pLoad->compStorage += pVgroup->compStorage;
sdbRelease(pSdb, pVgroup);
}
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
if (!mndIsMaster(pMnode)) return -1;
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc));
pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) {
return -1;
}
// cluster info
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
void *pIter = NULL;
while (1) {
SDnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonDnodeDesc desc = {0};
desc.dnode_id = pObj->id;
tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep));
if (mndIsDnodeOnline(pMnode, pObj, ms)) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
} else {
tstrncpy(desc.status, "offline", sizeof(desc.status));
}
taosArrayPush(pClusterInfo->dnodes, &desc);
sdbRelease(pSdb, pObj);
}
pIter = NULL;
while (1) {
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SMonMnodeDesc desc = {0};
desc.mnode_id = pObj->id;
tstrncpy(desc.mnode_ep, pObj->pDnode->ep, sizeof(desc.mnode_ep));
tstrncpy(desc.role, mndGetRoleStr(pObj->role), sizeof(desc.role));
taosArrayPush(pClusterInfo->mnodes, &desc);
sdbRelease(pSdb, pObj);
if (pObj->role == TAOS_SYNC_STATE_LEADER) {
pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
pClusterInfo->master_uptime = (ms - pObj->roleTime) / (86400000.0f);
}
}
// vgroup info
pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
pClusterInfo->vgroups_total++;
SMonVgroupDesc desc = {0};
desc.vgroup_id = pVgroup->vgId;
strncpy(desc.database_name, pVgroup->dbName, sizeof(desc.database_name));
desc.tables_num = pVgroup->numOfTables;
pGrantInfo->timeseries_used += pVgroup->numOfTimeSeries;
tstrncpy(desc.status, "unsynced", sizeof(desc.status));
for (int32_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
SMonVnodeDesc *pVnDesc = &desc.vnodes[i];
pVnDesc->dnode_id = pVgid->dnodeId;
tstrncpy(pVnDesc->vnode_role, mndGetRoleStr(pVgid->role), sizeof(pVnDesc->vnode_role));
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
tstrncpy(desc.status, "ready", sizeof(desc.status));
pClusterInfo->vgroups_alive++;
}
if (pVgid->role == TAOS_SYNC_STATE_LEADER || pVgid->role == TAOS_SYNC_STATE_CANDIDATE) {
pClusterInfo->vnodes_alive++;
}
pClusterInfo->vnodes_total++;
}
taosArrayPush(pVgroupInfo->vgroups, &desc);
sdbRelease(pSdb, pVgroup);
}
// grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;
if (pMnode->grant.expireTimeMS == 0) {
pGrantInfo->expire_time = INT32_MAX;
pGrantInfo->timeseries_total = INT32_MAX;
}
return 0;
}

12
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -28,12 +28,12 @@
extern "C" { extern "C" {
#endif #endif
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} #define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} #define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define SDB_MAX_SIZE (32 * 1024) #define SDB_MAX_SIZE (32 * 1024)

3
source/dnode/snode/src/snode.c
View File

@ -17,6 +17,7 @@
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = calloc(1, sizeof(SSnode)); SSnode *pSnode = calloc(1, sizeof(SSnode));
memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt));
return pSnode; return pSnode;
} }
@ -29,4 +30,4 @@ int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return 0; return 0;
} }
void sndDestroy(const char *path) {} void sndDestroy(const char *path) {}

2
source/dnode/vnode/CMakeLists.txt
View File

@ -54,5 +54,5 @@ elseif(${META_DB_IMPL} STREQUAL "TDB")
endif() endif()
if(${BUILD_TEST}) if(${BUILD_TEST})
# add_subdirectory(test) add_subdirectory(test)
endif(${BUILD_TEST}) endif(${BUILD_TEST})

29
source/dnode/vnode/inc/meta.h
View File

@ -24,8 +24,8 @@
extern "C" { extern "C" {
#endif #endif
#define META_SUPER_TABLE TD_SUPER_TABLE #define META_SUPER_TABLE TD_SUPER_TABLE
#define META_CHILD_TABLE TD_CHILD_TABLE #define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE
// Types exported // Types exported
@ -38,22 +38,29 @@ typedef struct SMetaCfg {
typedef struct SMTbCursor SMTbCursor; typedef struct SMTbCursor SMTbCursor;
typedef struct SMCtbCursor SMCtbCursor; typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMSmaCursor SMSmaCursor;
typedef SVCreateTbReq STbCfg; typedef SVCreateTbReq STbCfg;
typedef SVCreateTSmaReq SSmaCfg;
// SMeta operations // SMeta operations
SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); SMeta * metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
void metaClose(SMeta *pMeta); void metaClose(SMeta *pMeta);
void metaRemove(const char *path); void metaRemove(const char *path);
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaDropTable(SMeta *pMeta, tb_uid_t uid);
int metaCommit(SMeta *pMeta); int metaCommit(SMeta *pMeta);
int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
int32_t metaDropTSma(SMeta *pMeta, char *indexName);
// For Query // For Query
STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);
STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid);
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); STSchema * metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver);
STSma * metaGetSmaInfoByName(SMeta *pMeta, const char *indexName);
STSmaWrapper * metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid);
SArray * metaGetSmaTbUids(SMeta *pMeta, bool isDup);
SMTbCursor *metaOpenTbCursor(SMeta *pMeta); SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur); void metaCloseTbCursor(SMTbCursor *pTbCur);
@ -63,6 +70,10 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor *pCtbCur); void metaCloseCtbCurosr(SMCtbCursor *pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur);
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid);
void metaCloseSmaCurosr(SMSmaCursor *pSmaCur);
const char * metaSmaCursorNext(SMSmaCursor *pSmaCur);
// Options // Options
void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsInit(SMetaCfg *pMetaCfg);
void metaOptionsClear(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg);

21
source/dnode/vnode/inc/tsdb.h
View File

@ -87,6 +87,27 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp);
int tsdbPrepareCommit(STsdb *pTsdb); int tsdbPrepareCommit(STsdb *pTsdb);
int tsdbCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb);
/**
* @brief Insert tSma(Time-range-wise SMA) data from stream computing engine
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData);
/**
* @brief Insert RSma(Time-range-wise Rollup SMA) data.
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData);
// STsdbCfg // STsdbCfg
int tsdbOptionsInit(STsdbCfg *); int tsdbOptionsInit(STsdbCfg *);
void tsdbOptionsClear(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *);

21
source/dnode/vnode/inc/vnode.h
View File

@ -213,6 +213,10 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA
//} //}
static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash) {
taosHashClear(pHandle->tbIdHash);
}
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) { if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -227,6 +231,23 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S
return 0; return 0;
} }
static FORCE_INLINE int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) {
if (pHandle->tbIdHash == NULL) {
pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (pHandle->tbIdHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i);
taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
}
return 0;
}
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlock(STqReadHandle *pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);

4
source/dnode/vnode/src/inc/metaDef.h
View File

@ -33,6 +33,8 @@ int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta);
int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName);
// SMetaCache // SMetaCache
int metaOpenCache(SMeta* pMeta); int metaOpenCache(SMeta* pMeta);
@ -76,4 +78,4 @@ struct SMeta {
} }
#endif #endif
#endif /*_TD_META_DEF_H_*/ #endif /*_TD_META_DEF_H_*/

62
source/dnode/vnode/src/inc/tqInt.h
View File

@ -25,43 +25,41 @@
extern "C" { extern "C" {
#endif #endif
extern int32_t tqDebugFlag; #define tqFatal(...) \
{ \
#define tqFatal(...) \ if (tqDebugFlag & DEBUG_FATAL) { \
{ \ taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
if (tqDebugFlag & DEBUG_FATAL) { \ } \
taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \
} \
} }
#define tqError(...) \ #define tqError(...) \
{ \ { \
if (tqDebugFlag & DEBUG_ERROR) { \ if (tqDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} }
#define tqWarn(...) \ #define tqWarn(...) \
{ \ { \
if (tqDebugFlag & DEBUG_WARN) { \ if (tqDebugFlag & DEBUG_WARN) { \
taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} }
#define tqInfo(...) \ #define tqInfo(...) \
{ \ { \
if (tqDebugFlag & DEBUG_INFO) { \ if (tqDebugFlag & DEBUG_INFO) { \
taosPrintLog("TQ ", 255, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} }
#define tqDebug(...) \ #define tqDebug(...) \
{ \ { \
if (tqDebugFlag & DEBUG_DEBUG) { \ if (tqDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define tqTrace(...) \ #define tqTrace(...) \
{ \ { \
if (tqDebugFlag & DEBUG_TRACE) { \ if (tqDebugFlag & DEBUG_TRACE) { \
taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); \
} \ } \
} }
#define TQ_BUFFER_SIZE 8 #define TQ_BUFFER_SIZE 8

8
source/dnode/vnode/src/inc/tsdbCommit.h
View File

@ -16,6 +16,10 @@
#ifndef _TD_TSDB_COMMIT_H_ #ifndef _TD_TSDB_COMMIT_H_
#define _TD_TSDB_COMMIT_H_ #define _TD_TSDB_COMMIT_H_
#ifdef __cplusplus
extern "C" {
#endif
typedef struct { typedef struct {
int minFid; int minFid;
int midFid; int midFid;
@ -66,4 +70,8 @@ int tsdbApplyRtn(STsdbRepo *pRepo);
#endif #endif
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_COMMIT_H_ */ #endif /* _TD_TSDB_COMMIT_H_ */

1
source/dnode/vnode/src/inc/tsdbDef.h
View File

@ -35,6 +35,7 @@
#include "tsdbMemory.h" #include "tsdbMemory.h"
#include "tsdbOptions.h" #include "tsdbOptions.h"
#include "tsdbReadImpl.h" #include "tsdbReadImpl.h"
#include "tsdbSma.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

9
source/dnode/vnode/src/inc/tsdbFS.h
View File

@ -18,6 +18,10 @@
#include "tsdbFile.h" #include "tsdbFile.h"
#ifdef __cplusplus
extern "C" {
#endif
// ================== TSDB global config // ================== TSDB global config
extern bool tsdbForceKeepFile; extern bool tsdbForceKeepFile;
@ -38,6 +42,7 @@ typedef struct {
typedef struct { typedef struct {
STsdbFSMeta meta; // FS meta STsdbFSMeta meta; // FS meta
SArray * df; // data file array SArray * df; // data file array
SArray * smaf; // sma data file array
} SFSStatus; } SFSStatus;
typedef struct { typedef struct {
@ -111,4 +116,8 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
return 0; return 0;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_FS_H_ */ #endif /* _TD_TSDB_FS_H_ */

8
source/dnode/vnode/src/inc/tsdbFile.h
View File

@ -19,6 +19,10 @@
#include "tchecksum.h" #include "tchecksum.h"
#include "tfs.h" #include "tfs.h"
#ifdef __cplusplus
extern "C" {
#endif
#define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
@ -410,4 +414,8 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
return true; return true;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TS_TSDB_FILE_H_ */ #endif /* _TS_TSDB_FILE_H_ */

20
source/dnode/vnode/src/inc/tsdbLog.h
View File

@ -18,13 +18,21 @@
#include "tlog.h" #include "tlog.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int32_t tsdbDebugFlag; extern int32_t tsdbDebugFlag;
#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) #define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) #define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0)
#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) #define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0)
#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) #define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0)
#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_LOG_H_ */ #endif /* _TD_TSDB_LOG_H_ */

7
source/dnode/vnode/src/inc/tsdbMemory.h
View File

@ -16,6 +16,10 @@
#ifndef _TD_TSDB_MEMORY_H_ #ifndef _TD_TSDB_MEMORY_H_
#define _TD_TSDB_MEMORY_H_ #define _TD_TSDB_MEMORY_H_
#ifdef __cplusplus
extern "C" {
#endif
static void * taosTMalloc(size_t size); static void * taosTMalloc(size_t size);
static void * taosTCalloc(size_t nmemb, size_t size); static void * taosTCalloc(size_t nmemb, size_t size);
static void * taosTRealloc(void *ptr, size_t size); static void * taosTRealloc(void *ptr, size_t size);
@ -70,5 +74,8 @@ static FORCE_INLINE void* taosTZfree(void* ptr) {
return NULL; return NULL;
} }
#ifdef __cplusplus
}
#endif
#endif /* _TD_TSDB_MEMORY_H_ */ #endif /* _TD_TSDB_MEMORY_H_ */

8
source/dnode/vnode/src/inc/tsdbReadImpl.h
View File

@ -24,6 +24,10 @@
#include "tsdbMemory.h" #include "tsdbMemory.h"
#include "tcommon.h" #include "tcommon.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SReadH SReadH; typedef struct SReadH SReadH;
typedef struct { typedef struct {
@ -244,4 +248,8 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
return 0; return 0;
} }
#ifdef __cplusplus
}
#endif
#endif /*_TD_TSDB_READ_IMPL_H_*/ #endif /*_TD_TSDB_READ_IMPL_H_*/

44
source/dnode/vnode/src/inc/tsdbSma.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TSDB_SMA_H_
#define _TD_TSDB_SMA_H_
// insert/update interface
int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData);
int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData);
// query interface
// TODO: This is the basic params, and should wrap the params to a queryHandle.
int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult);
// management interface
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void* result);
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
// internal func
static FORCE_INLINE int32_t tsdbEncodeTSmaKey(uint64_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) {
int32_t len = 0;
len += taosEncodeFixedU64(pData, tableUid);
len += taosEncodeFixedU16(pData, colId);
len += taosEncodeFixedI64(pData, tsKey);
return len;
}
#endif /* _TD_TSDB_SMA_H_ */

63
source/dnode/vnode/src/inc/vnd.h
View File

@ -88,44 +88,41 @@ int vnodeScheduleTask(SVnodeTask* task);
int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
// For Log #define vFatal(...) \
extern int32_t vDebugFlag; do { \
if (vDebugFlag & DEBUG_FATAL) { \
#define vFatal(...) \ taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
do { \ } \
if (vDebugFlag & DEBUG_FATAL) { \
taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \
} \
} while (0) } while (0)
#define vError(...) \ #define vError(...) \
do { \ do { \
if (vDebugFlag & DEBUG_ERROR) { \ if (vDebugFlag & DEBUG_ERROR) { \
taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \ taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vWarn(...) \ #define vWarn(...) \
do { \ do { \
if (vDebugFlag & DEBUG_WARN) { \ if (vDebugFlag & DEBUG_WARN) { \
taosPrintLog("VND WARN ", 255, __VA_ARGS__); \ taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vInfo(...) \ #define vInfo(...) \
do { \ do { \
if (vDebugFlag & DEBUG_INFO) { \ if (vDebugFlag & DEBUG_INFO) { \
taosPrintLog("VND ", 255, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vDebug(...) \ #define vDebug(...) \
do { \ do { \
if (vDebugFlag & DEBUG_DEBUG) { \ if (vDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
#define vTrace(...) \ #define vTrace(...) \
do { \ do { \
if (vDebugFlag & DEBUG_TRACE) { \ if (vDebugFlag & DEBUG_TRACE) { \
taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \ taosPrintLog("VND ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); \
} \ } \
} while (0) } while (0)
// vnodeCfg.h // vnodeCfg.h

273
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -38,11 +38,14 @@ struct SMetaDB {
// DB // DB
DB *pTbDB; DB *pTbDB;
DB *pSchemaDB; DB *pSchemaDB;
DB *pSmaDB;
// IDX // IDX
DB *pNameIdx; DB *pNameIdx;
DB *pStbIdx; DB *pStbIdx;
DB *pNtbIdx; DB *pNtbIdx;
DB *pCtbIdx; DB *pCtbIdx;
DB *pSmaIdx;
// ENV // ENV
DB_ENV *pEvn; DB_ENV *pEvn;
}; };
@ -61,11 +64,12 @@ static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT
static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey);
static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg);
static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg);
static void metaClearTbCfg(STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg);
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW);
static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW);
static void metaDBWLock(SMetaDB *pDB); static void metaDBWLock(SMetaDB *pDB);
static void metaDBRLock(SMetaDB *pDB); static void metaDBRLock(SMetaDB *pDB);
static void metaDBULock(SMetaDB *pDB); static void metaDBULock(SMetaDB *pDB);
@ -100,6 +104,11 @@ int metaOpenDB(SMeta *pMeta) {
return -1; return -1;
} }
if (metaOpenBDBDb(&(pDB->pSmaDB), pDB->pEvn, "sma.db", false) < 0) {
metaCloseDB(pMeta);
return -1;
}
// Open Indices // Open Indices
if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) { if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "name.index", pDB->pTbDB, &metaNameIdxCb, false) < 0) {
metaCloseDB(pMeta); metaCloseDB(pMeta);
@ -121,15 +130,22 @@ int metaOpenDB(SMeta *pMeta) {
return -1; return -1;
} }
if (metaOpenBDBIdx(&(pDB->pSmaIdx), pDB->pEvn, "sma.index", pDB->pSmaDB, &metaSmaIdxCb, true) < 0) {
metaCloseDB(pMeta);
return -1;
}
return 0; return 0;
} }
void metaCloseDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) {
if (pMeta->pDB) { if (pMeta->pDB) {
metaCloseBDBIdx(pMeta->pDB->pSmaIdx);
metaCloseBDBIdx(pMeta->pDB->pCtbIdx); metaCloseBDBIdx(pMeta->pDB->pCtbIdx);
metaCloseBDBIdx(pMeta->pDB->pNtbIdx); metaCloseBDBIdx(pMeta->pDB->pNtbIdx);
metaCloseBDBIdx(pMeta->pDB->pStbIdx); metaCloseBDBIdx(pMeta->pDB->pStbIdx);
metaCloseBDBIdx(pMeta->pDB->pNameIdx); metaCloseBDBIdx(pMeta->pDB->pNameIdx);
metaCloseBDBDb(pMeta->pDB->pSmaDB);
metaCloseBDBDb(pMeta->pDB->pSchemaDB); metaCloseBDBDb(pMeta->pDB->pSchemaDB);
metaCloseBDBDb(pMeta->pDB->pTbDB); metaCloseBDBDb(pMeta->pDB->pTbDB);
metaCloseBDBEnv(pMeta->pDB->pEvn); metaCloseBDBEnv(pMeta->pDB->pEvn);
@ -142,7 +158,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
tb_uid_t uid; tb_uid_t uid;
char buf[512]; char buf[512];
char buf1[512]; char buf1[512];
void * pBuf; void *pBuf;
DBT key1, value1; DBT key1, value1;
DBT key2, value2; DBT key2, value2;
SSchema *pSchema = NULL; SSchema *pSchema = NULL;
@ -210,6 +226,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
return 0; return 0;
} }
int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
char buf[512] = {0}; // TODO: may overflow
void *pBuf = NULL;
DBT key1 = {0}, value1 = {0};
{
// save sma info
pBuf = buf;
key1.data = pSmaCfg->indexName;
key1.size = strlen(key1.data);
tEncodeTSma(&pBuf, pSmaCfg);
value1.data = buf;
value1.size = POINTER_DISTANCE(pBuf, buf);
value1.app_data = pSmaCfg;
}
metaDBWLock(pMeta->pDB);
pMeta->pDB->pSmaDB->put(pMeta->pDB->pSmaDB, NULL, &key1, &value1, 0);
metaDBULock(pMeta->pDB);
return 0;
}
int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) {
// TODO
#if 0
DBT key = {0};
key.data = (void *)indexName;
key.size = strlen(indexName);
metaDBWLock(pMeta->pDB);
// TODO: No guarantee of consistence.
// Use transaction or DB->sync() for some guarantee.
pMeta->pDB->pSmaDB->del(pMeta->pDB->pSmaDB, NULL, &key, 0);
metaDBULock(pMeta->pDB);
#endif
return 0;
}
/* ------------------------ STATIC METHODS ------------------------ */ /* ------------------------ STATIC METHODS ------------------------ */
static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) {
int tlen = 0; int tlen = 0;
@ -394,7 +453,7 @@ static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
STbCfg *pTbCfg = (STbCfg *)(pValue->app_data); STbCfg *pTbCfg = (STbCfg *)(pValue->app_data);
DBT * pDbt; DBT *pDbt;
if (pTbCfg->type == META_CHILD_TABLE) { if (pTbCfg->type == META_CHILD_TABLE) {
// pDbt = calloc(2, sizeof(DBT)); // pDbt = calloc(2, sizeof(DBT));
@ -425,6 +484,16 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey
} }
} }
static int metaSmaIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) {
STSma *pSmaCfg = (STSma *)(pValue->app_data);
memset(pSKey, 0, sizeof(*pSKey));
pSKey->data = &(pSmaCfg->tableUid);
pSKey->size = sizeof(pSmaCfg->tableUid);
return 0;
}
static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
int tsize = 0; int tsize = 0;
@ -479,7 +548,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) {
/* ------------------------ FOR QUERY ------------------------ */ /* ------------------------ FOR QUERY ------------------------ */
STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
STbCfg * pTbCfg = NULL; STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
@ -509,7 +578,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) {
} }
STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
STbCfg * pTbCfg = NULL; STbCfg *pTbCfg = NULL;
SMetaDB *pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
DBT key = {0}; DBT key = {0};
DBT pkey = {0}; DBT pkey = {0};
@ -540,13 +609,43 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
return pTbCfg; return pTbCfg;
} }
STSma *metaGetSmaInfoByName(SMeta *pMeta, const char *indexName) {
STSma * pCfg = NULL;
SMetaDB *pDB = pMeta->pDB;
DBT key = {0};
DBT value = {0};
int ret;
// Set key/value
key.data = (void *)indexName;
key.size = strlen(indexName);
// Query
metaDBRLock(pDB);
ret = pDB->pTbDB->get(pDB->pSmaDB, NULL, &key, &value, 0);
metaDBULock(pDB);
if (ret != 0) {
return NULL;
}
// Decode
pCfg = (STSma *)malloc(sizeof(STSma));
if (pCfg == NULL) {
return NULL;
}
tDecodeTSma(value.data, pCfg);
return pCfg;
}
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
uint32_t nCols; uint32_t nCols;
SSchemaWrapper *pSW = NULL; SSchemaWrapper *pSW = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
int ret; int ret;
void * pBuf; void *pBuf;
SSchema * pSchema; SSchema *pSchema;
SSchemaKey schemaKey = {uid, sver, 0}; SSchemaKey schemaKey = {uid, sver, 0};
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
@ -578,7 +677,7 @@ struct SMTbCursor {
SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
SMTbCursor *pTbCur = NULL; SMTbCursor *pTbCur = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur));
if (pTbCur == NULL) { if (pTbCur == NULL) {
@ -609,7 +708,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
DBT key = {0}; DBT key = {0};
DBT value = {0}; DBT value = {0};
STbCfg tbCfg; STbCfg tbCfg;
void * pBuf; void *pBuf;
for (;;) { for (;;) {
if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) {
@ -631,10 +730,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
STSchemaBuilder sb; STSchemaBuilder sb;
STSchema * pTSchema = NULL; STSchema *pTSchema = NULL;
SSchema * pSchema; SSchema *pSchema;
SSchemaWrapper *pSW; SSchemaWrapper *pSW;
STbCfg * pTbCfg; STbCfg *pTbCfg;
tb_uid_t quid; tb_uid_t quid;
pTbCfg = metaGetTbInfoByUid(pMeta, uid); pTbCfg = metaGetTbInfoByUid(pMeta, uid);
@ -662,13 +761,13 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
} }
struct SMCtbCursor { struct SMCtbCursor {
DBC * pCur; DBC *pCur;
tb_uid_t suid; tb_uid_t suid;
}; };
SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
SMCtbCursor *pCtbCur = NULL; SMCtbCursor *pCtbCur = NULL;
SMetaDB * pDB = pMeta->pDB; SMetaDB *pDB = pMeta->pDB;
int ret; int ret;
pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur)); pCtbCur = (SMCtbCursor *)calloc(1, sizeof(*pCtbCur));
@ -700,7 +799,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
DBT skey = {0}; DBT skey = {0};
DBT pkey = {0}; DBT pkey = {0};
DBT pval = {0}; DBT pval = {0};
void * pBuf; void *pBuf;
STbCfg tbCfg; STbCfg tbCfg;
// Set key // Set key
@ -718,6 +817,146 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
} }
} }
struct SMSmaCursor {
DBC *pCur;
tb_uid_t uid;
};
SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
SMSmaCursor *pCur = NULL;
SMetaDB *pDB = pMeta->pDB;
int ret;
pCur = (SMSmaCursor *)calloc(1, sizeof(*pCur));
if (pCur == NULL) {
return NULL;
}
pCur->uid = uid;
// TODO: lock?
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &(pCur->pCur), 0);
if (ret != 0) {
free(pCur);
return NULL;
}
return pCur;
}
void metaCloseSmaCurosr(SMSmaCursor *pCur) {
if (pCur) {
if (pCur->pCur) {
pCur->pCur->close(pCur->pCur);
}
free(pCur);
}
}
const char *metaSmaCursorNext(SMSmaCursor *pCur) {
DBT skey = {0};
DBT pkey = {0};
DBT pval = {0};
// Set key
skey.data = &(pCur->uid);
skey.size = sizeof(pCur->uid);
// TODO: lock?
if (pCur->pCur->pget(pCur->pCur, &skey, &pkey, &pval, DB_NEXT) == 0) {
const char *indexName = (const char *)pkey.data;
assert(indexName != NULL);
return indexName;
} else {
return NULL;
}
}
STSmaWrapper *metaGetSmaInfoByUid(SMeta *pMeta, tb_uid_t uid) {
STSmaWrapper *pSW = NULL;
pSW = calloc(sizeof(*pSW), 1);
if (pSW == NULL) {
return NULL;
}
SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
if (pCur == NULL) {
free(pSW);
return NULL;
}
DBT skey = {.data = &(pCur->uid), .size = sizeof(pCur->uid)};
DBT pval = {0};
void *pBuf = NULL;
while (true) {
// TODO: lock?
if (pCur->pCur->pget(pCur->pCur, &skey, NULL, &pval, DB_NEXT) == 0) {
++pSW->number;
STSma *tptr = (STSma *)realloc(pSW->tSma, pSW->number * sizeof(STSma));
if (tptr == NULL) {
metaCloseSmaCurosr(pCur);
tdDestroyTSmaWrapper(pSW);
tfree(pSW);
return NULL;
}
pSW->tSma = tptr;
pBuf = pval.data;
if (tDecodeTSma(pBuf, pSW->tSma + pSW->number - 1) == NULL) {
metaCloseSmaCurosr(pCur);
tdDestroyTSmaWrapper(pSW);
tfree(pSW);
return NULL;
}
continue;
}
break;
}
metaCloseSmaCurosr(pCur);
return pSW;
}
SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) {
SArray * pUids = NULL;
SMetaDB *pDB = pMeta->pDB;
DBC * pCur = NULL;
DBT pkey = {0}, pval = {0};
int ret;
pUids = taosArrayInit(16, sizeof(tb_uid_t));
if (!pUids) {
return NULL;
}
// TODO: lock?
ret = pDB->pCtbIdx->cursor(pDB->pSmaIdx, NULL, &pCur, 0);
if (ret != 0) {
taosArrayDestroy(pUids);
return NULL;
}
void *pBuf = NULL;
// TODO: lock?
while (true) {
ret = pCur->get(pCur, &pkey, &pval, isDup ? DB_NEXT_DUP : DB_NEXT_NODUP);
if(ret == 0) {
taosArrayPush(pUids, pkey.data);
continue;
}
break;
}
if (pCur) {
pCur->close(pCur);
}
return pUids;
}
static void metaDBWLock(SMetaDB *pDB) { static void metaDBWLock(SMetaDB *pDB) {
#if IMPL_WITH_LOCK #if IMPL_WITH_LOCK
pthread_rwlock_wrlock(&(pDB->rwlock)); pthread_rwlock_wrlock(&(pDB->rwlock));

25
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -106,3 +106,28 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
// TODO // TODO
return 0; return 0;
} }
int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
// TODO: Validate the cfg
// The table uid should exists and be super table or common table.
// Check other cfg value
// TODO: add atomicity
if (metaSaveSmaToDB(pMeta, &pCfg->tSma) < 0) {
// TODO: handle error
return -1;
}
return TSDB_CODE_SUCCESS;
}
int32_t metaDropTSma(SMeta *pMeta, char* indexName) {
// TODO: Validate the cfg
// TODO: add atomicity
if (metaRemoveSmaFromDb(pMeta, indexName) < 0) {
// TODO: handle error
return -1;
}
return TSDB_CODE_SUCCESS;
}

1
source/dnode/vnode/src/meta/metaTable.c
View File

@ -50,3 +50,4 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
return 0; return 0;
} }

9
source/dnode/vnode/src/tq/tq.c
View File

@ -72,6 +72,8 @@ void tqClose(STQ* pTq) {
} }
int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) { int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
// if waiting
// memcpy and send msg to fetch thread
// TODO: add reference // TODO: add reference
// if handle waiting, launch query and response to consumer // if handle waiting, launch query and response to consumer
// //
@ -210,7 +212,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont; SMqConsumeReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId; int64_t consumerId = pReq->consumerId;
int64_t fetchOffset; int64_t fetchOffset;
/*int64_t blockingTime = pReq->blockingTime;*/ int64_t blockingTime = pReq->blockingTime;
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0; fetchOffset = 0;
@ -374,7 +376,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0; pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta}; SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnodeMeta,
};
pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].pReadHandle = pReadHandle;
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
} }

1
source/dnode/vnode/src/tq/tqRead.c
View File

@ -28,6 +28,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
pReadHandle->sver = -1; pReadHandle->sver = -1;
pReadHandle->pSchema = NULL; pReadHandle->pSchema = NULL;
pReadHandle->pSchemaWrapper = NULL; pReadHandle->pSchemaWrapper = NULL;
pReadHandle->tbIdHash = NULL;
return pReadHandle; return pReadHandle;
} }

2
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -1054,7 +1054,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
// pfs->metaCacheComp = NULL; // pfs->metaCacheComp = NULL;
// } else { // } else {
// // remove meta.tmp file // // remove meta.tmp file
// remove(mf.f.aname); // taosRemoveFile(mf.f.aname);
// taosHashCleanup(pfs->metaCacheComp); // taosHashCleanup(pfs->metaCacheComp);
// pfs->metaCacheComp = NULL; // pfs->metaCacheComp = NULL;
// } // }

8
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -439,7 +439,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
return -1; return -1;
} }
@ -447,7 +447,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (fsheader.len > 0) { if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
return -1; return -1;
} }
@ -458,7 +458,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
(void)remove(tfname); (void)taosRemoveFile(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
} }
@ -468,7 +468,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
if (taosFsyncFile(pFile) < 0) { if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
remove(tfname); taosRemoveFile(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
} }

43
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -13,18 +13,19 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <tdatablock.h>
#include "os.h"
#include "talgo.h"
#include "tcompare.h"
#include "tdataformat.h"
#include "texception.h"
#include "tsdb.h" #include "tsdb.h"
#include "tsdbDef.h" #include "tsdbDef.h"
#include "tsdbFS.h" #include "tsdbFS.h"
#include "tsdbLog.h" #include "tsdbLog.h"
#include "tsdbReadImpl.h" #include "tsdbReadImpl.h"
#include "ttime.h"
#include "texception.h"
#include "os.h"
#include "talgo.h"
#include "tcompare.h"
#include "tdataformat.h"
#include "tskiplist.h" #include "tskiplist.h"
#include "ttime.h"
#include "taosdef.h" #include "taosdef.h"
#include "tlosertree.h" #include "tlosertree.h"
@ -1472,6 +1473,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
return numOfRows + num; return numOfRows + num;
} }
// TODO fix bug for reverse copy data
// TODO handle the null data
// Note: row1 always has high priority // Note: row1 always has high priority
static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1,
STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
@ -1515,7 +1518,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
} }
} }
int32_t i = 0, j = 0, k = 0; int32_t i = 0, j = 0, k = 0;
while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
@ -1586,7 +1588,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
tdSKvRowGetVal(row, colId, offset, chosen_itr, &sVal); tdSKvRowGetVal(row, colId, offset, chosen_itr, &sVal);
} }
if (colId == pColInfo->info.colId) { if (colId == pColInfo->info.colId) {
if (tdValTypeIsNorm(sVal.valType)) { if (tdValTypeIsNorm(sVal.valType)) {
switch (pColInfo->info.type) { switch (pColInfo->info.type) {
@ -1594,7 +1595,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
memcpy(pData, sVal.val, varDataTLen(sVal.val)); memcpy(pData, sVal.val, varDataTLen(sVal.val));
break; break;
case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
@ -1625,11 +1625,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
memcpy(pData, sVal.val, pColInfo->info.bytes); memcpy(pData, sVal.val, pColInfo->info.bytes);
} }
} else if (forceSetNull) { } else if (forceSetNull) {
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { colDataAppend(pColInfo, numOfRows, NULL, true);
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
} }
i++; i++;
@ -1640,11 +1636,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
} }
} else { } else {
if(forceSetNull) { if(forceSetNull) {
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { colDataAppend(pColInfo, numOfRows, NULL, true);
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
} }
i++; i++;
} }
@ -1653,18 +1645,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
if(forceSetNull) { if(forceSetNull) {
while (i < numOfCols) { // the remain columns are all null data while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { colDataAppend(pColInfo, numOfRows, NULL, true);
pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
} else {
pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
}
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
setVardataNull(pData, pColInfo->info.type);
} else {
setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
}
i++; i++;
} }
} }

550
source/dnode/vnode/src/tsdb/tsdbSma.c Normal file
View File

@ -0,0 +1,550 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdbDef.h"
#define SMA_STORAGE_TSDB_DAYS 30
#define SMA_STORAGE_SPLIT_HOURS 24
#define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8
#define SMA_STORE_SINGLE_BLOCKS // store SMA data by single block or multiple blocks
typedef enum {
SMA_STORAGE_LEVEL_TSDB = 0, // store TSma in dir e.g. vnode${N}/tsdb/.tsma
SMA_STORAGE_LEVEL_DFILESET = 1 // store TSma in file e.g. vnode${N}/tsdb/v2f1900.tsma.${sma_index_name}
} ESmaStorageLevel;
typedef struct {
STsdb * pTsdb;
char * pDFile; // TODO: use the real DFile type, not char*
int32_t interval; // interval with the precision of DB
int32_t blockSize; // size of SMA block item
// TODO
} STSmaWriteH;
typedef struct {
int32_t iter;
} SmaFsIter;
typedef struct {
STsdb * pTsdb;
char * pDFile; // TODO: use the real DFile type, not char*
int32_t interval; // interval with the precision of DB
int32_t blockSize; // size of SMA block item
int8_t storageLevel;
int8_t days;
SmaFsIter smaFsIter;
// TODO
} STSmaReadH;
// declaration of static functions
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit);
static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t sectionDataLen, int32_t nBlocks);
static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen);
static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
int32_t fid, int32_t *nSmaBlocks);
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision);
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
int32_t fid);
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData);
static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin);
/**
* @brief Judge the tSma storage level
*
* @param interval
* @param intervalUnit
* @return int32_t
*/
static int32_t tsdbJudgeStorageLevel(int64_t interval, int8_t intervalUnit) {
// TODO: configurable for SMA_STORAGE_SPLIT_HOURS?
switch (intervalUnit) {
case TD_TIME_UNIT_HOUR:
if (interval < SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MINUTE:
if (interval < 60 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_SEC:
if (interval < 3600 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MILLISEC:
if (interval < 3600 * 1e3 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_MICROSEC:
if (interval < 3600 * 1e6 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
case TD_TIME_UNIT_NANOSEC:
if (interval < 3600 * 1e9 * SMA_STORAGE_SPLIT_HOURS) {
return SMA_STORAGE_LEVEL_DFILESET;
}
break;
default:
break;
}
return SMA_STORAGE_LEVEL_TSDB;
}
/**
* @brief Insert TSma data blocks to B+Tree
*
* @param bTree
* @param smaKey
* @param pData
* @param dataLen
* @return int32_t
*/
static int32_t tsdbInsertTSmaBlocks(void *bTree, const char *smaKey, const char *pData, int32_t dataLen) {
// TODO: insert sma data blocks into B+Tree
printf("insert sma data blocks into B+Tree: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d\n",
*(uint64_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
return TSDB_CODE_SUCCESS;
}
static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) {
if (intervalUnit < TD_TIME_UNIT_MILLISEC) {
switch (intervalUnit) {
case TD_TIME_UNIT_YEAR:
case TD_TIME_UNIT_SEASON:
case TD_TIME_UNIT_MONTH:
case TD_TIME_UNIT_WEEK:
// illegal time unit
tsdbError("invalid interval unit: %d\n", intervalUnit);
TASSERT(0);
break;
case TD_TIME_UNIT_DAY: // the interval for tSma calculation must <= day
interval *= 86400 * 1e3;
break;
case TD_TIME_UNIT_HOUR:
interval *= 3600 * 1e3;
break;
case TD_TIME_UNIT_MINUTE:
interval *= 60 * 1e3;
break;
case TD_TIME_UNIT_SEC:
interval *= 1e3;
break;
default:
break;
}
}
switch (intervalUnit) {
case TD_TIME_UNIT_MILLISEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval * 1e3;
} else { // nano second
return interval * 1e6;
}
break;
case TD_TIME_UNIT_MICROSEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval / 1e3;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval;
} else { // nano second
return interval * 1e3;
}
break;
case TD_TIME_UNIT_NANOSEC:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval / 1e6;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval / 1e3;
} else { // nano second
return interval;
}
break;
default:
if (TSDB_TIME_PRECISION_MILLI == precision) {
return interval * 1e3;
} else if (TSDB_TIME_PRECISION_MICRO == precision) {
return interval * 1e6;
} else { // nano second
return interval * 1e9;
}
break;
}
return interval;
}
/**
* @brief Split the TSma data blocks into expected size and insert into B+Tree.
*
* @param pSmaH
* @param pData
* @param nOffset The nOffset of blocks since fid changes.
* @param nBlocks The nBlocks with the same fid since nOffset.
* @return int32_t
*/
static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, int32_t nOffset, int32_t nBlocks) {
STsdb *pTsdb = pSmaH->pTsdb;
TASSERT(pData->colIds != NULL);
tsdbDebug("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d", nOffset, nBlocks);
printf("tsdbInsertTSmaDataSection: nOffset %d, nBlocks %d\n", nOffset, nBlocks);
int32_t colDataLen = pData->dataLen / pData->numOfColIds;
int32_t sectionDataLen = pSmaH->blockSize * nBlocks;
for (col_id_t i = 0; i < pData->numOfColIds; ++i) {
// param: pointer of B+Tree, key, value, dataLen
void *bTree = pSmaH->pDFile;
#ifndef SMA_STORE_SINGLE_BLOCKS
// save tSma data blocks as a whole
char smaKey[SMA_KEY_LEN] = {0};
void *pSmaKey = &smaKey;
tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + nOffset * pSmaH->interval,
(void **)&pSmaKey);
if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + nOffset * pSmaH->blockSize, sectionDataLen) <
0) {
tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
#else
// save tSma data blocks separately
for (int32_t n = 0; n < nBlocks; ++n) {
char smaKey[SMA_KEY_LEN] = {0};
void *pSmaKey = &smaKey;
tsdbEncodeTSmaKey(pData->tableUid, *(pData->colIds + i), pData->tsWindow.skey + (nOffset + n) * pSmaH->interval,
(void **)&pSmaKey);
if (tsdbInsertTSmaBlocks(bTree, smaKey, pData->data + i * colDataLen + (nOffset + n) * pSmaH->blockSize,
pSmaH->blockSize) < 0) {
tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
}
#endif
}
return TSDB_CODE_SUCCESS;
}
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
int32_t fid) {
// TODO
pSmaH->pDFile = "tSma_interval_file_name";
return TSDB_CODE_SUCCESS;
} /**
* @brief Split the sma data blocks by fid.
*
* @param pSmaH
* @param param
* @param pData
* @param nOffset
* @param fid
* @param nSmaBlocks
* @return int32_t
*/
static int32_t tsdbTSmaDataSplit(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t days, int32_t nOffset,
int32_t fid, int32_t *nSmaBlocks) {
STsdbCfg *pCfg = REPO_CFG(pSmaH->pTsdb);
// TODO: use binary search
for (int32_t n = nOffset + 1; n < pData->numOfBlocks; ++n) {
// TODO: The tsWindow.skey should use the precision of DB.
int32_t tFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey + pSmaH->interval * n, days, pCfg->precision));
if (tFid > fid) {
*nSmaBlocks = n - nOffset;
break;
}
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief Insert/Update Time-range-wise SMA data.
* - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
* v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files.
* - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The
* days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d).
* - The destination file of one data block for some interval is determined by its start TS key.
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) {
STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSmaData * curData = pData;
STSmaWriteH tSmaH = {0};
tsdbInitTSmaWriteH(&tSmaH, pTsdb, param, pData);
if (pData->numOfBlocks <= 0 || pData->numOfColIds <= 0 || pData->dataLen <= 0) {
TASSERT(0);
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
// Step 1: Judge the storage level
int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
// Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
// - Set and open the DFile or the B+Tree file
int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, daysPerFile, pCfg->precision));
int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, daysPerFile, pCfg->precision));
if (minFid == maxFid) {
// Save all the TSma data to one file
// TODO: tsdbStartTSmaCommit();
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, minFid);
tsdbInsertTSmaDataSection(&tSmaH, pData, 0, pData->numOfBlocks);
// TODO:tsdbEndTSmaCommit();
} else if (minFid < maxFid) {
// Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
// actually.
// TODO: tsdbStartTSmaCommit();
int32_t tFid = minFid;
int32_t nOffset = 0;
int32_t nSmaBlocks = 0;
do {
tsdbTSmaDataSplit(&tSmaH, param, pData, daysPerFile, nOffset, tFid, &nSmaBlocks);
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
if (tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, nSmaBlocks) < 0) {
return terrno;
}
++tFid;
nOffset += nSmaBlocks;
if (tFid == maxFid) {
tsdbSetTSmaDataFile(&tSmaH, param, pData, storageLevel, tFid);
tsdbInsertTSmaDataSection(&tSmaH, pData, nOffset, pData->numOfBlocks - nOffset);
break;
}
} while (true);
// TODO:tsdbEndTSmaCommit();
} else {
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return TSDB_CODE_SUCCESS;
}
static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, SRSma *param, STSmaData *pData, int32_t fid) {
// TODO
pSmaH->pDFile = "rSma_interval_file_name";
return TSDB_CODE_SUCCESS;
}
int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
STsdbCfg * pCfg = REPO_CFG(pTsdb);
STSma * tParam = &param->tsma;
STSmaData * curData = pData;
STSmaWriteH tSmaH = {0};
tsdbInitTSmaWriteH(&tSmaH, pTsdb, tParam, pData);
int32_t nSmaBlocks = pData->numOfBlocks;
int32_t colDataLen = pData->dataLen / nSmaBlocks;
// Step 2.2: Storage of SMA_STORAGE_LEVEL_DFILESET
// TODO: Use the daysPerFile for rSma data, not for TS data.
// TODO: The lifecycle of rSma data should be processed like the TS data files.
int32_t minFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.skey, pCfg->daysPerFile, pCfg->precision));
int32_t maxFid = (int32_t)(TSDB_KEY_FID(pData->tsWindow.ekey, pCfg->daysPerFile, pCfg->precision));
if (minFid == maxFid) {
// Save all the TSma data to one file
tsdbSetRSmaDataFile(&tSmaH, param, pData, minFid);
// TODO: tsdbStartTSmaCommit();
tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
// TODO:tsdbEndTSmaCommit();
} else if (minFid < maxFid) {
// Split the TSma data and save to multiple files. As there is limit for the span, it can't span more than 2 files
// actually.
// TODO: tsdbStartTSmaCommit();
int32_t tmpFid = 0;
int32_t step = 0;
for (int32_t n = 0; n < pData->numOfBlocks; ++n) {
}
tsdbInsertTSmaDataSection(&tSmaH, pData, colDataLen, nSmaBlocks);
// TODO:tsdbEndTSmaCommit();
} else {
TASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
// Step 4: finish
return TSDB_CODE_SUCCESS;
}
/**
* @brief Init of tSma ReadH
*
* @param pSmaH
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
/**
* @brief Init of tSma FS
*
* @param pReadH
* @param param
* @param queryWin
* @return int32_t
*/
static int32_t tsdbInitTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
int32_t storageLevel = tsdbJudgeStorageLevel(param->interval, param->intervalUnit);
int32_t daysPerFile =
storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : REPO_CFG(pReadH->pTsdb)->daysPerFile;
pReadH->storageLevel = storageLevel;
pReadH->days = daysPerFile;
pReadH->smaFsIter.iter = 0;
}
/**
* @brief Set and open tSma file if it has key locates in queryWin.
*
* @param pReadH
* @param param
* @param queryWin
* @return true
* @return false
*/
static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, STSma *param, STimeWindow *queryWin) {
SArray *smaFs = pReadH->pTsdb->fs->cstatus->smaf;
int32_t nSmaFs = taosArrayGetSize(smaFs);
pReadH->pDFile = NULL;
while (pReadH->smaFsIter.iter < nSmaFs) {
void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
if (pSmaFile) { // match(indexName, queryWindow)
// TODO: select the file by index_name ...
pReadH->pDFile = pSmaFile;
++pReadH->smaFsIter.iter;
break;
}
++pReadH->smaFsIter.iter;
}
if (pReadH->pDFile != NULL) {
tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
return true;
}
return false;
}
/**
* @brief Return the data between queryWin and fill the pData.
*
* @param pTsdb
* @param param
* @param pData
* @param queryWin
* @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
* @return int32_t
*/
int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeWindow *queryWin, int32_t nMaxResult) {
STSmaReadH tReadH = {0};
tsdbInitTSmaReadH(&tReadH, pTsdb, param, pData);
tsdbInitTSmaFile(&tReadH, param, queryWin);
int32_t nResult = 0;
int64_t lastKey = 0;
while (true) {
if (nResult >= nMaxResult) {
break;
}
// set and open the file according to the STSma param
if (tsdbSetAndOpenTSmaFile(&tReadH, param, queryWin)) {
char bTree[100] = "\0";
while (strncmp(bTree, "has more nodes", 100) == 0) {
if (nResult >= nMaxResult) {
break;
}
// tsdbGetDataFromBTree(bTree, queryWin, lastKey)
// fill the pData
++nResult;
}
}
}
// read data from file and fill the result
return TSDB_CODE_SUCCESS;
}
/**
* @brief Get the start TS key of the last data block of one interval/sliding.
*
* @param pTsdb
* @param param
* @param result
* @return int32_t
* 1) Return 0 and fill the result if the check procedure is normal;
* 2) Return -1 if error occurs during the check procedure.
*/
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result) {
const char *procedure = "";
if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
return -1;
}
// fill the result
return TSDB_CODE_SUCCESS;
}
/**
* @brief Remove the tSma data files related to param between pWin.
*
* @param pTsdb
* @param param
* @param pWin
* @return int32_t
*/
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin) {
// for ("tSmaFiles of param-interval-sliding between pWin") {
// // remove the tSmaFile
// }
return TSDB_CODE_SUCCESS;
}

41
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -15,6 +15,14 @@
#include "tsdbDef.h" #include "tsdbDef.h"
/**
* @brief insert TS data
*
* @param pTsdb
* @param pMsg
* @param pRsp
* @return int
*/
int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
// Check if mem is there. If not, create one. // Check if mem is there. If not, create one.
if (pTsdb->mem == NULL) { if (pTsdb->mem == NULL) {
@ -24,4 +32,37 @@ int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
} }
} }
return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL); return tsdbMemTableInsert(pTsdb, pTsdb->mem, pMsg, NULL);
}
/**
* @brief Insert/Update tSma(Time-range-wise SMA) data from stream computing engine
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
* TODO: Who is responsible for resource allocate and release?
*/
int32_t tsdbInsertTSmaData(STsdb *pTsdb, STSma *param, STSmaData *pData) {
int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbInsertTSmaDataImpl(pTsdb, param, pData)) < 0) {
tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return code;
}
/**
* @brief Insert Time-range-wise Rollup Sma(RSma) data
*
* @param pTsdb
* @param param
* @param pData
* @return int32_t
*/
int32_t tsdbInsertRSmaData(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
int32_t code = TSDB_CODE_SUCCESS;
if ((code = tsdbInsertRSmaDataImpl(pTsdb, param, pData)) < 0) {
tsdbWarn("vgId:%d insert rSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
}
return code;
} }

5
source/dnode/vnode/src/vnd/vnodeInt.c
View File

@ -32,6 +32,11 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
pLoad->totalStorage = 300; pLoad->totalStorage = 300;
pLoad->compStorage = 200; pLoad->compStorage = 200;
pLoad->pointsWritten = 100; pLoad->pointsWritten = 100;
pLoad->numOfSelectReqs = 1;
pLoad->numOfInsertReqs = 3;
pLoad->numOfInsertSuccessReqs = 2;
pLoad->numOfBatchInsertReqs = 5;
pLoad->numOfBatchInsertSuccessReqs = 4;
return 0; return 0;
} }

69
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -41,10 +41,8 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
return 0; return 0;
} }
int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SVCreateTbReq vCreateTbReq; void *ptr = NULL;
SVCreateTbBatchReq vCreateTbBatchReq;
void *ptr = NULL;
if (pVnode->config.streamMode == 0) { if (pVnode->config.streamMode == 0) {
ptr = vnodeMalloc(pVnode, pMsg->contLen); ptr = vnodeMalloc(pVnode, pMsg->contLen);
@ -64,18 +62,21 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_VND_CREATE_STB: case TDMT_VND_CREATE_STB: {
SVCreateTbReq vCreateTbReq = {0};
tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq);
if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) {
// TODO: handle error // TODO: handle error
} }
// TODO: maybe need to clear the requst struct // TODO: maybe need to clear the request struct
free(vCreateTbReq.stbCfg.pSchema); free(vCreateTbReq.stbCfg.pSchema);
free(vCreateTbReq.stbCfg.pTagSchema); free(vCreateTbReq.stbCfg.pTagSchema);
free(vCreateTbReq.name); free(vCreateTbReq.name);
break; break;
case TDMT_VND_CREATE_TABLE: }
case TDMT_VND_CREATE_TABLE: {
SVCreateTbBatchReq vCreateTbBatchReq = {0};
tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); tDeserializeSVCreateTbBatchReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq);
for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
@ -97,14 +98,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray));
taosArrayDestroy(vCreateTbBatchReq.pArray); taosArrayDestroy(vCreateTbBatchReq.pArray);
break; break;
}
case TDMT_VND_ALTER_STB: case TDMT_VND_ALTER_STB: {
SVCreateTbReq vAlterTbReq = {0};
vTrace("vgId:%d, process alter stb req", pVnode->vgId); vTrace("vgId:%d, process alter stb req", pVnode->vgId);
tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq);
free(vCreateTbReq.stbCfg.pSchema); free(vAlterTbReq.stbCfg.pSchema);
free(vCreateTbReq.stbCfg.pTagSchema); free(vAlterTbReq.stbCfg.pTagSchema);
free(vCreateTbReq.name); free(vAlterTbReq.name);
break; break;
}
case TDMT_VND_DROP_STB: case TDMT_VND_DROP_STB:
vTrace("vgId:%d, process drop stb req", pVnode->vgId); vTrace("vgId:%d, process drop stb req", pVnode->vgId);
break; break;
@ -129,6 +132,46 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
} }
} break; } break;
case TDMT_VND_CREATE_SMA: { // timeRangeSMA
SSmaCfg vCreateSmaReq = {0};
if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) {
// TODO: handle error
tdDestroyTSma(&vCreateSmaReq.tSma);
return -1;
}
// TODO: send msg to stream computing to create tSma
// if ((send msg to stream computing) < 0) {
// tdDestroyTSma(&vCreateSmaReq);
// return -1;
// }
tdDestroyTSma(&vCreateSmaReq.tSma);
// TODO: return directly or go on follow steps?
} break;
case TDMT_VND_CANCEL_SMA: { // timeRangeSMA
} break;
case TDMT_VND_DROP_SMA: { // timeRangeSMA
SVDropTSmaReq vDropSmaReq = {0};
if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexName) < 0) {
// TODO: handle error
return -1;
}
// TODO: send msg to stream computing to drop tSma
// if ((send msg to stream computing) < 0) {
// tdDestroyTSma(&vCreateSmaReq);
// return -1;
// }
// TODO: return directly or go on follow steps?
} break;
default: default:
ASSERT(0); ASSERT(0);
break; break;

Some files were not shown because too many files have changed in this diff Show More