diff --git a/.gitmodules b/.gitmodules index 07e4bb2b9c..bc38453f19 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "examples/rust"] path = examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git +[submodule "tools/taos-tools"] + path = tools/taos-tools + url = https://github.com/taosdata/taos-tools diff --git a/CMakeLists.txt b/CMakeLists.txt index df3aab5cb3..9d64966861 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,32 @@ project( DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)" ) +IF ("${BUILD_TOOLS}" STREQUAL "") + IF (TD_LINUX) + IF (TD_ARM_32) + SET(BUILD_TOOLS "false") + ELSEIF (TD_ARM_64) + SET(BUILD_TOOLS "false") + ELSE () + SET(BUILD_TOOLS "false") + ENDIF () + ELSEIF (TD_DARWIN) + SET(BUILD_TOOLS "false") + ELSE () + SET(BUILD_TOOLS "false") + ENDIF () +ENDIF () + +IF ("${BUILD_TOOLS}" MATCHES "false") + MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}") + SET(TD_TAOS_TOOLS FALSE) +ELSE () + MESSAGE("") + MESSAGE("${Green} Will build taos_tools! ${ColourReset}") + MESSAGE("") + SET(TD_TAOS_TOOLS TRUE) +ENDIF () + set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) diff --git a/include/client/taos.h b/include/client/taos.h index f098a48631..e1e85ac3a4 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -257,6 +257,10 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co void tmqShowMsg(tmq_message_t *tmq_message); int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); +typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); + +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT* stmt); + #ifdef __cplusplus } #endif diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 99360b8d3f..9e5e5ebdcf 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -63,6 +63,8 @@ typedef enum { extern char *qtypeStr[]; +#define TSDB_PORT_HTTP 11 + #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8d45af97ed..0dcf554433 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1129,18 +1129,34 @@ typedef struct { char* sql; char* physicalPlan; char* logicalPlan; -} SMCreateTopicReq; +} SCMCreateStreamReq; -int32_t tSerializeMCreateTopicReq(void* buf, int32_t bufLen, const SMCreateTopicReq* pReq); -int32_t tDeserializeSMCreateTopicReq(void* buf, int32_t bufLen, SMCreateTopicReq* pReq); -void tFreeSMCreateTopicReq(SMCreateTopicReq* pReq); +typedef struct { + int64_t streamId; +} SCMCreateStreamRsp; + +int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateStreamReq* pReq); +int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq); +void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq); + +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igExists; + char* sql; + char* physicalPlan; + char* logicalPlan; +} SCMCreateTopicReq; + +int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); +int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq); +void tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq); typedef struct { int64_t topicId; -} SMCreateTopicRsp; +} SCMCreateTopicRsp; -int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp); -int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp); +int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp); +int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp); typedef struct { int32_t topicNum; @@ -1877,19 +1893,27 @@ typedef enum { TD_TIME_UNIT_MICROSEC = 9, TD_TIME_UNIT_NANOSEC = 10 } ETDTimeUnit; + typedef struct { - uint8_t version; // for compatibility - uint8_t intervalUnit; - uint8_t slidingUnit; - char indexName[TSDB_INDEX_NAME_LEN + 1]; - col_id_t numOfColIds; - uint16_t numOfFuncIds; - uint64_t tableUid; // super/common table uid - int64_t interval; - int64_t sliding; - col_id_t* colIds; // sorted column ids - uint16_t* funcIds; // sorted sma function ids -} STSma; // Time-range-wise SMA + uint16_t funcId; + uint16_t nColIds; + col_id_t* colIds; // sorted colIds +} SFuncColIds; + +typedef struct { + uint8_t version; // for compatibility + uint8_t intervalUnit; + uint8_t slidingUnit; + char indexName[TSDB_INDEX_NAME_LEN]; + char timezone[TD_TIMEZONE_LEN]; + uint16_t nFuncColIds; + uint16_t tagsFilterLen; + tb_uid_t tableUid; // super/common table uid + int64_t interval; + int64_t sliding; + SFuncColIds* funcColIds; // sorted funcIds + char* tagsFilter; +} STSma; // Time-range-wise SMA typedef struct { int64_t ver; // use a general definition @@ -1898,13 +1922,13 @@ typedef struct { typedef struct { int8_t type; // 0 status report, 1 update data - char indexName[TSDB_INDEX_NAME_LEN + 1]; // + char indexName[TSDB_INDEX_NAME_LEN]; // STimeWindow windows; } STSmaMsg; typedef struct { int64_t ver; // use a general definition - char indexName[TSDB_INDEX_NAME_LEN + 1]; + char indexName[TSDB_INDEX_NAME_LEN]; } SVDropTSmaReq; typedef struct { } SVCreateTSmaRsp, SVDropTSmaRsp; @@ -1955,8 +1979,14 @@ typedef struct { static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { if (pSma) { - tfree(pSma->colIds); - tfree(pSma->funcIds); + if (pSma->funcColIds != NULL) { + for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) { + tfree((pSma->funcColIds + i)->colIds); + } + tfree(pSma->funcColIds); + } + + tfree(pSma->tagsFilter); } } @@ -1978,18 +2008,24 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { tlen += taosEncodeFixedU8(buf, pSma->intervalUnit); tlen += taosEncodeFixedU8(buf, pSma->slidingUnit); tlen += taosEncodeString(buf, pSma->indexName); - tlen += taosEncodeFixedU16(buf, pSma->numOfColIds); - tlen += taosEncodeFixedU16(buf, pSma->numOfFuncIds); - tlen += taosEncodeFixedU64(buf, pSma->tableUid); + tlen += taosEncodeString(buf, pSma->timezone); + tlen += taosEncodeFixedU16(buf, pSma->nFuncColIds); + tlen += taosEncodeFixedU16(buf, pSma->tagsFilterLen); + tlen += taosEncodeFixedI64(buf, pSma->tableUid); tlen += taosEncodeFixedI64(buf, pSma->interval); tlen += taosEncodeFixedI64(buf, pSma->sliding); - for (col_id_t i = 0; i < pSma->numOfColIds; ++i) { - tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i)); + for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) { + SFuncColIds* funcColIds = pSma->funcColIds + i; + tlen += taosEncodeFixedU16(buf, funcColIds->funcId); + tlen += taosEncodeFixedU16(buf, funcColIds->nColIds); + for (uint16_t j = 0; j < funcColIds->nColIds; ++j) { + tlen += taosEncodeFixedU16(buf, *(funcColIds->colIds + j)); + } } - for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { - tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i)); + if (pSma->tagsFilterLen > 0) { + tlen += taosEncodeString(buf, pSma->tagsFilter); } return tlen; @@ -2010,34 +2046,52 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) { buf = taosDecodeFixedU8(buf, &pSma->intervalUnit); buf = taosDecodeFixedU8(buf, &pSma->slidingUnit); buf = taosDecodeStringTo(buf, pSma->indexName); - buf = taosDecodeFixedU16(buf, &pSma->numOfColIds); - buf = taosDecodeFixedU16(buf, &pSma->numOfFuncIds); - buf = taosDecodeFixedU64(buf, &pSma->tableUid); + buf = taosDecodeStringTo(buf, pSma->timezone); + buf = taosDecodeFixedU16(buf, &pSma->nFuncColIds); + buf = taosDecodeFixedU16(buf, &pSma->tagsFilterLen); + buf = taosDecodeFixedI64(buf, &pSma->tableUid); buf = taosDecodeFixedI64(buf, &pSma->interval); buf = taosDecodeFixedI64(buf, &pSma->sliding); - if (pSma->numOfColIds > 0) { - pSma->colIds = (col_id_t*)calloc(pSma->numOfColIds, sizeof(STSma)); - if (pSma->colIds == NULL) { + if (pSma->nFuncColIds > 0) { + pSma->funcColIds = (SFuncColIds*)calloc(pSma->nFuncColIds, sizeof(SFuncColIds)); + if (pSma->funcColIds == NULL) { + tdDestroyTSma(pSma); return NULL; } - for (uint16_t i = 0; i < pSma->numOfColIds; ++i) { - buf = taosDecodeFixedU16(buf, pSma->colIds + i); + for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) { + SFuncColIds* funcColIds = pSma->funcColIds + i; + buf = taosDecodeFixedU16(buf, &funcColIds->funcId); + buf = taosDecodeFixedU16(buf, &funcColIds->nColIds); + if (funcColIds->nColIds > 0) { + funcColIds->colIds = (col_id_t*)calloc(funcColIds->nColIds, sizeof(col_id_t)); + if (funcColIds->colIds != NULL) { + for (uint16_t j = 0; j < funcColIds->nColIds; ++j) { + buf = taosDecodeFixedU16(buf, funcColIds->colIds + j); + } + } else { + tdDestroyTSma(pSma); + return NULL; + } + } else { + funcColIds->colIds = NULL; + } } } else { - pSma->colIds = NULL; + pSma->funcColIds = NULL; } - if (pSma->numOfFuncIds > 0) { - pSma->funcIds = (uint16_t*)calloc(pSma->numOfFuncIds, sizeof(STSma)); - if (pSma->funcIds == NULL) { + if (pSma->tagsFilterLen > 0) { + pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1); + if (pSma->tagsFilter != NULL) { + buf = taosDecodeStringTo(buf, pSma->tagsFilter); + } else { + tdDestroyTSma(pSma); return NULL; } - for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) { - buf = taosDecodeFixedU16(buf, pSma->funcIds + i); - } + } else { - pSma->funcIds = NULL; + pSma->tagsFilter = NULL; } return buf; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6831397a9b..03f8daad42 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -150,6 +150,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index bf48a8523c..d04e9f817e 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,15 +113,16 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_OFFSET = 10, - SDB_SUBSCRIBE = 11, - SDB_CONSUMER = 12, - SDB_TOPIC = 13, - SDB_VGROUP = 14, - SDB_STB = 15, - SDB_DB = 16, - SDB_FUNC = 17, - SDB_MAX = 18 + SDB_STREAM = 10, + SDB_OFFSET = 11, + SDB_SUBSCRIBE = 12, + SDB_CONSUMER = 13, + SDB_TOPIC = 14, + SDB_VGROUP = 15, + SDB_STB = 16, + SDB_DB = 17, + SDB_FUNC = 18, + SDB_MAX = 19 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index ccbeb00bfd..68ca9eff17 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -24,6 +24,7 @@ extern "C" { #include #include "taosdef.h" #include "trpc.h" +#include "wal.h" typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; @@ -87,25 +88,22 @@ typedef struct SSyncFSM { } SSyncFSM; +struct SSyncRaftEntry; +typedef struct SSyncRaftEntry SSyncRaftEntry; + // abstract definition of log store in raft // SWal implements it typedef struct SSyncLogStore { void* data; // append one log entry - int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SRpcMsg* pBuf); + int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); - // get one log entry, user need to free pBuf->data - int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SRpcMsg* pBuf); + // get one log entry, user need to free pEntry->pCont + SSyncRaftEntry* (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index); - // update log store commit index with "index" - int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); - - // truncate log with index, entries after the given index (>index) will be deleted - int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index); - - // return commit index of log - SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + // truncate log with index, entries after the given index (>=index) will be deleted + int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex); // return index of last entry SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore); @@ -113,6 +111,12 @@ typedef struct SSyncLogStore { // return term of last entry SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore); + // update log store commit index with "index" + int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index); + + // return commit index of log + SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); + } SSyncLogStore; // raft need to persist two variables in storage: currentTerm, voteFor @@ -134,7 +138,7 @@ typedef struct SSyncInfo { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - char walPath[TSDB_FILENAME_LEN]; + SWal* pWal; SSyncFSM* pFsm; void* rpcClient; @@ -153,7 +157,7 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo* pSyncInfo); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg); -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); ESyncState syncGetMyRole(int64_t rid); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole); diff --git a/include/os/osSleep.h b/include/os/osSleep.h index 686bdd292e..feb9472995 100644 --- a/include/os/osSleep.h +++ b/include/os/osSleep.h @@ -20,7 +20,17 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define Sleep SLEEP_FUNC_TAOS_FORBID + #define sleep SLEEP_FUNC_TAOS_FORBID + #define usleep USLEEP_FUNC_TAOS_FORBID + #define nanosleep NANOSLEEP_FUNC_TAOS_FORBID +#endif + +void taosSsleep(int32_t s); void taosMsleep(int32_t ms); +void taosUsleep(int32_t us); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 54a3cfef7b..aec63f3322 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -47,7 +47,6 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes); int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes); -int32_t taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); char *taosGetCmdlineByPID(int32_t pid); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 8554c4ad12..f130e9d8f1 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -20,11 +20,23 @@ extern "C" { #endif +// If the error is in a third-party library, place this header file under the third-party library header file. +#ifndef ALLOW_FORBID_FUNC + #define popen POPEN_FUNC_TAOS_FORBID + #define pclose PCLOSE_FUNC_TAOS_FORBID + #define tcsetattr TCSETATTR_FUNC_TAOS_FORBID + #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID +#endif + +int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize); void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); int32_t taosSetConsoleEcho(bool on); +void setTerminalMode(); +int32_t getOldTerminalMode(); +void resetTerminalMode(); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 5d42646238..3a1343b384 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -267,9 +267,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) -#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) - +// mnode-stream +#define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) +#define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) +#define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/include/util/tdef.h b/include/util/tdef.h index a61735ba46..a53b81894a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -207,10 +207,11 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_INDEX_NAME_LEN 32 +#define TSDB_INDEX_NAME_LEN 33 // 32 + 1 '\0' #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_STREAM_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2cd51e9443..7163444719 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -450,7 +450,7 @@ static void hbStopThread() { } while (2 != atomic_load_8(&clientHbMgr.threadStop)) { - usleep(10); + taosUsleep(10); } tscDebug("hb thread stopped"); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 5bcc287116..09a0233e04 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -343,3 +343,77 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // TODO } + +TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) { + // TODO + return NULL; +} + +TAOS_RES *taos_consume(TAOS_SUB *tsub) { + // TODO + return NULL; +} + +void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { + // TODO +} + +TAOS_STMT* taos_stmt_init(TAOS* taos) { + // TODO + return NULL; +} + +int taos_stmt_close(TAOS_STMT* stmt) { + // TODO + return -1; +} + +int taos_stmt_execute(TAOS_STMT* stmt) { + // TODO + return -1; +} + +char *taos_stmt_errstr(TAOS_STMT *stmt) { + // TODO + return NULL; +} + +int taos_stmt_affected_rows(TAOS_STMT* stmt) { + // TODO + return -1; +} + +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { + // TODO + return NULL; +} + +int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { + // TODO + return -1; +} + +int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { + // TODO + return -1; +} + +int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) { + // TODO + return -1; +} + +int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) { + // TODO + return -1; +} + +int taos_stmt_add_batch(TAOS_STMT* stmt) { + // TODO + return -1; +} + +int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { + // TODO + return -1; +} diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 04d80027ac..846329f0f4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -511,7 +511,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, topicName, T_NAME_TABLE); - SMCreateTopicReq req = { + SCMCreateTopicReq req = { .igExists = 1, .physicalPlan = (char*)pStr, .sql = (char*)sql, @@ -519,13 +519,13 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i }; tNameExtractFullName(&name, req.name); - int tlen = tSerializeMCreateTopicReq(NULL, 0, &req); + int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req); void* buf = malloc(tlen); if (buf == NULL) { goto _return; } - tSerializeMCreateTopicReq(buf, tlen, &req); + tSerializeSCMCreateTopicReq(buf, tlen, &req); /*printf("formatted: %s\n", dagStr);*/ pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; @@ -1146,13 +1146,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); /*printf("over1\n");*/ - usleep(blocking_time * 1000); + taosMsleep(blocking_time); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { /*printf("over2\n");*/ - usleep(blocking_time * 1000); + taosMsleep(blocking_time); return NULL; } @@ -1165,14 +1165,14 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); - usleep(blocking_time * 1000); + taosMsleep(blocking_time); return NULL; } SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam)); if (param == NULL) { ASSERT(false); - usleep(blocking_time * 1000); + taosMsleep(blocking_time); return NULL; } param->tmq = tmq; @@ -1204,7 +1204,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (tmq_message == NULL) { if (beginVgIdx == pTopic->nextVgIdx) { - usleep(blocking_time * 1000); + taosMsleep(blocking_time); } else { continue; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4c8dafcf71..ff853145fa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1988,7 +1988,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR return 0; } -int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopicReq *pReq) { +int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) { int32_t sqlLen = 0; int32_t physicalPlanLen = 0; int32_t logicalPlanLen = 0; @@ -2016,7 +2016,7 @@ int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopic return tlen; } -int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq *pReq) { +int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) { int32_t sqlLen = 0; int32_t physicalPlanLen = 0; int32_t logicalPlanLen = 0; @@ -2045,13 +2045,13 @@ int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq return 0; } -void tFreeSMCreateTopicReq(SMCreateTopicReq *pReq) { +void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { tfree(pReq->sql); tfree(pReq->physicalPlan); tfree(pReq->logicalPlan); } -int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) { +int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2064,7 +2064,7 @@ int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopi return tlen; } -int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) { +int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2655,3 +2655,43 @@ void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) { return buf; } + +int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1; + if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1; + if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { + tfree(pReq->sql); + tfree(pReq->physicalPlan); + tfree(pReq->logicalPlan); +} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 3856cd3ff3..9ed6104140 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -84,6 +84,7 @@ typedef enum { TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, TRN_TYPE_COMMIT_OFFSET = 1018, + TRN_TYPE_CREATE_STREAM = 1019, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -679,7 +680,23 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons } typedef struct { -} SStreamScheduler; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + SRWLatch lock; + int8_t status; + // int32_t sqlLen; + char* sql; + char* logicalPlan; + char* physicalPlan; +} SStreamObj; + +int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); +int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj); typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h new file mode 100644 index 0000000000..b1145b0c6b --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_STREAM_H_ +#define _TD_MND_STREAM_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitStream(SMnode *pMnode); +void mndCleanupStream(SMnode *pMnode); + +SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); +void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); + +SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); +SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_STREAM_H_*/ diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c new file mode 100644 index 0000000000..6e8d9aa79f --- /dev/null +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndDef.h" + +int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { + if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + return pEncoder->pos; +} + +int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { + if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1; + if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1; + if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1; + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c new file mode 100644 index 0000000000..54ad9cd7e2 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndStream.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_STREAM_VER_NUMBER 1 +#define MND_STREAM_RESERVE_SIZE 64 + +static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); +static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); +static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); +static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq); +/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/ +/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/ +static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq); +static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); + +int32_t mndInitStream(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_STREAM, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStreamActionEncode, + .decodeFp = (SdbDecodeFp)mndStreamActionDecode, + .insertFp = (SdbInsertFp)mndStreamActionInsert, + .updateFp = (SdbUpdateFp)mndStreamActionUpdate, + .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); + /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ + /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TP, mndGetStreamMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TP, mndRetrieveStream); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TP, mndCancelGetNextStream); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupStream(SMnode *pMnode) {} + +SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + SCoder encoder; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); + if (tEncodeSStreamObj(NULL, pStream) < 0) { + tCoderClear(&encoder); + goto STREAM_ENCODE_OVER; + } + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + + int32_t size = sizeof(int32_t) + tlen + MND_STREAM_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size); + if (pRaw == NULL) goto STREAM_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto STREAM_ENCODE_OVER; + + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); + if (tEncodeSStreamObj(NULL, pStream) < 0) { + tCoderClear(&encoder); + goto STREAM_ENCODE_OVER; + } + tCoderClear(&encoder); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, STREAM_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, STREAM_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, STREAM_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +STREAM_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream); + return pRaw; +} + +SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER; + + if (sver != MND_STREAM_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto STREAM_DECODE_OVER; + } + + int32_t size = sizeof(SStreamObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto STREAM_DECODE_OVER; + + SStreamObj *pStream = sdbGetRowObj(pRow); + if (pStream == NULL) goto STREAM_DECODE_OVER; + + int32_t tlen; + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto STREAM_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER); + + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER); + if (tDecodeSStreamObj(&decoder, pStream) < 0) { + goto STREAM_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +STREAM_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream); + return pRow; +} + +static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) { + mTrace("stream:%s, perform insert action", pStream->name); + return 0; +} + +static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { + mTrace("stream:%s, perform delete action", pStream->name); + return 0; +} + +static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) { + mTrace("stream:%s, perform update action", pOldStream->name); + atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime); + atomic_exchange_32(&pOldStream->version, pNewStream->version); + + taosWLockLatch(&pOldStream->lock); + + // TODO handle update + + taosWUnLockLatch(&pOldStream->lock); + return 0; +} + +SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { + SSdb *pSdb = pMnode->pSdb; + SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName); + if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + } + return pStream; +} + +void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pStream); +} + +static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { + SName name = {0}; + tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_STREAM_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { + if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; + return -1; + } + return 0; +} + +static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { + mDebug("stream:%s to create", pCreate->name); + SStreamObj streamObj = {0}; + tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); + tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN); + streamObj.createTime = taosGetTimestampMs(); + streamObj.updateTime = streamObj.createTime; + streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); + streamObj.dbUid = pDb->uid; + streamObj.version = 1; + streamObj.sql = pCreate->sql; + streamObj.physicalPlan = pCreate->physicalPlan; + streamObj.logicalPlan = pCreate->logicalPlan; + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + if (pTrans == NULL) { + mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SStreamObj *pStream = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCMCreateStreamReq createStreamReq = {0}; + + if (tDeserializeSCMCreateStreamReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createStreamReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_STREAM_OVER; + } + + mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql); + + if (mndCheckCreateStreamReq(&createStreamReq) != 0) { + mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); + goto CREATE_STREAM_OVER; + } + + pStream = mndAcquireStream(pMnode, createStreamReq.name); + if (pStream != NULL) { + if (createStreamReq.igExists) { + mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name); + code = 0; + goto CREATE_STREAM_OVER; + } else { + terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + goto CREATE_STREAM_OVER; + } + } else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) { + goto CREATE_STREAM_OVER; + } + + pDb = mndAcquireDbByStream(pMnode, createStreamReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto CREATE_STREAM_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_STREAM_OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto CREATE_STREAM_OVER; + } + + code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_STREAM_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); + } + + mndReleaseStream(pMnode, pStream); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + tFreeSCMCreateStreamReq(&createStreamReq); + return code; +} + +static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { + SSdb *pSdb = pMnode->pSdb; + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfStreams = 0; + void *pIter = NULL; + while (1) { + SStreamObj *pStream = NULL; + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + + if (pStream->dbUid == pDb->uid) { + numOfStreams++; + } + + sdbRelease(pSdb, pStream); + } + + *pNumOfStreams = numOfStreams; + mndReleaseDb(pMnode, pDb); + return 0; +} + +static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchemas; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = pShow->bytes[cols]; + cols++; + + pMeta->numOfColumns = cols; + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_STREAM); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbName, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SStreamObj *pStream = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[TSDB_DB_FNAME_LEN] = {0}; + + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + + tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); + if (pShow->pIter == NULL) break; + + if (pStream->dbUid != pDb->uid) { + if (strncmp(pStream->name, prefix, prefixLen) != 0) { + mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid); + } + + sdbRelease(pSdb, pStream); + continue; + } + + cols = 0; + + char streamName[TSDB_TABLE_NAME_LEN] = {0}; + tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, streamName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pStream->createTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]); + cols++; + + numOfRows++; + sdbRelease(pSdb, pStream); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 9822550ee5..7d7c5f9975 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE #include "mndTopic.h" #include "mndAuth.h" #include "mndDb.h" @@ -229,7 +228,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq return pDrop; } -static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) { +static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; return -1; @@ -237,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) { return 0; } -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -278,14 +277,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq } static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - int32_t code = -1; - SMqTopicObj *pTopic = NULL; - SDbObj *pDb = NULL; - SUserObj *pUser = NULL; - SMCreateTopicReq createTopicReq = {0}; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SMqTopicObj *pTopic = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCMCreateTopicReq createTopicReq = {0}; - if (tDeserializeSMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) { + if (tDeserializeSCMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto CREATE_TOPIC_OVER; } @@ -338,7 +337,7 @@ CREATE_TOPIC_OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); - tFreeSMCreateTopicReq(&createTopicReq); + tFreeSCMCreateTopicReq(&createTopicReq); return code; } @@ -409,35 +408,6 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { return 0; } -#if 0 -static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { - SSdb *pSdb = pMnode->pSdb; - SDbObj *pDb = mndAcquireDb(pMnode, dbName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } - - int32_t numOfTopics = 0; - void *pIter = NULL; - while (1) { - SMqTopicObj *pTopic = NULL; - pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); - if (pIter == NULL) break; - - if (pTopic->dbUid == pDb->uid) { - numOfTopics++; - } - - sdbRelease(pSdb, pTopic); - } - - *pNumOfTopics = numOfTopics; - mndReleaseDb(pMnode, pDb); - return 0; -} -#endif - static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index 8a4e17d054..f58d0a6771 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -61,16 +61,16 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { } void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql, int32_t* pContLen) { - SMCreateTopicReq createReq = {0}; + SCMCreateTopicReq createReq = {0}; strcpy(createReq.name, topicName); createReq.igExists = 0; createReq.sql = (char*)sql; createReq.physicalPlan = (char*)"physicalPlan"; createReq.logicalPlan = (char*)"logicalPlan"; - int32_t contLen = tSerializeMCreateTopicReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeMCreateTopicReq(pReq, contLen, &createReq); + tSerializeSCMCreateTopicReq(pReq, contLen, &createReq); *pContLen = contLen; return pReq; @@ -100,9 +100,7 @@ TEST_F(MndTestTopic, 01_Create_Topic) { ASSERT_EQ(pRsp->code, 0); } - { - test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, ""); - } + { test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, ""); } { int32_t contLen = 0; diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index aff82e4ae7..5851e18478 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -20,6 +20,7 @@ #include "tlog.h" #include "tmsg.h" +#include "tqueue.h" #include "trpc.h" #include "snode.h" @@ -28,12 +29,51 @@ extern "C" { #endif +enum { + STREAM_STATUS__READY = 1, + STREAM_STATUS__STOPPED, + STREAM_STATUS__CREATING, + STREAM_STATUS__STOPING, + STREAM_STATUS__RESUMING, + STREAM_STATUS__DELETING, +}; + +enum { + STREAM_RUNNER__RUNNING = 1, + STREAM_RUNNER__STOP, +}; + typedef struct SSnode { SSnodeOpt cfg; } SSnode; +typedef struct { + int64_t streamId; + int32_t IdxInLevel; + int32_t level; +} SStreamInfo; + +typedef struct { + SStreamInfo meta; + int8_t status; + void* executor; + STaosQueue* queue; + void* stateStore; + // storage handle +} SStreamRunner; + +typedef struct { + SHashObj* pHash; +} SStreamMeta; + +int32_t sndCreateStream(); +int32_t sndDropStream(); + +int32_t sndStopStream(); +int32_t sndResumeStream(); + #ifdef __cplusplus } #endif -#endif /*_TD_SNODE_INT_H_*/ \ No newline at end of file +#endif /*_TD_SNODE_INT_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h index 173e991631..a7275f57fd 100644 --- a/source/dnode/vnode/src/inc/tsdbFS.h +++ b/source/dnode/vnode/src/inc/tsdbFS.h @@ -43,12 +43,30 @@ typedef struct { STsdbFSMeta meta; // FS meta SArray * df; // data file array - // SArray * v2f100.tsma.index_name + // SArray * v2t100.index_name - SArray * smaf; // sma data file array v2f1900.tsma.index_name + SArray * smaf; // sma data file array v2t1900.index_name } SFSStatus; -typedef struct { +/** + * @brief Directory structure of .tsma data files. + * + * root@cary /vnode2/tsdb $ tree .tsma/ + * .tsma/ + * ├── v2t100.index_name_1 + * ├── v2t101.index_name_1 + * ├── v2t102.index_name_1 + * ├── v2t1900.index_name_3 + * ├── v2t1901.index_name_3 + * ├── v2t1902.index_name_3 + * ├── v2t200.index_name_2 + * ├── v2t201.index_name_2 + * └── v2t202.index_name_2 + * + * 0 directories, 9 files + */ + + typedef struct { pthread_rwlock_t lock; SFSStatus *cstatus; // current status diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 1034ae015a..d6ce33c389 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -56,10 +56,11 @@ typedef enum { TSDB_FILE_SMAL, // .smal(Block-wise SMA) TSDB_FILE_MAX, // TSDB_FILE_META, // meta - TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA - TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA -} TSDB_FILE_T; + TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA + TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA +} E_TSDB_FILE_T; +typedef int32_t TSDB_FILE_T; typedef enum { TSDB_FS_VER_0 = 0, TSDB_FS_VER_MAX, diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index 6e4ad909ae..1a5ccbdc21 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -31,7 +31,7 @@ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeW int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg); int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); -int32_t tsdbFreeSmaState(SSmaStat *pSmaStat); +int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); // internal func diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00e97c7b61..b756cc9862 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -23,8 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = { "smal", // TSDB_FILE_SMAL "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META - "tsma", // TSDB_FILE_TSMA - "rsma", // TSDB_FILE_RSMA + "sma", // TSDB_FILE_TSMA(directory name) + "sma", // TSDB_FILE_RSMA(directory name) }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 1b3e00f090..0c82911226 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -89,7 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { tsdbFreeFS(pTsdb->fs); - tsdbFreeSmaState(pTsdb->pSmaStat); + tsdbDestroySmaState(pTsdb->pSmaStat); tfree(pTsdb->path); free(pTsdb); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index c5f1261282..51abcbd9dd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -128,7 +128,7 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { return pItem; } -int32_t tsdbFreeSmaState(SSmaStat *pSmaStat) { +int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { if (pSmaStat) { // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL); @@ -203,6 +203,23 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) { return TSDB_CODE_SUCCESS; } +static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, const char *indexName, void *timeWindow) { + SSmaStatItem *pItem = NULL; + + if (pTsdb->pSmaStat && pTsdb->pSmaStat->smaStatItems) { + pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName)); + } + + if (pItem != NULL) { + // TODO: reset time windows for the sma data blocks + while (true) { + TSKEY thisWindow = 0; + taosHashRemove(pItem->expiredWindows, &thisWindow, sizeof(thisWindow)); + } + } + return TSDB_CODE_SUCCESS; +} + /** * @brief Judge the tSma storage level * @@ -387,7 +404,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, i static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) { pSmaH->pTsdb = pTsdb; pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision); - pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); + // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); } static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel, @@ -495,6 +512,9 @@ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) { return terrno; } + // reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, param->indexName, &pData->tsWindow); + return TSDB_CODE_SUCCESS; } @@ -542,6 +562,10 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) { TASSERT(0); return TSDB_CODE_INVALID_PARA; } + + // reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, param->tsma.indexName, &pData->tsWindow); + // Step 4: finish return TSDB_CODE_SUCCESS; } @@ -558,7 +582,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) { static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) { pSmaH->pTsdb = pTsdb; pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision); - pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); + // pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t); } /** diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index f3a61bdfa4..5a5c5e1530 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -20,6 +20,7 @@ #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -41,17 +42,20 @@ TEST(testCase, tSmaEncodeDecodeTest) { tSma.slidingUnit = TD_TIME_UNIT_HOUR; tSma.sliding = 0; tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN); + tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN); tSma.tableUid = 1234567890; - tSma.numOfColIds = 2; - tSma.numOfFuncIds = 5; // sum/min/max/avg/last - tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t)); - tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t)); - - for (int32_t i = 0; i < tSma.numOfColIds; ++i) { - *(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); - } - for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) { - *(tSma.funcIds + i) = (i + 2); + tSma.nFuncColIds = 5; + tSma.funcColIds = (SFuncColIds *)calloc(tSma.nFuncColIds, sizeof(SFuncColIds)); + ASSERT(tSma.funcColIds != NULL); + for (int32_t n = 0; n < tSma.nFuncColIds; ++n) { + SFuncColIds *funcColIds = tSma.funcColIds + n; + funcColIds->funcId = n; + funcColIds->nColIds = 10; + funcColIds->colIds = (col_id_t *)calloc(funcColIds->nColIds, sizeof(col_id_t)); + ASSERT(funcColIds->colIds != NULL); + for (int32_t i = 0; i < funcColIds->nColIds; ++i) { + *(funcColIds->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); + } } STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma}; @@ -80,16 +84,21 @@ TEST(testCase, tSmaEncodeDecodeTest) { EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit); EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit); EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName); - EXPECT_EQ(pSma->numOfColIds, qSma->numOfColIds); - EXPECT_EQ(pSma->numOfFuncIds, qSma->numOfFuncIds); + EXPECT_STRCASEEQ(pSma->timezone, qSma->timezone); + EXPECT_EQ(pSma->nFuncColIds, qSma->nFuncColIds); EXPECT_EQ(pSma->tableUid, qSma->tableUid); EXPECT_EQ(pSma->interval, qSma->interval); EXPECT_EQ(pSma->sliding, qSma->sliding); - for (uint32_t j = 0; j < pSma->numOfColIds; ++j) { - EXPECT_EQ(*(col_id_t *)(pSma->colIds + j), *(col_id_t *)(qSma->colIds + j)); - } - for (uint32_t j = 0; j < pSma->numOfFuncIds; ++j) { - EXPECT_EQ(*(uint16_t *)(pSma->funcIds + j), *(uint16_t *)(qSma->funcIds + j)); + EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); + EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); + for (uint32_t j = 0; j < pSma->nFuncColIds; ++j) { + SFuncColIds *pFuncColIds = pSma->funcColIds + j; + SFuncColIds *qFuncColIds = qSma->funcColIds + j; + EXPECT_EQ(pFuncColIds->funcId, qFuncColIds->funcId); + EXPECT_EQ(pFuncColIds->nColIds, qFuncColIds->nColIds); + for (uint32_t k = 0; k < pFuncColIds->nColIds; ++k) { + EXPECT_EQ(*(pFuncColIds->colIds + k), *(qFuncColIds->colIds + k)); + } } } @@ -99,9 +108,11 @@ TEST(testCase, tSmaEncodeDecodeTest) { } TEST(testCase, tSma_DB_Put_Get_Del_Test) { - const char *smaIndexName1 = "sma_index_test_1"; - const char *smaIndexName2 = "sma_index_test_2"; - const char *smaTestDir = "./smaTest"; + const char * smaIndexName1 = "sma_index_test_1"; + const char * smaIndexName2 = "sma_index_test_2"; + const char * timeZone = "Asia/Shanghai"; + const char * tagsFilter = "I'm tags filter"; + const char * smaTestDir = "./smaTest"; const uint64_t tbUid = 1234567890; const uint32_t nCntTSma = 2; // encode @@ -112,21 +123,27 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { tSma.slidingUnit = TD_TIME_UNIT_HOUR; tSma.sliding = 0; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); + tstrncpy(tSma.timezone, timeZone, TD_TIMEZONE_LEN); tSma.tableUid = tbUid; - tSma.numOfColIds = 2; - tSma.numOfFuncIds = 5; // sum/min/max/avg/last - tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t)); - tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t)); - - for (int32_t i = 0; i < tSma.numOfColIds; ++i) { - *(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); - } - for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) { - *(tSma.funcIds + i) = (i + 2); + tSma.nFuncColIds = 5; + tSma.funcColIds = (SFuncColIds *)calloc(tSma.nFuncColIds, sizeof(SFuncColIds)); + ASSERT(tSma.funcColIds != NULL); + for (int32_t n = 0; n < tSma.nFuncColIds; ++n) { + SFuncColIds *funcColIds = tSma.funcColIds + n; + funcColIds->funcId = n; + funcColIds->nColIds = 10; + funcColIds->colIds = (col_id_t *)calloc(funcColIds->nColIds, sizeof(col_id_t)); + ASSERT(funcColIds->colIds != NULL); + for (int32_t i = 0; i < funcColIds->nColIds; ++i) { + *(funcColIds->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID); + } } + tSma.tagsFilterLen = strlen(tagsFilter); + tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1); + tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); SMeta * pMeta = NULL; - STSma * pSmaCfg = &tSma; + STSma * pSmaCfg = &tSma; const SMetaCfg *pMetaCfg = &defaultMetaOptions; taosRemoveDir(smaTestDir); @@ -151,6 +168,8 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName1); assert(qSmaCfg != NULL); printf("name1 = %s\n", qSmaCfg->indexName); + printf("timezone1 = %s\n", qSmaCfg->timezone); + printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); @@ -159,6 +178,8 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName2); assert(qSmaCfg != NULL); printf("name2 = %s\n", qSmaCfg->indexName); + printf("timezone2 = %s\n", qSmaCfg->timezone); + printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); EXPECT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); @@ -169,7 +190,7 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { assert(pSmaCur != NULL); uint32_t indexCnt = 0; while (1) { - const char* indexName = metaSmaCursorNext(pSmaCur); + const char *indexName = metaSmaCursorNext(pSmaCur); if (indexName == NULL) { break; } @@ -184,10 +205,14 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { assert(pSW != NULL); EXPECT_EQ(pSW->number, nCntTSma); EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); + EXPECT_STRCASEEQ(pSW->tSma->timezone, timeZone); + EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); EXPECT_EQ(pSW->tSma->tableUid, tSma.tableUid); EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); + EXPECT_STRCASEEQ((pSW->tSma + 1)->timezone, timeZone); + EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); EXPECT_EQ((pSW->tSma + 1)->tableUid, tSma.tableUid); - + tdDestroyTSmaWrapper(pSW); tfree(pSW); @@ -211,9 +236,9 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) { #if 0 TEST(testCase, tSmaInsertTest) { - STSma tSma = {0}; - STSmaData* pSmaData = NULL; - STsdb tsdb = {0}; + STSma tSma = {0}; + STSmaData *pSmaData = NULL; + STsdb tsdb = {0}; // init tSma.intervalUnit = TD_TIME_UNIT_DAY; @@ -226,7 +251,7 @@ TEST(testCase, tSmaInsertTest) { int32_t dataLen = numOfColIds * numOfBlocks * blockSize; - pSmaData = (STSmaData*)malloc(sizeof(STSmaData) + dataLen); + pSmaData = (STSmaData *)malloc(sizeof(STSmaData) + dataLen); ASSERT_EQ(pSmaData != NULL, true); pSmaData->tableUid = 3232329230; pSmaData->numOfColIds = numOfColIds; @@ -234,7 +259,7 @@ TEST(testCase, tSmaInsertTest) { pSmaData->dataLen = dataLen; pSmaData->tsWindow.skey = 1640000000; pSmaData->tsWindow.ekey = 1645788649; - pSmaData->colIds = (col_id_t*)malloc(sizeof(col_id_t) * numOfColIds); + pSmaData->colIds = (col_id_t *)malloc(sizeof(col_id_t) * numOfColIds); ASSERT_EQ(pSmaData->colIds != NULL, true); for (int32_t i = 0; i < numOfColIds; ++i) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 7236dfc98e..e1ccb03c66 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2705,7 +2705,7 @@ void catalogDestroy(void) { tsem_post(&gCtgMgmt.sem); while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { - usleep(1); + taosUsleep(1); } CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 00f6a508b7..cc0e5bb1a9 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -723,7 +723,7 @@ void *ctgTestGetDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(taosRand() % 5); + taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); @@ -747,7 +747,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(taosRand() % 5); + taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -771,7 +771,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(taosRand() % 5); + taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -801,7 +801,7 @@ void *ctgTestGetCtableMetaThread(void *param) { tfree(tbMeta); if (ctgTestEnableSleep) { - usleep(taosRand() % 5); + taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { @@ -838,7 +838,7 @@ void *ctgTestSetCtableMetaThread(void *param) { } if (ctgTestEnableSleep) { - usleep(taosRand() % 5); + taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -880,7 +880,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.epSet.numOfEps, 3); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) { - usleep(50000); + taosMsleep(50); } ctgTestSetRspTableMeta(); @@ -901,7 +901,7 @@ TEST(tableMeta, normalTable) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -949,7 +949,7 @@ TEST(tableMeta, normalTable) { allDbNum += dbNum; allStbNum += stbNum; - sleep(2); + taosSsleep(2); } ASSERT_EQ(allDbNum, 1); @@ -996,7 +996,7 @@ TEST(tableMeta, childTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1058,7 +1058,7 @@ TEST(tableMeta, childTableCase) { allDbNum += dbNum; allStbNum += stbNum; - sleep(2); + taosSsleep(2); } ASSERT_EQ(allDbNum, 1); @@ -1105,7 +1105,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1132,7 +1132,7 @@ TEST(tableMeta, superTableCase) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1181,7 +1181,7 @@ TEST(tableMeta, superTableCase) { allDbNum += dbNum; allStbNum += stbNum; - sleep(2); + taosSsleep(2); } ASSERT_EQ(allDbNum, 1); @@ -1230,7 +1230,7 @@ TEST(tableMeta, rmStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1244,7 +1244,7 @@ TEST(tableMeta, rmStbMeta) { int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM); if (n || m) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1300,7 +1300,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (0 == n) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1320,7 +1320,7 @@ TEST(tableMeta, updateStbMeta) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -1392,7 +1392,7 @@ TEST(refreshGetMeta, normal2normal) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1410,7 +1410,7 @@ TEST(refreshGetMeta, normal2normal) { tfree(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); @@ -1471,7 +1471,7 @@ TEST(refreshGetMeta, normal2notexist) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1489,7 +1489,7 @@ TEST(refreshGetMeta, normal2notexist) { tfree(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); @@ -1545,7 +1545,7 @@ TEST(refreshGetMeta, normal2child) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1563,7 +1563,7 @@ TEST(refreshGetMeta, normal2child) { tfree(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); @@ -1629,7 +1629,7 @@ TEST(refreshGetMeta, stable2child) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1648,7 +1648,7 @@ TEST(refreshGetMeta, stable2child) { tfree(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } ctgTestCurrentSTableName = ctgTestSTablename; @@ -1714,7 +1714,7 @@ TEST(refreshGetMeta, stable2stable) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1733,7 +1733,7 @@ TEST(refreshGetMeta, stable2stable) { tfree(tableMeta); while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); @@ -1802,7 +1802,7 @@ TEST(refreshGetMeta, child2stable) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } STableMeta *tableMeta = NULL; @@ -1819,7 +1819,7 @@ TEST(refreshGetMeta, child2stable) { tfree(tableMeta); while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) { - usleep(50000); + taosMsleep(50); } ctgTestCurrentSTableName = ctgTestTablename; @@ -2019,7 +2019,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { if (n > 0) { break; } - usleep(50000); + taosMsleep(50); } code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); @@ -2043,7 +2043,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { uint64_t n = 0; ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n); if (n != 3) { - usleep(50000); + taosMsleep(50); } else { break; } @@ -2100,20 +2100,20 @@ TEST(multiThread, getSetRmSameDbVgroup) { pthread_t thread1, thread2; pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); - sleep(1); + taosSsleep(1); pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(ctgTestMTRunSec); + taosSsleep(ctgTestMTRunSec); break; } } ctgTestStop = true; - sleep(1); + taosSsleep(1); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2152,20 +2152,20 @@ TEST(multiThread, getSetRmDiffDbVgroup) { pthread_t thread1, thread2; pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); - sleep(1); + taosSsleep(1); pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(ctgTestMTRunSec); + taosSsleep(ctgTestMTRunSec); break; } } ctgTestStop = true; - sleep(1); + taosSsleep(1); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2203,20 +2203,20 @@ TEST(multiThread, ctableMeta) { pthread_t thread1, thread2; pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); - sleep(1); + taosSsleep(1); pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(ctgTestMTRunSec); + taosSsleep(ctgTestMTRunSec); break; } } ctgTestStop = true; - sleep(2); + taosSsleep(2); catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); @@ -2267,7 +2267,7 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) { - usleep(50000); + taosMsleep(50); } code = catalogGetExpiredDBs(pCtg, &dbs, &num); @@ -2292,7 +2292,7 @@ TEST(rentTest, allRent) { } printf("*************************************************\n"); - sleep(2); + taosSsleep(2); } catalogDestroy(); diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 9ca7442d55..72ce0f7c37 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -63,7 +63,7 @@ int main(int argc, char** argv) { TEST(testCase, async_task_test) { SParam* p = (SParam*)calloc(1, sizeof(SParam)); taosAsyncExec(testPrint, p, NULL); - usleep(5000); + taosMsleep(5); } TEST(testCase, many_async_task_test) { @@ -73,14 +73,14 @@ TEST(testCase, many_async_task_test) { taosAsyncExec(testPrint, p, NULL); } - usleep(10000); + taosMsleep(10); } TEST(testCase, error_in_async_test) { int32_t code = 0; SParam* p = (SParam*) calloc(1, sizeof(SParam)); taosAsyncExec(testPrintError, p, &code); - usleep(1000); + taosMsleep(1); printf("Error code:%d after asynchronously exec function\n", code); } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 8658c4cead..2e262abcd0 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -308,7 +308,7 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (qwtTestEnableSleep) { if (runTime) { - usleep(runTime); + taosUsleep(runTime); } } @@ -590,7 +590,7 @@ void *queryThread(void *param) { qwtBuildQueryReqMsg(&queryRpc); qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc); if (qwtTestEnableSleep) { - usleep(taosRand()%5); + taosUsleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("query:%d\n", n); @@ -612,7 +612,7 @@ void *readyThread(void *param) { qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); if (qwtTestEnableSleep) { - usleep(taosRand()%5); + taosUsleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("ready:%d\n", n); @@ -634,7 +634,7 @@ void *fetchThread(void *param) { qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); if (qwtTestEnableSleep) { - usleep(taosRand()%5); + taosUsleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("fetch:%d\n", n); @@ -656,7 +656,7 @@ void *dropThread(void *param) { qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); if (qwtTestEnableSleep) { - usleep(taosRand()%5); + taosUsleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("drop:%d\n", n); @@ -678,7 +678,7 @@ void *statusThread(void *param) { qwtBuildStatusReqMsg(&statusMsg, &statusRpc); code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); if (qwtTestEnableSleep) { - usleep(taosRand()%5); + taosUsleep(taosRand()%5); } if (++n % qwtTestPrintNum == 0) { printf("status:%d\n", n); @@ -696,7 +696,7 @@ void *qwtclientThread(void *param) { void *mockPointer = (void *)0x1; SRpcMsg queryRpc = {0}; - sleep(1); + taosSsleep(1); while (!qwtTestStop) { qwtTestCaseFinished = false; @@ -705,7 +705,7 @@ void *qwtclientThread(void *param) { qwtPutReqToQueue((void *)0x1, &queryRpc); while (!qwtTestCaseFinished) { - usleep(1); + taosUsleep(1); } @@ -751,7 +751,7 @@ void *queryQueueThread(void *param) { int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { - usleep(delay); + taosUsleep(delay); } } @@ -807,7 +807,7 @@ void *fetchQueueThread(void *param) { int32_t delay = taosRand() % qwtTestReqMaxDelayUsec; if (delay) { - usleep(delay); + taosUsleep(delay); } } @@ -982,21 +982,21 @@ TEST(seqTest, randCase) { qwtBuildReadyReqMsg(&readyMsg, &readyRpc); code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc); if (qwtTestEnableSleep) { - usleep(1); + taosUsleep(1); } } else if (r >= maxr * 2/5 && r < maxr* 3/5) { printf("Fetch,%d\n", t++); qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc); code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc); if (qwtTestEnableSleep) { - usleep(1); + taosUsleep(1); } } else if (r >= maxr * 3/5 && r < maxr * 4/5) { printf("Drop,%d\n", t++); qwtBuildDropReqMsg(&dropMsg, &dropRpc); code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc); if (qwtTestEnableSleep) { - usleep(1); + taosUsleep(1); } } else if (r >= maxr * 4/5 && r < maxr-1) { printf("Status,%d\n", t++); @@ -1004,7 +1004,7 @@ TEST(seqTest, randCase) { code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc); ASSERT_EQ(code, 0); if (qwtTestEnableSleep) { - usleep(1); + taosUsleep(1); } } else { printf("QUIT RAND NOW"); @@ -1042,15 +1042,15 @@ TEST(seqTest, multithreadRand) { while (true) { if (qwtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(qwtTestMTRunSec); + taosSsleep(qwtTestMTRunSec); break; } } qwtTestStop = true; - sleep(3); + taosSsleep(3); qWorkerDestroy(&mgmt); } @@ -1099,9 +1099,9 @@ TEST(rcTest, shortExecshortDelay) { while (true) { if (qwtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(qwtTestMTRunSec); + taosSsleep(qwtTestMTRunSec); break; } } @@ -1113,14 +1113,14 @@ TEST(rcTest, shortExecshortDelay) { break; } - sleep(1); + taosSsleep(1); if (qwtTestCaseFinished) { if (qwtTestQuitThreadNum < 3) { tsem_post(&qwtTestQuerySem); tsem_post(&qwtTestFetchSem); - usleep(10); + taosUsleep(10); } } @@ -1180,9 +1180,9 @@ TEST(rcTest, longExecshortDelay) { while (true) { if (qwtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(qwtTestMTRunSec); + taosSsleep(qwtTestMTRunSec); break; } } @@ -1195,14 +1195,14 @@ TEST(rcTest, longExecshortDelay) { break; } - sleep(1); + taosSsleep(1); if (qwtTestCaseFinished) { if (qwtTestQuitThreadNum < 3) { tsem_post(&qwtTestQuerySem); tsem_post(&qwtTestFetchSem); - usleep(10); + taosUsleep(10); } } @@ -1263,9 +1263,9 @@ TEST(rcTest, shortExeclongDelay) { while (true) { if (qwtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(qwtTestMTRunSec); + taosSsleep(qwtTestMTRunSec); break; } } @@ -1278,14 +1278,14 @@ TEST(rcTest, shortExeclongDelay) { break; } - sleep(1); + taosSsleep(1); if (qwtTestCaseFinished) { if (qwtTestQuitThreadNum < 3) { tsem_post(&qwtTestQuerySem); tsem_post(&qwtTestFetchSem); - usleep(10); + taosUsleep(10); } } @@ -1342,15 +1342,15 @@ TEST(rcTest, dropTest) { while (true) { if (qwtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(qwtTestMTRunSec); + taosSsleep(qwtTestMTRunSec); break; } } qwtTestStop = true; - sleep(3); + taosSsleep(3); qWorkerDestroy(&mgmt); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 0e2ec9f00d..e1bb9a0533 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -345,7 +345,7 @@ void *schtSendRsp(void *param) { break; } - usleep(1000); + taosMsleep(1); } pJob = schAcquireJob(job); @@ -370,7 +370,7 @@ void *schtCreateFetchRspThread(void *param) { int64_t job = *(int64_t *)param; SSchJob* pJob = schAcquireJob(job); - sleep(1); + taosSsleep(1); int32_t code = 0; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp)); @@ -394,7 +394,7 @@ void *schtFetchRspThread(void *aa) { continue; } - usleep(1); + taosUsleep(1); param = (SSchCallbackParam *)calloc(1, sizeof(*param)); @@ -599,7 +599,7 @@ void* schtRunJobThread(void *aa) { void* schtFreeJobThread(void *aa) { while (!schtTestStop) { - usleep(taosRand() % 100); + taosUsleep(taosRand() % 100); schtFreeQueryJob(1); } } @@ -861,15 +861,15 @@ TEST(multiThread, forceFree) { while (true) { if (schtTestDeadLoop) { - sleep(1); + taosSsleep(1); } else { - sleep(schtTestMTRunSec); + taosSsleep(schtTestMTRunSec); break; } } schtTestStop = true; - sleep(3); + taosSsleep(3); } int main(int argc, char** argv) { diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 35d3046d66..5999ef8300 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -25,7 +25,6 @@ extern "C" { #include #include "syncInt.h" #include "syncMessage.h" -#include "syncRaft.h" #include "taosdef.h" // TLA+ Spec diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 75b82aa531..c0c1f76707 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -25,7 +25,6 @@ extern "C" { #include #include "syncInt.h" #include "syncMessage.h" -#include "syncRaft.h" #include "taosdef.h" // TLA+ Spec diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8b77e292c4..2932240ec1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -116,7 +116,8 @@ typedef struct SSyncNode { SyncGroupId vgId; SSyncCfg syncCfg; char path[TSDB_FILENAME_LEN]; - char walPath[TSDB_FILENAME_LEN]; + char raftStorePath[TSDB_FILENAME_LEN * 2]; + SWal* pWal; void* rpcClient; int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg); void* queue; @@ -195,8 +196,6 @@ typedef struct SSyncNode { SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeClose(SSyncNode* pSyncNode); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); @@ -213,6 +212,11 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); +// for debug +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); +void syncNodePrint(char* s, const SSyncNode* pSyncNode); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index 3405f0f6cc..6231cb8399 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -24,8 +24,7 @@ extern "C" { #include #include #include "cJSON.h" -#include "sync.h" -#include "syncRaftEntry.h" +#include "syncInt.h" #include "taosdef.h" // encode as uint32 @@ -46,6 +45,7 @@ typedef enum ESyncMessageType { // --------------------------------------------- cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg); cJSON* syncRpcUnknownMsg2Json(); +char* syncRpcMsg2Str(SRpcMsg* pRpcMsg); // --------------------------------------------- typedef enum ESyncTimeoutType { @@ -123,12 +123,22 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId); typedef struct SyncClientRequest { uint32_t bytes; uint32_t msgType; - int64_t seqNum; + uint32_t originalRpcType; + uint64_t seqNum; bool isWeak; uint32_t dataLen; char data[]; } SyncClientRequest; +SyncClientRequest* syncClientRequestBuild(uint32_t dataLen); +void syncClientRequestDestroy(SyncClientRequest* pMsg); +void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen); +void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg); +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg); +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg); +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak); + // --------------------------------------------- typedef struct SyncClientRequestReply { uint32_t bytes; diff --git a/source/libs/sync/inc/syncOnMessage.h b/source/libs/sync/inc/syncOnMessage.h index 8eae4fed4d..7cb186a812 100644 --- a/source/libs/sync/inc/syncOnMessage.h +++ b/source/libs/sync/inc/syncOnMessage.h @@ -23,7 +23,6 @@ extern "C" { #include #include #include -#include "syncRaft.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h deleted file mode 100644 index bc5cf26a4c..0000000000 --- a/source/libs/sync/inc/syncRaft.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_LIBS_SYNC_RAFT_H -#define _TD_LIBS_SYNC_RAFT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include "sync.h" -#include "syncMessage.h" -#include "taosdef.h" - -#if 0 - -typedef struct SRaftId { - SyncNodeId addr; - SyncGroupId vgId; -} SRaftId; - -typedef struct SRaft { - SRaftId id; - SSyncFSM* pFsm; - - int32_t (*FpPing)(struct SRaft* ths, const RaftPing* pMsg); - - int32_t (*FpOnPing)(struct SRaft* ths, RaftPing* pMsg); - - int32_t (*FpOnPingReply)(struct SRaft* ths, RaftPingReply* pMsg); - - int32_t (*FpRequestVote)(struct SRaft* ths, const RaftRequestVote* pMsg); - - int32_t (*FpOnRequestVote)(struct SRaft* ths, RaftRequestVote* pMsg); - - int32_t (*FpOnRequestVoteReply)(struct SRaft* ths, RaftRequestVoteReply* pMsg); - - int32_t (*FpAppendEntries)(struct SRaft* ths, const RaftAppendEntries* pMsg); - - int32_t (*FpOnAppendEntries)(struct SRaft* ths, RaftAppendEntries* pMsg); - - int32_t (*FpOnAppendEntriesReply)(struct SRaft* ths, RaftAppendEntriesReply* pMsg); - -} SRaft; - -SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm); - -void raftClose(SRaft* pRaft); - -static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg); - -static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg); - -static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg); - -static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg); - -static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg); - -static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg); - -static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg); - -static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg); - -static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg); - -int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); - -static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); - -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_LIBS_SYNC_RAFT_H*/ diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 516bef4d48..be25675db4 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -24,15 +24,31 @@ extern "C" { #include #include #include "syncInt.h" +#include "syncMessage.h" #include "taosdef.h" typedef struct SSyncRaftEntry { - SyncTerm term; - SyncIndex index; - SSyncBuffer data; - int8_t flag; + uint32_t bytes; + uint32_t msgType; + uint32_t originalRpcType; + uint64_t seqNum; + bool isWeak; + SyncTerm term; + SyncIndex index; + uint32_t dataLen; + char data[]; } SSyncRaftEntry; +SSyncRaftEntry* syncEntryBuild(uint32_t dataLen); +SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index); +void syncEntryDestory(SSyncRaftEntry* pEntry); +char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len); +SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len); +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry); +char* syncEntry2Str(const SSyncRaftEntry* pEntry); +void syncEntryPrint(const SSyncRaftEntry* pEntry); +void syncEntryPrint2(char *s, const SSyncRaftEntry* pEntry); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index ee971062cf..d59b3206b5 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -24,27 +24,47 @@ extern "C" { #include #include #include "syncInt.h" +#include "syncRaftEntry.h" #include "taosdef.h" -int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); +typedef struct SSyncLogStoreData { + SSyncNode* pSyncNode; + SWal* pWal; +} SSyncLogStoreData; -// get one log entry, user need to free pBuf->data -int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf); +SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); -// update log store commit index with "index" -int32_t raftLogUpdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index); +void logStoreDestory(SSyncLogStore* pLogStore); -// truncate log with index, entries after the given index (>index) will be deleted -int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index); +// append one log entry +int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); -// return commit index of log -SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore); +// get one log entry, user need to free pEntry->pCont +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index); + +// truncate log with index, entries after the given index (>=index) will be deleted +int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex); // return index of last entry -SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore); +SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore); // return term of last entry -SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore); +SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore); + +// update log store commit index with "index" +int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index); + +// return commit index of log +SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); + +SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore); + +cJSON* logStore2Json(SSyncLogStore* pLogStore); + +char* logStore2Str(SSyncLogStore* pLogStore); + +// for debug +void logStorePrint(SSyncLogStore* pLogStore); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 591a5b9963..1c25b799b4 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -25,7 +25,6 @@ extern "C" { #include #include "cJSON.h" #include "syncInt.h" -#include "syncRaft.h" #include "taosdef.h" #define RAFT_STORE_BLOCK_SIZE 512 diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 8bb4976de2..fd4ccd5371 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -25,7 +25,6 @@ extern "C" { #include #include "syncInt.h" #include "syncMessage.h" -#include "syncRaft.h" #include "taosdef.h" // TLA+ Spec diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index ab9430b857..bcaf71a541 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -25,7 +25,6 @@ extern "C" { #include #include "syncInt.h" #include "syncMessage.h" -#include "syncRaft.h" #include "taosdef.h" // TLA+ Spec diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 89fcb230fb..611f33a0f2 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -24,7 +24,6 @@ extern "C" { #include #include #include "syncInt.h" -#include "syncRaft.h" #include "taosdef.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot); diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index efd5aae48e..25c26c909d 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -25,7 +25,6 @@ extern "C" { #include #include "syncInt.h" #include "syncMessage.h" -#include "syncRaft.h" #include "taosdef.h" // TLA+ Spec diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index f2c979da99..2bbc1948dd 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -49,6 +49,9 @@ cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); char* syncUtilRaftId2Str(const SRaftId* p); const char* syncUtilState2String(ESyncState state); +bool syncUtilCanPrint(char c); +char* syncUtilprintBin(char* ptr, uint32_t len); +char* syncUtilprintBin2(char* ptr, uint32_t len); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 7cb42c9f87..9d2afb03ca 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -15,7 +15,7 @@ #include "syncIO.h" #include -#include "syncOnMessage.h" +#include "syncMessage.h" #include "tglobal.h" #include "ttimer.h" #include "tutil.h" @@ -220,12 +220,17 @@ static void *syncIOConsumerFunc(void *param) { while (1) { int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); sTrace("syncIOConsumerFunc %d msgs are received", numOfMsgs); - if (numOfMsgs <= 0) break; + if (numOfMsgs <= 0) { + break; + } for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); + + char *s = syncRpcMsg2Str(pRpcMsg); sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen, - (char *)(pRpcMsg->pCont)); + s); + free(s); if (pRpcMsg->msgType == SYNC_PING) { if (io->FpOnSyncPing != NULL) { @@ -247,7 +252,7 @@ static void *syncIOConsumerFunc(void *param) { } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) { - if (io->FpOnSyncRequestVote) { + if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg; pSyncMsg = syncRequestVoteBuild(pRpcMsg->contLen); syncRequestVoteFromRpcMsg(pRpcMsg, pSyncMsg); @@ -256,7 +261,7 @@ static void *syncIOConsumerFunc(void *param) { } } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) { - if (io->FpOnSyncRequestVoteReply) { + if (io->FpOnSyncRequestVoteReply != NULL) { SyncRequestVoteReply *pSyncMsg; pSyncMsg = SyncRequestVoteReplyBuild(); syncRequestVoteReplyFromRpcMsg(pRpcMsg, pSyncMsg); @@ -265,7 +270,7 @@ static void *syncIOConsumerFunc(void *param) { } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) { - if (io->FpOnSyncAppendEntries) { + if (io->FpOnSyncAppendEntries != NULL) { SyncAppendEntries *pSyncMsg; pSyncMsg = syncAppendEntriesBuild(pRpcMsg->contLen); syncAppendEntriesFromRpcMsg(pRpcMsg, pSyncMsg); @@ -274,7 +279,7 @@ static void *syncIOConsumerFunc(void *param) { } } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) { - if (io->FpOnSyncAppendEntriesReply) { + if (io->FpOnSyncAppendEntriesReply != NULL) { SyncAppendEntriesReply *pSyncMsg; pSyncMsg = syncAppendEntriesReplyBuild(); syncAppendEntriesReplyFromRpcMsg(pRpcMsg, pSyncMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3b8d716dbe..5cd7149b72 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -18,8 +18,9 @@ #include "syncAppendEntries.h" #include "syncAppendEntriesReply.h" #include "syncEnv.h" +#include "syncIndexMgr.h" #include "syncInt.h" -#include "syncRaft.h" +#include "syncRaftLog.h" #include "syncRaftStore.h" #include "syncRequestVote.h" #include "syncRequestVoteReply.h" @@ -59,13 +60,32 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; } -void syncStop(int64_t rid) {} +void syncStop(int64_t rid) { + SSyncNode* pSyncNode = NULL; // get pointer from rid + syncNodeClose(pSyncNode); +} int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; } -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak) { return 0; } +int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { + SSyncNode* pSyncNode = NULL; // get pointer from rid + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak); + SRpcMsg rpcMsg; + syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); + pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg); + syncClientRequestDestroy(pSyncMsg); + } else { + sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state)); + return -1; // need define err code !! + } + return 0; +} -ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } +ESyncState syncGetMyRole(int64_t rid) { + SSyncNode* pSyncNode = NULL; // get pointer from rid + return pSyncNode->state; +} void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} @@ -78,7 +98,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->syncCfg = pSyncInfo->syncCfg; memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); - memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath)); + snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s/raft_store.json", pSyncInfo->path); + pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->rpcClient = pSyncInfo->rpcClient; pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; pSyncNode->queue = pSyncInfo->queue; @@ -114,20 +135,27 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init life cycle - // init server vars + // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath); + pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); assert(pSyncNode->pRaftStore != NULL); - // init candidate vars + // init TLA+ candidate vars pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); assert(pSyncNode->pVotesGranted != NULL); pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); assert(pSyncNode->pVotesRespond != NULL); - // init leader vars - pSyncNode->pNextIndex = NULL; - pSyncNode->pMatchIndex = NULL; + // init TLA+ leader vars + pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode); + assert(pSyncNode->pNextIndex != NULL); + pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode); + assert(pSyncNode->pMatchIndex != NULL); + + // init TLA+ log vars + pSyncNode->pLogStore = logStoreCreate(pSyncNode); + assert(pSyncNode->pLogStore != NULL); + pSyncNode->commitIndex = 0; // init ping timer pSyncNode->pPingTimer = NULL; @@ -177,7 +205,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // init by SSyncInfo cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId); cJSON_AddStringToObject(pRoot, "path", pSyncNode->path); - cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath); + snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient); cJSON_AddStringToObject(pRoot, "rpcClient", u64buf); @@ -298,6 +327,13 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +void syncNodePrint(char* s, const SSyncNode* pSyncNode) { + char* ss = syncNode2Str(pSyncNode); + // sTrace("syncNodePrint: %s [len:%lu]| %s", s, strlen(ss), ss); + fprintf(stderr, "syncNodePrint: %s [len:%lu]| %s", s, strlen(ss), ss); + free(ss); +} + int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; syncUtilraftId2EpSet(destRaftId, &epSet); @@ -472,6 +508,8 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { } static void syncNodeEqPingTimer(void* param, void* tmrId) { + sTrace("<-- syncNodeEqPingTimer -->"); + SSyncNode* pSyncNode = (SSyncNode*)param; if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) { SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock), @@ -484,7 +522,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { // reset timer ms // pSyncNode->pingTimerMS += 100; - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { sTrace("syncNodeEqPingTimer: pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", pSyncNode->pingTimerLogicClock, @@ -506,7 +544,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { // reset timer ms pSyncNode->electTimerMS = syncUtilElectRandomMS(); - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { sTrace("syncNodeEqElectTimer: electTimerLogicClock:%lu, electTimerLogicClockUser:%lu", @@ -530,7 +568,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { // reset timer ms // pSyncNode->heartbeatTimerMS += 100; - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager, + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu", diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 14f139a803..baef49d748 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,7 +14,6 @@ */ #include "syncMessage.h" -#include "syncRaft.h" #include "syncUtil.h" #include "tcoding.h" @@ -36,7 +35,8 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { pRoot = syncPingReply2Json(pSyncMsg); } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) { - pRoot = syncRpcUnknownMsg2Json(); + SyncClientRequest* pSyncMsg = (SyncClientRequest*)pRpcMsg->pCont; + pRoot = syncClientRequest2Json(pSyncMsg); } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST_REPLY) { pRoot = syncRpcUnknownMsg2Json(); @@ -76,6 +76,13 @@ cJSON* syncRpcUnknownMsg2Json() { return pJson; } +char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { + cJSON* pJson = syncRpcMsg2Json(pRpcMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + // ---- message process SyncTimeout---- SyncTimeout* syncTimeoutBuild() { uint32_t bytes = sizeof(SyncTimeout); @@ -149,6 +156,7 @@ SyncPing* syncPingBuild(uint32_t dataLen) { pMsg->bytes = bytes; pMsg->msgType = SYNC_PING; pMsg->dataLen = dataLen; + return pMsg; } void syncPingDestroy(SyncPing* pMsg) { @@ -247,6 +255,7 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { pMsg->bytes = bytes; pMsg->msgType = SYNC_PING_REPLY; pMsg->dataLen = dataLen; + return pMsg; } void syncPingReplyDestroy(SyncPingReply* pMsg) { @@ -337,6 +346,73 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId) return pMsg; } +// ---- message process SyncClientRequest---- +SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { + uint32_t bytes = sizeof(SyncClientRequest) + dataLen; + SyncClientRequest* pMsg = malloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->msgType = SYNC_CLIENT_REQUEST; + pMsg->seqNum = 0; + pMsg->isWeak = false; + pMsg->dataLen = dataLen; + return pMsg; +} + +void syncClientRequestDestroy(SyncClientRequest* pMsg) { + if (pMsg != NULL) { + free(pMsg); + } +} + +void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) { + assert(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) { + memcpy(pMsg, buf, len); + assert(len == pMsg->bytes); +} + +void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncClientRequestSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg) { + syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pMsg->isWeak); + cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncClientRequest", pRoot); + return pJson; +} + +SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) { + SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen); + pMsg->originalRpcType = pOriginalRpcMsg->msgType; + pMsg->seqNum = seqNum; + pMsg->isWeak = isWeak; + memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen); + return pMsg; +} + // ---- message process SyncRequestVote---- SyncRequestVote* syncRequestVoteBuild() { uint32_t bytes = sizeof(SyncRequestVote); @@ -344,6 +420,7 @@ SyncRequestVote* syncRequestVoteBuild() { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE; + return pMsg; } void syncRequestVoteDestroy(SyncRequestVote* pMsg) { @@ -429,6 +506,7 @@ SyncRequestVoteReply* SyncRequestVoteReplyBuild() { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_REQUEST_VOTE_REPLY; + return pMsg; } void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { @@ -512,6 +590,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) { pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES; pMsg->dataLen = dataLen; + return pMsg; } void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { @@ -604,6 +683,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild() { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY; + return pMsg; } void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c deleted file mode 100644 index b07c6ea797..0000000000 --- a/source/libs/sync/src/syncRaft.c +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "syncRaft.h" -#include "sync.h" - -#if 0 - -SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm) { - SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft)); - assert(pRaft != NULL); - - pRaft->id = raftId; - pRaft->pFsm = pFsm; - - pRaft->FpPing = doRaftPing; - pRaft->FpOnPing = onRaftPing; - pRaft->FpOnPingReply = onRaftPingReply; - - pRaft->FpRequestVote = doRaftRequestVote; - pRaft->FpOnRequestVote = onRaftRequestVote; - pRaft->FpOnRequestVoteReply = onRaftRequestVoteReply; - - pRaft->FpAppendEntries = doRaftAppendEntries; - pRaft->FpOnAppendEntries = onRaftAppendEntries; - pRaft->FpOnAppendEntriesReply = onRaftAppendEntriesReply; - - return pRaft; -} - -void raftClose(SRaft* pRaft) { - assert(pRaft != NULL); - free(pRaft); -} - -static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg) { return 0; } - -static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg) { return 0; } - -static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg) { return 0; } - -static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg) { return 0; } - -static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg) { return 0; } - -static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg) { return 0; } - -static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg) { return 0; } - -static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg) { return 0; } - -static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg) { return 0; } - -int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } - -static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; } - -#endif \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index e525d3c7c2..959bf49ee7 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -14,3 +14,104 @@ */ #include "syncRaftEntry.h" +#include "syncUtil.h" + +SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { + uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; + SSyncRaftEntry* pEntry = malloc(bytes); + assert(pEntry != NULL); + memset(pEntry, 0, bytes); + pEntry->bytes = bytes; + pEntry->dataLen = dataLen; + return pEntry; +} + +SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { + SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + assert(pEntry != NULL); + + pEntry->msgType = pMsg->msgType; + pEntry->originalRpcType = pMsg->originalRpcType; + pEntry->seqNum = pMsg->seqNum; + pEntry->isWeak = pMsg->isWeak; + pEntry->term = term; + pEntry->index = index; + pEntry->dataLen = pMsg->dataLen; + memcpy(pEntry->data, pMsg->data, pMsg->dataLen); + + return pEntry; +} + +void syncEntryDestory(SSyncRaftEntry* pEntry) { + if (pEntry != NULL) { + free(pEntry); + } +} + +char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { + char* buf = malloc(pEntry->bytes); + assert(buf != NULL); + memcpy(buf, pEntry, pEntry->bytes); + if (len != NULL) { + *len = pEntry->bytes; + } + return buf; +} + +SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SSyncRaftEntry* pEntry = malloc(bytes); + assert(pEntry != NULL); + memcpy(pEntry, buf, len); + assert(len == pEntry->bytes); + return pEntry; +} + +cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { + char u64buf[128]; + + cJSON* pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); + cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); + cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->seqNum); + cJSON_AddStringToObject(pRoot, "seqNum", u64buf); + cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); + cJSON_AddStringToObject(pRoot, "index", u64buf); + cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); + + char* s; + s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data", s); + free(s); + + s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen); + cJSON_AddStringToObject(pRoot, "data2", s); + free(s); + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot); + return pJson; +} + +char* syncEntry2Str(const SSyncRaftEntry* pEntry) { + cJSON* pJson = syncEntry2Json(pEntry); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncEntryPrint(const SSyncRaftEntry* pEntry) { + char* s = syncEntry2Str(pEntry); + sTrace("%s", s); + free(s); +} + +void syncEntryPrint2(char* s, const SSyncRaftEntry* pEntry) { + char* ss = syncEntry2Str(pEntry); + sTrace("%s | %s", s, ss); + free(ss); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index e467057c8f..d177d3ac9b 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,46 +14,160 @@ */ #include "syncRaftLog.h" +#include "wal.h" -int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } +SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { + SSyncLogStore* pLogStore = malloc(sizeof(SSyncLogStore)); + assert(pLogStore != NULL); -// get one log entry, user need to free pBuf->data -int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf) { return 0; } + pLogStore->data = malloc(sizeof(SSyncLogStoreData)); + assert(pLogStore->data != NULL); -// TLA+ Spec -// \* Leader i advances its commitIndex. -// \* This is done as a separate step from handling AppendEntries responses, -// \* in part to minimize atomic regions, and in part so that leaders of -// \* single-server clusters are able to mark entries committed. -// AdvanceCommitIndex(i) == -// /\ state[i] = Leader -// /\ LET \* The set of servers that agree up through index. -// Agree(index) == {i} \cup {k \in Server : -// matchIndex[i][k] >= index} -// \* The maximum indexes for which a quorum agrees -// agreeIndexes == {index \in 1..Len(log[i]) : -// Agree(index) \in Quorum} -// \* New value for commitIndex'[i] -// newCommitIndex == -// IF /\ agreeIndexes /= {} -// /\ log[i][Max(agreeIndexes)].term = currentTerm[i] -// THEN -// Max(agreeIndexes) -// ELSE -// commitIndex[i] -// IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex] -// /\ UNCHANGED <> -// -int32_t raftLogupdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + SSyncLogStoreData* pData = pLogStore->data; + pData->pSyncNode = pSyncNode; + pData->pWal = pSyncNode->pWal; -// truncate log with index, entries after the given index (>index) will be deleted -int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; } + pLogStore->appendEntry = logStoreAppendEntry; + pLogStore->getEntry = logStoreGetEntry; + pLogStore->truncate = logStoreTruncate; + pLogStore->getLastIndex = logStoreLastIndex; + pLogStore->getLastTerm = logStoreLastTerm; + pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; + pLogStore->getCommitIndex = logStoreGetCommitIndex; +} -// return commit index of log -SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore) { return 0; } +void logStoreDestory(SSyncLogStore* pLogStore) { + if (pLogStore != NULL) { + free(pLogStore->data); + free(pLogStore); + } +} + +// append one log entry +int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + + assert(pEntry->index == logStoreLastIndex(pLogStore) + 1); + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(serialized != NULL); + + int code; + code = walWrite(pWal, pEntry->index, pEntry->msgType, serialized, len); + assert(code == 0); + + walFsync(pWal, true); + free(serialized); +} + +// get one log entry, user need to free pEntry->pCont +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SSyncRaftEntry* pEntry; + + SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + walReadWithHandle(pWalHandle, index); + pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len); + assert(pEntry != NULL); + + // need to hold, do not new every time!! + walCloseReadHandle(pWalHandle); + return pEntry; +} + +// truncate log with index, entries after the given index (>=index) will be deleted +int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + walRollback(pWal, fromIndex); +} // return index of last entry -SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore) { return 0; } +SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastIndex = walGetLastVer(pWal); + return lastIndex; +} // return term of last entry -SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore) { return 0; } \ No newline at end of file +SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { + SyncTerm lastTerm = 0; + SSyncRaftEntry* pLastEntry = logStoreGetLastEntry(pLogStore); + if (pLastEntry != NULL) { + lastTerm = pLastEntry->term; + free(pLastEntry); + } + return lastTerm; +} + +// update log store commit index with "index" +int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + walCommit(pWal, index); +} + +// return commit index of log +SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + return pData->pSyncNode->commitIndex; +} + +SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + SyncIndex lastIndex = walGetLastVer(pWal); + + SSyncRaftEntry* pEntry = NULL; + if (lastIndex > 0) { + pEntry = logStoreGetEntry(pLogStore, lastIndex); + } + return pEntry; +} + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128]; + + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + SyncIndex lastIndex = logStoreLastIndex(pLogStore); + for (SyncIndex i = 0; i <= lastIndex; ++i) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} + +char* logStore2Str(SSyncLogStore* pLogStore) { + cJSON* pJson = logStore2Json(pLogStore); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +// for debug +void logStorePrint(SSyncLogStore* pLogStore) { + char* s = logStore2Str(pLogStore); + // sTrace("%s", s); + fprintf(stderr, "logStorePrint: [len:%lu]| %s \n", strlen(s), s); + + free(s); +} \ No newline at end of file diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index da194780ff..42b2bd993b 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,7 +14,6 @@ */ #include "syncSnapshot.h" -#include "syncRaft.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 216dccd62c..b78971bf37 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -148,4 +148,40 @@ const char* syncUtilState2String(ESyncState state) { } else { return "TAOS_SYNC_STATE_UNKNOWN"; } +} + +bool syncUtilCanPrint(char c) { + if (c >= 32 && c <= 126) { + return true; + } else { + return false; + } +} + +char* syncUtilprintBin(char* ptr, uint32_t len) { + char* s = malloc(len + 1); + assert(s != NULL); + memset(s, 0, len + 1); + memcpy(s, ptr, len); + + for (int i = 0; i < len; ++i) { + if (!syncUtilCanPrint(s[i])) { + s[i] = '.'; + } + } + return s; +} + +char* syncUtilprintBin2(char* ptr, uint32_t len) { + uint32_t len2 = len * 4 + 1; + char* s = malloc(len2); + assert(s != NULL); + memset(s, 0, len2); + + char* p = s; + for (int i = 0; i < len; ++i) { + int n = sprintf(p, "%d,", ptr[i]); + p += n; + } + return s; } \ No newline at end of file diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 5a5186c7e2..bfde08ffac 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -15,6 +15,8 @@ add_executable(syncUtilTest "") add_executable(syncVotesGrantedTest "") add_executable(syncVotesRespondTest "") add_executable(syncIndexMgrTest "") +add_executable(syncLogStoreTest "") +add_executable(syncEntryTest "") target_sources(syncTest @@ -85,6 +87,14 @@ target_sources(syncIndexMgrTest PRIVATE "syncIndexMgrTest.cpp" ) +target_sources(syncLogStoreTest + PRIVATE + "syncLogStoreTest.cpp" +) +target_sources(syncEntryTest + PRIVATE + "syncEntryTest.cpp" +) target_include_directories(syncTest @@ -172,6 +182,16 @@ target_include_directories(syncIndexMgrTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncLogStoreTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncEntryTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -242,6 +262,14 @@ target_link_libraries(syncIndexMgrTest sync gtest_main ) +target_link_libraries(syncLogStoreTest + sync + gtest_main +) +target_link_libraries(syncEntryTest + sync + gtest_main +) enable_testing() @@ -249,3 +277,5 @@ add_test( NAME sync_test COMMAND syncTest ) + + diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index e2bc9a73ae..e1706bb40b 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -2,6 +2,7 @@ #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" +#include "syncMessage.h" #include "syncRaftStore.h" void logTest() { diff --git a/source/libs/sync/test/syncEntryTest.cpp b/source/libs/sync/test/syncEntryTest.cpp new file mode 100644 index 0000000000..e54daaa79a --- /dev/null +++ b/source/libs/sync/test/syncEntryTest.cpp @@ -0,0 +1,81 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void test1() { + SSyncRaftEntry* pEntry = syncEntryBuild(10); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100; + pEntry->index = 200; + strcpy(pEntry->data, "test1"); + + syncEntryPrint(pEntry); + syncEntryDestory(pEntry); +} + +void test2() { + SyncClientRequest* pSyncMsg = syncClientRequestBuild(10); + pSyncMsg->originalRpcType = 33; + pSyncMsg->seqNum = 11; + pSyncMsg->isWeak = 1; + strcpy(pSyncMsg->data, "test2"); + + SSyncRaftEntry* pEntry = syncEntryBuild2(pSyncMsg, 100, 200); + syncEntryPrint(pEntry); + + syncClientRequestDestroy(pSyncMsg); + syncEntryDestory(pEntry); +} + +void test3() { + SSyncRaftEntry* pEntry = syncEntryBuild(10); + assert(pEntry != NULL); + pEntry->msgType = 11; + pEntry->originalRpcType = 22; + pEntry->seqNum = 33; + pEntry->isWeak = true; + pEntry->term = 44; + pEntry->index = 55; + strcpy(pEntry->data, "test3"); + syncEntryPrint(pEntry); + + uint32_t len; + char* serialized = syncEntrySerialize(pEntry, &len); + assert(serialized != NULL); + SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len); + syncEntryPrint(pEntry2); + + free(serialized); + syncEntryDestory(pEntry2); + syncEntryDestory(pEntry); +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + test1(); + test2(); + test3(); + + return 0; +} diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index 1ac4357c5a..101c0efe9a 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -3,6 +3,7 @@ #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "ttime.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,24 +14,13 @@ void logTest() { sFatal("--- sync log test: fatal"); } -void doSync() { - SSyncInfo syncInfo; - syncInfo.vgId = 1; +void *pTimer = NULL; +void *pTimerMgr = NULL; +int g = 300; - SSyncCfg* pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = 3; - - pCfg->nodeInfo[0].nodePort = 7010; - taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = 7110; - taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = 7210; - taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); - - SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); - assert(pSyncNode != NULL); +static void timerFp(void *param, void *tmrId) { + printf("param:%p, tmrId:%p, pTimer:%p, pTimerMgr:%p \n", param, tmrId, pTimer, pTimerMgr); + taosTmrReset(timerFp, 1000, param, pTimerMgr, &pTimer); } int main() { @@ -41,13 +31,12 @@ int main() { logTest(); - // ret = syncIOStart(); - // assert(ret == 0); - ret = syncEnvStart(); assert(ret == 0); - // doSync(); + // timer + pTimerMgr = taosTmrInit(1000, 50, 10000, "SYNC-ENV-TEST"); + taosTmrStart(timerFp, 1000, &g, pTimerMgr); while (1) { taosMsleep(1000); diff --git a/source/libs/sync/test/syncIOSendMsgClientTest.cpp b/source/libs/sync/test/syncIOSendMsgClientTest.cpp index 83ac724789..250054fd5a 100644 --- a/source/libs/sync/test/syncIOSendMsgClientTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgClientTest.cpp @@ -39,11 +39,11 @@ int main() { rpcMsg.msgType = 77; syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); - sleep(1); + taosSsleep(1); } while (1) { - sleep(1); + taosSsleep(1); } return 0; diff --git a/source/libs/sync/test/syncIOSendMsgServerTest.cpp b/source/libs/sync/test/syncIOSendMsgServerTest.cpp index b0f177962f..1d7402e461 100644 --- a/source/libs/sync/test/syncIOSendMsgServerTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgServerTest.cpp @@ -26,7 +26,7 @@ int main() { assert(ret == 0); while (1) { - sleep(1); + taosSsleep(1); } return 0; diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index c25ad3b1dd..ed88fbb03e 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -39,11 +39,11 @@ int main() { rpcMsg.msgType = 77; syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg); - sleep(1); + taosSsleep(1); } while (1) { - sleep(1); + taosSsleep(1); } return 0; diff --git a/source/libs/sync/test/syncIOTickPingTest.cpp b/source/libs/sync/test/syncIOTickPingTest.cpp index 777fc035e2..8be93e6fc0 100644 --- a/source/libs/sync/test/syncIOTickPingTest.cpp +++ b/source/libs/sync/test/syncIOTickPingTest.cpp @@ -29,7 +29,7 @@ int main() { assert(ret == 0); while (1) { - sleep(1); + taosSsleep(1); } return 0; } diff --git a/source/libs/sync/test/syncIOTickQTest.cpp b/source/libs/sync/test/syncIOTickQTest.cpp index 5615058cc3..76f5e33e82 100644 --- a/source/libs/sync/test/syncIOTickQTest.cpp +++ b/source/libs/sync/test/syncIOTickQTest.cpp @@ -29,7 +29,7 @@ int main() { assert(ret == 0); while (1) { - sleep(1); + taosSsleep(1); } return 0; } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 4e4cd9222b..6bad8f09cf 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -33,8 +33,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); - snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index 669c4e68a5..b6794544eb 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -30,8 +30,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); - snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; @@ -54,6 +53,7 @@ SSyncNode* syncNodeInit() { gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; @@ -89,11 +89,11 @@ int main(int argc, char** argv) { SSyncNode* pSyncNode = syncInitTest(); assert(pSyncNode != NULL); - char* serialized = syncNode2Str(pSyncNode); - printf("%s\n", serialized); - free(serialized); + syncNodePrint((char*)"syncInitTest", pSyncNode); initRaftId(pSyncNode); + //-------------------------------------------------------------- + return 0; -} +} \ No newline at end of file diff --git a/source/libs/sync/test/syncLogStoreTest.cpp b/source/libs/sync/test/syncLogStoreTest.cpp new file mode 100644 index 0000000000..a5eb748de6 --- /dev/null +++ b/source/libs/sync/test/syncLogStoreTest.cpp @@ -0,0 +1,141 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* pSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen("./wal_test", &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void logStoreTest() { + logStorePrint(pSyncNode->pLogStore); + for (int i = 0; i < 5; ++i) { + int32_t dataLen = 10; + SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); + assert(pEntry != NULL); + pEntry->msgType = 1; + pEntry->originalRpcType = 2; + pEntry->seqNum = 3; + pEntry->isWeak = true; + pEntry->term = 100; + pEntry->index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1; + snprintf(pEntry->data, dataLen, "value%d", i); + + //syncEntryPrint2((char*)"write entry:", pEntry); + pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry); + syncEntryDestory(pEntry); + } + logStorePrint(pSyncNode->pLogStore); + + pSyncNode->pLogStore->truncate(pSyncNode->pLogStore, 3); + logStorePrint(pSyncNode->pLogStore); +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + + //syncNodePrint((char*)"syncLogStoreTest", pSyncNode); + //initRaftId(pSyncNode); + + logStoreTest(); + + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp index 450e097cc8..83f1f67eb1 100644 --- a/source/libs/sync/test/syncPingTest.cpp +++ b/source/libs/sync/test/syncPingTest.cpp @@ -1,8 +1,10 @@ +#include #include #include "syncEnv.h" #include "syncIO.h" #include "syncInt.h" #include "syncRaftStore.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,59 +15,65 @@ void logTest() { sFatal("--- sync log test: fatal"); } -uint16_t ports[3] = {7010, 7110, 7210}; +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; -SSyncNode* doSync(int myIndex) { - SSyncFSM* pFsm; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; - SSyncInfo syncInfo; - syncInfo.vgId = 1; +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; syncInfo.rpcClient = gSyncIO->clientRpc; syncInfo.FpSendMsg = syncIOSendMsg; syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path"); - snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; - pCfg->replicaNum = 3; + pCfg->replicaNum = replicaNum; - pCfg->nodeInfo[0].nodePort = ports[0]; - snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); - - pCfg->nodeInfo[1].nodePort = ports[1]; - snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); - - pCfg->nodeInfo[2].nodePort = ports[2]; - snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1"); - // taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); assert(pSyncNode != NULL); gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; gSyncIO->pSyncNode = pSyncNode; return pSyncNode; } -void timerPingAll(void* param, void* tmrId) { - SSyncNode* pSyncNode = (SSyncNode*)param; - syncNodePingAll(pSyncNode); +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } } int main(int argc, char** argv) { - // taosInitLog((char*)"syncPingTest.log", 100000, 10); + // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; sDebugFlag = 143 + 64; - logTest(); - - int myIndex = 0; + myIndex = 0; if (argc >= 2) { myIndex = atoi(argv[1]); } @@ -76,30 +84,45 @@ int main(int argc, char** argv) { ret = syncEnvStart(); assert(ret == 0); - SSyncNode* pSyncNode = doSync(myIndex); - gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; - gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; - gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + SSyncNode* pSyncNode = syncInitTest(); + assert(pSyncNode != NULL); + syncNodePrint((char*)"----1", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + + sTrace("syncNodeStartPingTimer ..."); ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); + syncNodePrint((char*)"----2", pSyncNode); + sTrace("sleep ..."); taosMsleep(10000); + sTrace("syncNodeStopPingTimer ..."); ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); + syncNodePrint((char*)"----3", pSyncNode); - taosMsleep(10000); + sTrace("sleep ..."); + taosMsleep(5000); + sTrace("syncNodeStartPingTimer ..."); ret = syncNodeStartPingTimer(pSyncNode); assert(ret == 0); + syncNodePrint((char*)"----4", pSyncNode); + sTrace("sleep ..."); taosMsleep(10000); + sTrace("syncNodeStopPingTimer ..."); ret = syncNodeStopPingTimer(pSyncNode); assert(ret == 0); + syncNodePrint((char*)"----5", pSyncNode); while (1) { + sTrace("while 1 sleep ..."); taosMsleep(1000); } diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index 3edde509f8..504fa3034a 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -32,8 +32,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); - snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 74d42cd531..0b6abef212 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -32,8 +32,7 @@ SSyncNode* syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path"); - snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path"); + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./"); SSyncCfg* pCfg = &syncInfo.syncCfg; pCfg->myIndex = myIndex; diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f4babc9980..bdb754ea14 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -107,7 +107,7 @@ static void *sendRequest(void *param) { tDebug("recv response succefully"); - // usleep(100000000); + // taosSsleep(100); } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); @@ -223,7 +223,7 @@ int main(int argc, char *argv[]) { } do { - usleep(1); + taosUsleep(1); } while (tcount < appThreads); gettimeofday(&systemTime, NULL); diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index c763b7bd8a..f4ad73f743 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -77,7 +77,7 @@ void processShellMsg() { taosFreeQitem(pRpcMsg); { - // sleep(1); + // taosSsleep(1); SRpcMsg nRpcMsg = {0}; nRpcMsg.pCont = rpcMallocCont(msgSize); nRpcMsg.contLen = msgSize; @@ -176,7 +176,7 @@ int main(int argc, char *argv[]) { tError("failed to start RPC server"); return -1; } - // sleep(5); + // taosSsleep(5); tInfo("RPC server is running, ctrl-c to exit"); diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 6fc935cb61..f3940e34f9 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -84,7 +84,7 @@ static void *sendRequest(void *param) { tDebug("recv response succefully"); - // usleep(100000000); + // taosSsleep(100); } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); @@ -200,7 +200,7 @@ int main(int argc, char *argv[]) { } do { - usleep(1); + taosUsleep(1); } while (tcount < appThreads); gettimeofday(&systemTime, NULL); diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 26a02eb05b..f2a963b83f 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -178,7 +178,7 @@ int main(int argc, char *argv[]) { } do { - usleep(1); + taosUsleep(1); } while ( tcount < appThreads); gettimeofday(&systemTime, NULL); diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index f43fa7aae6..f84ced5374 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -85,7 +85,7 @@ static void *sendRequest(void *param) { tDebug("recv response succefully"); - // usleep(100000000); + // taosSsleep(100); } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); @@ -201,7 +201,7 @@ int main(int argc, char *argv[]) { } do { - usleep(1); + taosUsleep(1); } while (tcount < appThreads); gettimeofday(&systemTime, NULL); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index f44fc71964..69ae83154a 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -124,8 +124,13 @@ class WalRetentionEnv : public ::testing::Test { void SetUp() override { SWalCfg cfg; - cfg.rollPeriod = -1, cfg.segSize = -1, cfg.retentionPeriod = -1, cfg.retentionSize = 0, cfg.rollPeriod = 0, - cfg.vgId = 0, cfg.level = TAOS_WAL_FSYNC; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 0; + cfg.level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, &cfg); ASSERT(pWal != NULL); } diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index 3b90fbdad8..724347b0bc 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -13,47 +13,35 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -void taosMsleep(int32_t ms) { Sleep(ms); } - -#else +#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #include +#endif -/* - to make taosMsleep work, - signal SIGALRM shall be blocked in the calling thread, - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGALRM); - pthread_sigmask(SIG_BLOCK, &set, NULL); -*/ -void taosMsleep(int32_t mseconds) { -#if 1 - usleep(mseconds * 1000); +void taosSsleep(int32_t s) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + Sleep(1000 * s); #else - struct timeval timeout; - int32_t seconds, useconds; - - seconds = mseconds / 1000; - useconds = (mseconds % 1000) * 1000; - timeout.tv_sec = seconds; - timeout.tv_usec = useconds; - - /* sigset_t set; */ - /* sigemptyset(&set); */ - /* sigaddset(&set, SIGALRM); */ - /* pthread_sigmask(SIG_BLOCK, &set, NULL); */ - - select(0, NULL, NULL, NULL, &timeout); - -/* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ + sleep(s); #endif } +void taosMsleep(int32_t ms) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + Sleep(ms); +#else + usleep(ms * 1000); #endif +} + +void taosUsleep(int32_t us) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + nanosleep(1000 * us); +#else + usleep(us); +#endif +} diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 88ea4b3e15..0fab7a941e 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -291,11 +291,10 @@ int32_t twcslen(const wchar_t *wcs) { return n; } - int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) { - for (int32_t i = 0; i < bytes; ++i) { - int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4); - int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4); + for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) { + int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i); + int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i); if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { return f1 - f2; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 02d7e6c0e9..25c748a8d8 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,15 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" -bool taosCheckSystemIsSmallEnd() { - union check{ - int16_t i; - char ch[2]; - }c; - c.i=1; - return c.ch[0]==1; -} - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -48,115 +39,6 @@ bool taosCheckSystemIsSmallEnd() { #include #pragma warning(pop) -int32_t taosGetTotalMemory(int64_t *totalKB) { - MEMORYSTATUSEX memsStat; - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; - } - - *totalKB = memsStat.ullTotalPhys / 1024; - return 0; -} - -int32_t taosGetSysMemory(int64_t *usedKB) { - MEMORYSTATUSEX memsStat; - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return -1; - } - - int64_t nMemFree = memsStat.ullAvailPhys / 1024; - int64_t nMemTotal = memsStat.ullTotalPhys / 1024.0; - - *usedKB = nMemTotal - nMemFree; - return 0; -} - -int32_t taosGetProcMemory(int64_t *usedKB) { - unsigned bytes_used = 0; - -#if defined(_WIN64) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif - - *usedKB = bytes_used / 1024; - return 0; -} - -int32_t taosGetCpuCores(float *numOfCores) { - SYSTEM_INFO info; - GetSystemInfo(&info); - *numOfCores = info.dwNumberOfProcessors; - return 0; -} - -int32_t taosGetCpuUsage(double *sysCpuUsage, double *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return 0; -} - -int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { - unsigned _int64 i64FreeBytesToCaller; - unsigned _int64 i64TotalBytes; - unsigned _int64 i64FreeBytes; - - BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - diskSize->tsize = (int64_t)(i64TotalBytes); - diskSize->avail = (int64_t)(i64FreeBytesToCaller); - diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); - return 0; - } else { - // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } -} - -int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { - *receive_bytes = 0; - *transmit_bytes = 0; - return 0; -} - -int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - IO_COUNTERS io_counter; - if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { - if (rchars) *rchars = io_counter.ReadTransferCount; - if (wchars) *wchars = io_counter.WriteTransferCount; - if (read_bytes) *read_bytes = 0; - if (write_bytes) *write_bytes = 0; - return 0; - } - return -1; -} - -void taosGetSystemInfo() { - taosGetCpuCores(&tsNumOfCores); - taosGetTotalMemory(&tsTotalMemoryKB); - - double tmp1, tmp2, tmp3, tmp4; - taosGetCpuUsage(&tmp1, &tmp2); -} - -void taosKillSystem() { - // printf("function taosKillSystem, exit!"); - exit(0); -} - -int taosSystem(const char *cmd) { - // printf("taosSystem not support"); - return -1; -} - LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, IN MINIDUMP_TYPE DumpType, @@ -193,124 +75,13 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { return EXCEPTION_CONTINUE_SEARCH; } -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } - -int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - GUID guid; - CoCreateGuid(&guid); - - sprintf(uid, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], - guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); - - return 0; -} - -char *taosGetCmdlineByPID(int pid) { return ""; } - #elif defined(_TD_DARWIN_64) -/* - * darwin implementation - */ - #include #include -void taosKillSystem() { - // printf("function taosKillSystem, exit!"); - exit(0); -} - -int32_t taosGetCpuCores(float *numOfCores) { - *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); - return 0; -} - -void taosGetSystemInfo() { - long physical_pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGESIZE); - tsTotalMemoryKB = physical_pages * page_size / 1024; - tsPageSizeKB = page_size / 1024; - tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); -} - -int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - if (rchars) *rchars = 0; - if (wchars) *wchars = 0; - if (read_bytes) *read_bytes = 0; - if (write_bytes) *write_bytes = 0; - return 0; -} - -int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { - *receive_bytes = 0; - *transmit_bytes = 0; - return 0; -} - -int32_t taosGetCpuUsage(double *sysCpuUsage, double *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return 0; -} - -int32_t taosGetProcMemory(int64_t *usedKB) { - *usedKB = 0; - return 0; -} - -int32_t taosGetSysMemory(int64_t *usedKB) { - *usedKB = 0; - return 0; -} - -int taosSystem(const char *cmd) { - // printf("un support funtion"); - return -1; -} - -void taosSetCoreDump() {} - -int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { - struct statvfs info; - if (statvfs(dataDir, &info)) { - // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } else { - diskSize->tsize = info.f_blocks * info.f_frsize; - diskSize->avail = info.f_bavail * info.f_frsize; - diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; - return 0; - } -} - -int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - uuid_t uuid = {0}; - uuid_generate(uuid); - // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null - uuid_unparse_lower(uuid, uid); - return 0; -} - -char *taosGetCmdlineByPID(int pid) { - static char cmdline[1024]; - errno = 0; - - if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { - fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); - return strerror(errno); - } - - return cmdline; -} - #else -/* - * linux implementation - */ - #include #include #include @@ -354,49 +125,6 @@ static void taosGetProcIOnfos() { snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } -int32_t taosGetTotalMemory(int64_t *totalKB) { - *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); - return 0; -} - -int32_t taosGetSysMemory(int64_t *usedKB) { - *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; - return 0; -} - -int32_t taosGetProcMemory(int64_t *usedKB) { - TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - // printf("open file:%s failed", tsProcMemFile); - return -1; - } - - ssize_t _bytes = 0; - char *line = NULL; - while (!taosEOFFile(pFile)) { - _bytes = taosGetLineFile(pFile, &line); - if ((_bytes < 0) || (line == NULL)) { - break; - } - if (strstr(line, "VmRSS:") != NULL) { - break; - } - } - - if (line == NULL) { - // printf("read file:%s failed", tsProcMemFile); - taosCloseFile(&pFile); - return -1; - } - - char tmp[10]; - sscanf(line, "%s %" PRId64, tmp, usedKB); - - if (line != NULL) tfree(line); - taosCloseFile(&pFile); - return 0; -} - static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { @@ -450,12 +178,210 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { return 0; } +#endif + +bool taosCheckSystemIsSmallEnd() { + union check{ + int16_t i; + char ch[2]; + }c; + c.i=1; + return c.ch[0]==1; +} + +void taosGetSystemInfo() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); + + double tmp1, tmp2, tmp3, tmp4; + taosGetCpuUsage(&tmp1, &tmp2); +#elif defined(_TD_DARWIN_64) + long physical_pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + tsTotalMemoryKB = physical_pages * page_size / 1024; + tsPageSizeKB = page_size / 1024; + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); +#else + taosGetProcIOnfos(); + taosGetCpuCores(&tsNumOfCores); + taosGetTotalMemory(&tsTotalMemoryKB); + + double tmp1, tmp2, tmp3, tmp4; + taosGetCpuUsage(&tmp1, &tmp2); +#endif +} + +int32_t taosGetEmail(char *email, int32_t maxLen) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#elif defined(_TD_DARWIN_64) + const char *filepath = "/usr/local/taos/email"; + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) return false; + + if (taosReadFile(pFile, (void *)email, maxLen) < 0) { + taosCloseFile(&pFile); + return -1; + } + + taosCloseFile(&pFile); + return 0; +#else + const char *filepath = "/usr/local/taos/email"; + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) return false; + + if (taosReadFile(pFile, (void *)email, maxLen) < 0) { + taosCloseFile(&pFile); + return -1; + } + + taosCloseFile(&pFile); + return 0; +#endif +} + +int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#elif defined(_TD_DARWIN_64) + char *line = NULL; + size_t size = 0; + int32_t code = -1; + + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "PRETTY_NAME", 11) == 0) { + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(releaseName, p, maxLen); + code = 0; + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return code; +#else + char *line = NULL; + size_t size = 0; + int32_t code = -1; + + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while ((size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (strncmp(line, "PRETTY_NAME", 11) == 0) { + const char *p = strchr(line, '=') + 1; + if (*p == '"') { + p++; + line[size - 2] = 0; + } + tstrncpy(releaseName, p, maxLen); + code = 0; + break; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + return code; +#endif +} + +int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#elif defined(_TD_DARWIN_64) + char *line = NULL; + size_t size = 0; + int32_t done = 0; + int32_t code = -1; + + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { + const char *v = strchr(line, ':') + 2; + tstrncpy(cpuModel, v, maxLen); + code = 0; + done |= 1; + } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { + const char *v = strchr(line, ':') + 2; + *numOfCores = atof(v); + done |= 2; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + + return code; +#else + char *line = NULL; + size_t size = 0; + int32_t done = 0; + int32_t code = -1; + + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return false; + + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { + line[size - 1] = '\0'; + if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { + const char *v = strchr(line, ':') + 2; + tstrncpy(cpuModel, v, maxLen); + code = 0; + done |= 1; + } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { + const char *v = strchr(line, ':') + 2; + *numOfCores = atof(v); + done |= 2; + } + } + + if (line != NULL) free(line); + taosCloseFile(&pFile); + + return code; +#endif +} + int32_t taosGetCpuCores(float *numOfCores) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + SYSTEM_INFO info; + GetSystemInfo(&info); + *numOfCores = info.dwNumberOfProcessors; + return 0; +#elif defined(_TD_DARWIN_64) *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); return 0; +#else + *numOfCores = sysconf(_SC_NPROCESSORS_ONLN); + return 0; +#endif } int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + *sysCpuUsage = 0; + *procCpuUsage = 0; + return 0; +#elif defined(_TD_DARWIN_64) + *sysCpuUsage = 0; + *procCpuUsage = 0; + return 0; +#else static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; static uint64_t lastProcTotal = 0; @@ -492,9 +418,132 @@ int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine) { lastProcTotal = curProcTotal; return 0; +#endif +} + +int32_t taosGetTotalMemory(int64_t *totalKB) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return -1; + } + + *totalKB = memsStat.ullTotalPhys / 1024; + return 0; +#elif defined(_TD_DARWIN_64) + return 0; +#else + *totalKB = (int64_t)(sysconf(_SC_PHYS_PAGES) * tsPageSizeKB); + return 0; +#endif +} + +int32_t taosGetProcMemory(int64_t *usedKB) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + unsigned bytes_used = 0; + +#if defined(_WIN64) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *usedKB = bytes_used / 1024; + return 0; +#elif defined(_TD_DARWIN_64) + *usedKB = 0; + return 0; +#else + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { + // printf("open file:%s failed", tsProcMemFile); + return -1; + } + + ssize_t _bytes = 0; + char *line = NULL; + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); + if ((_bytes < 0) || (line == NULL)) { + break; + } + if (strstr(line, "VmRSS:") != NULL) { + break; + } + } + + if (line == NULL) { + // printf("read file:%s failed", tsProcMemFile); + taosCloseFile(&pFile); + return -1; + } + + char tmp[10]; + sscanf(line, "%s %" PRId64, tmp, usedKB); + + if (line != NULL) tfree(line); + taosCloseFile(&pFile); + return 0; +#endif +} + +int32_t taosGetSysMemory(int64_t *usedKB) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return -1; + } + + int64_t nMemFree = memsStat.ullAvailPhys / 1024; + int64_t nMemTotal = memsStat.ullTotalPhys / 1024.0; + + *usedKB = nMemTotal - nMemFree; + return 0; +#elif defined(_TD_DARWIN_64) + *usedKB = 0; + return 0; +#else + *usedKB = sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB; + return 0; +#endif } int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + unsigned _int64 i64FreeBytesToCaller; + unsigned _int64 i64TotalBytes; + unsigned _int64 i64FreeBytes; + + BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + diskSize->tsize = (int64_t)(i64TotalBytes); + diskSize->avail = (int64_t)(i64FreeBytesToCaller); + diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); + return 0; + } else { + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } +#elif defined(_TD_DARWIN_64) + struct statvfs info; + if (statvfs(dataDir, &info)) { + // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + return 0; + } +#else struct statvfs info; if (statvfs(dataDir, &info)) { return -1; @@ -504,9 +553,79 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { diskSize->used = diskSize->total - diskSize->avail; return 0; } +#endif +} + +int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + IO_COUNTERS io_counter; + if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { + if (rchars) *rchars = io_counter.ReadTransferCount; + if (wchars) *wchars = io_counter.WriteTransferCount; + if (read_bytes) *read_bytes = 0; + if (write_bytes) *write_bytes = 0; + return 0; + } + return -1; +#elif defined(_TD_DARWIN_64) + if (rchars) *rchars = 0; + if (wchars) *wchars = 0; + if (read_bytes) *read_bytes = 0; + if (write_bytes) *write_bytes = 0; + return 0; +#else + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return -1; + + ssize_t _bytes = 0; + char *line = NULL; + char tmp[24]; + int readIndex = 0; + + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); + if (_bytes < 10 || line == NULL) { + break; + } + if (strstr(line, "rchar:") != NULL) { + sscanf(line, "%s %" PRId64, tmp, rchars); + readIndex++; + } else if (strstr(line, "wchar:") != NULL) { + sscanf(line, "%s %" PRId64, tmp, wchars); + readIndex++; + } else if (strstr(line, "read_bytes:") != NULL) { // read_bytes + sscanf(line, "%s %" PRId64, tmp, read_bytes); + readIndex++; + } else if (strstr(line, "write_bytes:") != NULL) { // write_bytes + sscanf(line, "%s %" PRId64, tmp, write_bytes); + readIndex++; + } else { + } + + if (readIndex >= 4) break; + } + + if (line != NULL) tfree(line); + taosCloseFile(&pFile); + + if (readIndex < 4) { + return -1; + } + + return 0; +#endif } int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + *receive_bytes = 0; + *transmit_bytes = 0; + return 0; +#elif defined(_TD_DARWIN_64) + *receive_bytes = 0; + *transmit_bytes = 0; + return 0; +#else TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return -1; @@ -549,93 +668,100 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { taosCloseFile(&pFile); return 0; -} - -int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { - TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return -1; - - ssize_t _bytes = 0; - char *line = NULL; - char tmp[24]; - int readIndex = 0; - - while (!taosEOFFile(pFile)) { - _bytes = taosGetLineFile(pFile, &line); - if (_bytes < 10 || line == NULL) { - break; - } - if (strstr(line, "rchar:") != NULL) { - sscanf(line, "%s %" PRId64, tmp, rchars); - readIndex++; - } else if (strstr(line, "wchar:") != NULL) { - sscanf(line, "%s %" PRId64, tmp, wchars); - readIndex++; - } else if (strstr(line, "read_bytes:") != NULL) { // read_bytes - sscanf(line, "%s %" PRId64, tmp, read_bytes); - readIndex++; - } else if (strstr(line, "write_bytes:") != NULL) { // write_bytes - sscanf(line, "%s %" PRId64, tmp, write_bytes); - readIndex++; - } else { - } - - if (readIndex >= 4) break; - } - - if (line != NULL) tfree(line); - taosCloseFile(&pFile); - - if (readIndex < 4) { - return -1; - } - - return 0; -} - -void taosGetSystemInfo() { - taosGetProcIOnfos(); - taosGetCpuCores(&tsNumOfCores); - taosGetTotalMemory(&tsTotalMemoryKB); - - double tmp1, tmp2, tmp3, tmp4; - taosGetCpuUsage(&tmp1, &tmp2); +#endif } void taosKillSystem() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + printf("function taosKillSystem, exit!"); + exit(0); +#elif defined(_TD_DARWIN_64) + printf("function taosKillSystem, exit!"); + exit(0); +#else // SIGINT - // printf("taosd will shut down soon"); + printf("taosd will shut down soon"); kill(tsProcId, 2); +#endif } -int taosSystem(const char *cmd) { - FILE *fp; - int res; - char buf[1024]; - if (cmd == NULL) { - // printf("taosSystem cmd is NULL!"); - return -1; - } +int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + GUID guid; + CoCreateGuid(&guid); - if ((fp = popen(cmd, "r")) == NULL) { - // printf("popen cmd:%s error: %s", cmd, strerror(errno)); + sprintf(uid, "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", guid.Data1, guid.Data2, guid.Data3, guid.Data4[0], + guid.Data4[1], guid.Data4[2], guid.Data4[3], guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); + + return 0; +#elif defined(_TD_DARWIN_64) + uuid_t uuid = {0}; + uuid_generate(uuid); + // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null + uuid_unparse_lower(uuid, uid); + return 0; +#else + int len = 0; + + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - while (fgets(buf, sizeof(buf), fp)) { - // printf("popen result:%s", buf); - } - - if ((res = pclose(fp)) == -1) { - // printf("close popen file pointer fp error!"); - } else { - // printf("popen res is :%d", res); - } - - return res; + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } + + if (len >= 36) { + uid[36] = 0; + return 0; + } + + return 0; +#endif +} + +char *taosGetCmdlineByPID(int pid) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return ""; +#elif defined(_TD_DARWIN_64) + static char cmdline[1024]; + errno = 0; + + if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { + fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); + return strerror(errno); + } + + return cmdline; +#else + static char cmdline[1024]; + sprintf(cmdline, "/proc/%d/cmdline", pid); + + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); + if (n < 0) n = 0; + + if (n > 0 && cmdline[n - 1] == '\n') --n; + + cmdline[n] = 0; + + taosCloseFile(&pFile); + } else { + cmdline[0] = 0; + } + + return cmdline; +#endif } void taosSetCoreDump(bool enable) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + SetUnhandledExceptionFilter(&FlCrashDump); +#elif defined(_TD_DARWIN_64) +#else if (!enable) return; // 1. set ulimit -c unlimited @@ -708,55 +834,12 @@ void taosSetCoreDump(bool enable) { // printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif -} - -int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int len = 0; - - // fd = open("/proc/sys/kernel/random/uuid", 0); - TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); - if (pFile == NULL) { - return -1; - } else { - len = taosReadFile(pFile, uid, uidlen); - taosCloseFile(&pFile); - } - - if (len >= 36) { - uid[36] = 0; - return 0; - } - - return 0; -} - -char *taosGetCmdlineByPID(int pid) { - static char cmdline[1024]; - sprintf(cmdline, "/proc/%d/cmdline", pid); - - // int fd = open(cmdline, O_RDONLY); - TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); - if (pFile != NULL) { - int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); - if (n < 0) n = 0; - - if (n > 0 && cmdline[n - 1] == '\n') --n; - - cmdline[n] = 0; - - taosCloseFile(&pFile); - } else { - cmdline[0] = 0; - } - - return cmdline; -} - #endif - -#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) +} SysNameInfo taosGetSysNameInfo() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#elif defined(_TD_DARWIN_64) SysNameInfo info = {0}; struct utsname uts; @@ -769,77 +852,18 @@ SysNameInfo taosGetSysNameInfo() { } return info; -} +#else + SysNameInfo info = {0}; -int32_t taosGetEmail(char *email, int32_t maxLen) { - const char *filepath = "/usr/local/taos/email"; - - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); - if (pFile == NULL) return false; - - if (taosReadFile(pFile, (void *)email, maxLen) < 0) { - taosCloseFile(&pFile); - return -1; + struct utsname uts; + if (!uname(&uts)) { + tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); + tstrncpy(info.nodename, uts.nodename, sizeof(info.nodename)); + tstrncpy(info.release, uts.release, sizeof(info.release)); + tstrncpy(info.version, uts.version, sizeof(info.version)); + tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } - taosCloseFile(&pFile); - return 0; -} - -int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { - char *line = NULL; - size_t size = 0; - int32_t code = -1; - - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while ((size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char *p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - tstrncpy(releaseName, p, maxLen); - code = 0; - break; - } - } - - if (line != NULL) free(line); - taosCloseFile(&pFile); - return code; -} - -int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { - char *line = NULL; - size_t size = 0; - int32_t done = 0; - int32_t code = -1; - - TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { - const char *v = strchr(line, ':') + 2; - tstrncpy(cpuModel, v, maxLen); - code = 0; - done |= 1; - } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { - const char *v = strchr(line, ':') + 2; - *numOfCores = atof(v); - done |= 2; - } - } - - if (line != NULL) free(line); - taosCloseFile(&pFile); - - return code; -} - + return info; #endif +} diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 717cae0fbd..5c98db33d1 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -13,20 +13,126 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#elif defined(_TD_DARWIN_64) +#else +#include +#include +#include +#endif -/* - * windows implementation - */ +struct termios oldtio; -void* taosLoadDll(const char* filename) { return NULL; } -void* taosLoadSym(void* handle, char* name) { return NULL; } -void taosCloseDll(void* handle) {} +int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + FILE *fp; + if (cmd == NULL) { + // printf("taosSystem cmd is NULL!"); + return -1; + } + + if ((fp = _popen(cmd, "r")) == NULL) { + // printf("popen cmd:%s error: %s", cmd, strerror(errno)); + return -1; + } else { + while (fgets(buf, bufSize, fp)) { + // printf("popen result:%s", buf); + } + + if (!_pclose(fp)) { + // printf("close popen file pointer fp error!"); + return -1; + } else { + // printf("popen res is :%d", res); + } + + return 0; +#elif defined(_TD_DARWIN_64) + printf("no support funtion"); + return -1; +#else + FILE *fp; + int32_t res; + if (cmd == NULL) { + // printf("taosSystem cmd is NULL!"); + return -1; + } + + if ((fp = popen(cmd, "r")) == NULL) { + // printf("popen cmd:%s error: %s", cmd, strerror(errno)); + return -1; + } else { + while (fgets(buf, bufSize, fp)) { + // printf("popen result:%s", buf); + } + + if ((res = pclose(fp)) == -1) { + // printf("close popen file pointer fp error!"); + } else { + // printf("popen res is :%d", res); + } + + return res; + } +#endif +} + +void* taosLoadDll(const char* filename) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#elif defined(_TD_DARWIN_64) + return NULL; +#else + void* handle = dlopen(filename, RTLD_LAZY); + if (!handle) { + //printf("load dll:%s failed, error:%s", filename, dlerror()); + return NULL; + } + + //printf("dll %s loaded", filename); + + return handle; +#endif +} + +void* taosLoadSym(void* handle, char* name) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#elif defined(_TD_DARWIN_64) + return NULL; +#else + void* sym = dlsym(handle, name); + char* error = NULL; + + if ((error = dlerror()) != NULL) { + //printf("load sym:%s failed, error:%s", name, dlerror()); + return NULL; + } + + //printf("sym %s loaded", name); + + return sym; +#endif +} + +void taosCloseDll(void* handle) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return; +#elif defined(_TD_DARWIN_64) + return; +#else + if (handle) { + dlclose(handle); + } +#endif +} int taosSetConsoleEcho(bool on) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); DWORD mode = 0; GetConsoleMode(hStdin, &mode); @@ -38,19 +144,7 @@ int taosSetConsoleEcho(bool on) { SetConsoleMode(hStdin, mode); return 0; -} - #elif defined(_TD_DARWIN_64) - -/* - * darwin implementation - */ - -void* taosLoadDll(const char* filename) { return NULL; } -void* taosLoadSym(void* handle, char* name) { return NULL; } -void taosCloseDll(void* handle) {} - -int taosSetConsoleEcho(bool on) { #define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) int err; struct termios term; @@ -72,51 +166,7 @@ int taosSetConsoleEcho(bool on) { } return 0; -} - #else - -/* - * linux implementation - */ - -#include -#include -#include - -void* taosLoadDll(const char* filename) { - void* handle = dlopen(filename, RTLD_LAZY); - if (!handle) { - //printf("load dll:%s failed, error:%s", filename, dlerror()); - return NULL; - } - - //printf("dll %s loaded", filename); - - return handle; -} - -void* taosLoadSym(void* handle, char* name) { - void* sym = dlsym(handle, name); - char* error = NULL; - - if ((error = dlerror()) != NULL) { - //printf("load sym:%s failed, error:%s", name, dlerror()); - return NULL; - } - - //printf("sym %s loaded", name); - - return sym; -} - -void taosCloseDll(void* handle) { - if (handle) { - dlclose(handle); - } -} - -int taosSetConsoleEcho(bool on) { #define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL) int err; struct termios term; @@ -138,6 +188,111 @@ int taosSetConsoleEcho(bool on) { } return 0; +#endif } +void setTerminalMode() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#elif defined(_TD_DARWIN_64) + struct termios newtio; + + /* if (atexit() != 0) { */ + /* fprintf(stderr, "Error register exit function!\n"); */ + /* exit(EXIT_FAILURE); */ + /* } */ + + memcpy(&newtio, &oldtio, sizeof(oldtio)); + + // Set new terminal attributes. + newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP); + newtio.c_iflag |= IGNBRK; + + // newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET); + newtio.c_oflag |= OPOST; + newtio.c_oflag |= ONLCR; + newtio.c_oflag &= ~(OCRNL | ONLRET); + + newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG); + newtio.c_cc[VMIN] = 1; + newtio.c_cc[VTIME] = 0; + + if (tcsetattr(0, TCSANOW, &newtio) != 0) { + fprintf(stderr, "Fail to set terminal properties!\n"); + exit(EXIT_FAILURE); + } +#else + struct termios newtio; + + /* if (atexit() != 0) { */ + /* fprintf(stderr, "Error register exit function!\n"); */ + /* exit(EXIT_FAILURE); */ + /* } */ + + memcpy(&newtio, &oldtio, sizeof(oldtio)); + + // Set new terminal attributes. + newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP); + newtio.c_iflag |= IGNBRK; + + // newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET); + newtio.c_oflag |= OPOST; + newtio.c_oflag |= ONLCR; + newtio.c_oflag &= ~(OCRNL | ONLRET); + + newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG); + newtio.c_cc[VMIN] = 1; + newtio.c_cc[VTIME] = 0; + + if (tcsetattr(0, TCSANOW, &newtio) != 0) { + fprintf(stderr, "Fail to set terminal properties!\n"); + exit(EXIT_FAILURE); + } #endif +} + +int32_t getOldTerminalMode() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#elif defined(_TD_DARWIN_64) + /* Make sure stdin is a terminal. */ + if (!isatty(STDIN_FILENO)) { + return -1; + } + + // Get the parameter of current terminal + if (tcgetattr(0, &oldtio) != 0) { + return -1; + } + + return 1; +#else + /* Make sure stdin is a terminal. */ + if (!isatty(STDIN_FILENO)) { + return -1; + } + + // Get the parameter of current terminal + if (tcgetattr(0, &oldtio) != 0) { + return -1; + } + + return 1; +#endif +} + +void resetTerminalMode() { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + +#elif defined(_TD_DARWIN_64) + if (tcsetattr(0, TCSANOW, &oldtio) != 0) { + fprintf(stderr, "Fail to reset the terminal properties!\n"); + exit(EXIT_FAILURE); + } +#else + if (tcsetattr(0, TCSANOW, &oldtio) != 0) { + fprintf(stderr, "Fail to reset the terminal properties!\n"); + exit(EXIT_FAILURE); + } +#endif +} \ No newline at end of file diff --git a/source/util/test/cacheTest.cpp b/source/util/test/cacheTest.cpp index e1c8c8c14e..748cf31b67 100644 --- a/source/util/test/cacheTest.cpp +++ b/source/util/test/cacheTest.cpp @@ -14,7 +14,7 @@ TEST(cacheTest, client_cache_test) { char data1[] = "test11"; char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1); - sleep(REFRESH_TIME_IN_SEC+1); + taosSsleep(REFRESH_TIME_IN_SEC+1); printf("obj is still valid: %s\n", cachedObj); @@ -37,7 +37,7 @@ TEST(cacheTest, client_cache_test) { taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false); - sleep(3); + taosSsleep(3); char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); assert(d == NULL); diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 46c95556d4..25314842fb 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -174,8 +174,8 @@ TEST(td_encode_test, encode_decode_variant_len_integer) { } TEST(td_encode_test, encode_decode_cstr) { - uint8_t * buf = new uint8_t[1024 * 1024]; - char * cstr = new char[1024 * 1024]; + uint8_t *buf = new uint8_t[1024 * 1024]; + char *cstr = new char[1024 * 1024]; const char *dcstr; SCoder encoder; SCoder decoder; @@ -208,7 +208,7 @@ TEST(td_encode_test, encode_decode_cstr) { typedef struct { int32_t A_a; int64_t A_b; - char * A_c; + char *A_c; } SStructA_v1; static int32_t tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { @@ -240,7 +240,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { typedef struct { int32_t A_a; int64_t A_b; - char * A_c; + char *A_c; // -------------------BELOW FEILDS ARE ADDED IN A NEW VERSION-------------- int16_t A_d; int16_t A_e; @@ -437,4 +437,4 @@ TEST(td_encode_test, compound_struct_encode_test) { tCoderClear(&decoder); } #endif -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 586151d782..58d9d2202e 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -42,7 +42,7 @@ void *addRef(void *param) { pSpace->p[id] = malloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); } - usleep(100); + taosUsleep(100); } return NULL; @@ -60,7 +60,7 @@ void *removeRef(void *param) { if (code == 0) pSpace->rid[id] = 0; } - usleep(100); + taosUsleep(100); } return NULL; @@ -76,7 +76,7 @@ void *acquireRelease(void *param) { id = random() % pSpace->refNum; void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { - usleep(id % 5 + 1); + taosUsleep(id % 5 + 1); taosReleaseRef(pSpace->rsetId, (int64_t) pSpace->p[id]); } } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 386df15d86..5336a557e3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -5089,7 +5089,7 @@ int main(int argc, char *argv[]) //pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]); while(1) { - sleep(1); + taosSsleep(1); } return 0; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 629677c9a5..dae0a1c840 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1 +1,10 @@ -add_subdirectory(shell) \ No newline at end of file +IF (TD_TAOS_TOOLS) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/tools/taos_tools/deps/avro/lang/c/src) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/client) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/common) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/util) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/os) + ADD_SUBDIRECTORY(taos-tools) +ENDIF () + +add_subdirectory(shell) diff --git a/tools/shell/inc/shell.h b/tools/shell/inc/shell.h index cf31e9d9d9..7a16ee9d2c 100644 --- a/tools/shell/inc/shell.h +++ b/tools/shell/inc/shell.h @@ -86,10 +86,6 @@ extern char PROMPT_HEADER[]; extern char CONTINUE_PROMPT[]; extern int prompt_size; extern SShellHistory history; -extern struct termios oldtio; -extern void set_terminal_mode(); -extern int get_old_terminal_mode(struct termios* tio); -extern void reset_terminal_mode(); extern SShellArguments args; extern int64_t result; diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index f2ab468574..e4cf09358b 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -358,7 +358,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { void *shellLoopQuery(void *arg) { if (indicator) { - get_old_terminal_mode(&oldtio); + getOldTerminalMode(); indicator = 0; } @@ -379,12 +379,12 @@ void *shellLoopQuery(void *arg) { do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); - set_terminal_mode(); + setTerminalMode(); err = shellReadCommand(con, command); if (err) { break; } - reset_terminal_mode(); + resetTerminalMode(); } while (shellRunCommand(con, command) == 0); tfree(command); @@ -395,56 +395,6 @@ void *shellLoopQuery(void *arg) { return NULL; } -int get_old_terminal_mode(struct termios *tio) { - /* Make sure stdin is a terminal. */ - if (!isatty(STDIN_FILENO)) { - return -1; - } - - // Get the parameter of current terminal - if (tcgetattr(0, &oldtio) != 0) { - return -1; - } - - return 1; -} - -void reset_terminal_mode() { - if (tcsetattr(0, TCSANOW, &oldtio) != 0) { - fprintf(stderr, "Fail to reset the terminal properties!\n"); - exit(EXIT_FAILURE); - } -} - -void set_terminal_mode() { - struct termios newtio; - - /* if (atexit(reset_terminal_mode) != 0) { */ - /* fprintf(stderr, "Error register exit function!\n"); */ - /* exit(EXIT_FAILURE); */ - /* } */ - - memcpy(&newtio, &oldtio, sizeof(oldtio)); - - // Set new terminal attributes. - newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP); - newtio.c_iflag |= IGNBRK; - - // newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET); - newtio.c_oflag |= OPOST; - newtio.c_oflag |= ONLCR; - newtio.c_oflag &= ~(OCRNL | ONLRET); - - newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG); - newtio.c_cc[VMIN] = 1; - newtio.c_cc[VTIME] = 0; - - if (tcsetattr(0, TCSANOW, &newtio) != 0) { - fprintf(stderr, "Fail to set terminal properties!\n"); - exit(EXIT_FAILURE); - } -} - void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { @@ -541,9 +491,9 @@ void showOnScreen(Command *cmd) { fflush(stdout); } -void cleanup_handler(void *arg) { tcsetattr(0, TCSANOW, &oldtio); } +void cleanup_handler(void *arg) { resetTerminalMode(); } void exitShell() { - tcsetattr(0, TCSANOW, &oldtio); + resetTerminalMode(); exit(EXIT_SUCCESS); } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 36489a448b..b42f77e87e 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -39,14 +39,14 @@ static int shellGetFilesNum(const char *directoryName, const char *prefix) char cmd[1024] = { 0 }; sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { + char buf[1024] = { 0 }; + if (taosSystem(cmd, buf, sizeof(buf)) < 0) { fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); exit(0); } int fileNum = 0; - if (fscanf(fp, "%d", &fileNum) != 1) { + if (sscanf(buf, "%d", &fileNum) != 1) { fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd); exit(0); } @@ -56,7 +56,6 @@ static int shellGetFilesNum(const char *directoryName, const char *prefix) exit(0); } - pclose(fp); return fileNum; } @@ -65,14 +64,14 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c char cmd[1024] = { 0 }; sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { + char buf[1024] = { 0 }; + if (taosSystem(cmd, buf, sizeof(buf)) < 0) { fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); exit(0); } int fileNum = 0; - while (fscanf(fp, "%128s", fileArray[fileNum++])) { + while (sscanf(buf, "%128s", fileArray[fileNum++])) { if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) { fileNum--; } @@ -85,8 +84,6 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName); exit(0); } - - pclose(fp); } static void shellCheckTablesSQLFile(const char *directoryName) diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 57d6df0051..cc497688d1 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -388,7 +388,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { void *shellLoopQuery(void *arg) { if (indicator) { - get_old_terminal_mode(&oldtio); + getOldTerminalMode(); indicator = 0; } @@ -409,12 +409,12 @@ void *shellLoopQuery(void *arg) { do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); - set_terminal_mode(); + setTerminalMode(); err = shellReadCommand(con, command); if (err) { break; } - reset_terminal_mode(); + resetTerminalMode(); } while (shellRunCommand(con, command) == 0); tfree(command); @@ -425,56 +425,6 @@ void *shellLoopQuery(void *arg) { return NULL; } -int get_old_terminal_mode(struct termios *tio) { - /* Make sure stdin is a terminal. */ - if (!isatty(STDIN_FILENO)) { - return -1; - } - - // Get the parameter of current terminal - if (tcgetattr(0, &oldtio) != 0) { - return -1; - } - - return 1; -} - -void reset_terminal_mode() { - if (tcsetattr(0, TCSANOW, &oldtio) != 0) { - fprintf(stderr, "Fail to reset the terminal properties!\n"); - exit(EXIT_FAILURE); - } -} - -void set_terminal_mode() { - struct termios newtio; - - /* if (atexit(reset_terminal_mode) != 0) { */ - /* fprintf(stderr, "Error register exit function!\n"); */ - /* exit(EXIT_FAILURE); */ - /* } */ - - memcpy(&newtio, &oldtio, sizeof(oldtio)); - - // Set new terminal attributes. - newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP); - newtio.c_iflag |= IGNBRK; - - // newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET); - newtio.c_oflag |= OPOST; - newtio.c_oflag |= ONLCR; - newtio.c_oflag &= ~(OCRNL | ONLRET); - - newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG); - newtio.c_cc[VMIN] = 1; - newtio.c_cc[VTIME] = 0; - - if (tcsetattr(0, TCSANOW, &newtio) != 0) { - fprintf(stderr, "Fail to set terminal properties!\n"); - exit(EXIT_FAILURE); - } -} - void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { @@ -571,10 +521,10 @@ void showOnScreen(Command *cmd) { fflush(stdout); } -void cleanup_handler(void *arg) { tcsetattr(0, TCSANOW, &oldtio); } +void cleanup_handler(void *arg) { resetTerminalMode(); } void exitShell() { - /*int32_t ret =*/ tcsetattr(STDIN_FILENO, TCSANOW, &oldtio); + /*int32_t ret =*/ resetTerminalMode(); taos_cleanup(); exit(EXIT_SUCCESS); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index f62c43773d..2832855517 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -41,11 +41,11 @@ void *cancelHandler(void *arg) { taosReleaseRef(tscObjRef, rid); #endif #else - reset_terminal_mode(); + resetTerminalMode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); #endif - reset_terminal_mode(); + resetTerminalMode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); } diff --git a/tools/taos-tools b/tools/taos-tools new file mode 160000 index 0000000000..f36b07f710 --- /dev/null +++ b/tools/taos-tools @@ -0,0 +1 @@ +Subproject commit f36b07f710d661dca88fdd70e73b5e3e16a960e0