xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #10691 from taosdata/feature/3.0_query_integrate_wxy 

TD-13747 merge 3.0
This commit is contained in:
Xiaoyu Wang 2022-03-11 09:21:15 +08:00 committed by GitHub
parent 89d7eb2d29 abb55e4a87
commit 8fcf2cc66a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
94 changed files with 2750 additions and 1303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -13,3 +13,6 @@
[submodule "examples/rust"]
path = examples/rust
url = https://github.com/songtianyi/tdengine-rust-bindings.git
[submodule "tools/taos-tools"]
path = tools/taos-tools
url = https://github.com/taosdata/taos-tools

26
CMakeLists.txt
View File

@ -6,6 +6,32 @@ project(
DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)"
)
IF ("${BUILD_TOOLS}" STREQUAL "")
IF (TD_LINUX)
IF (TD_ARM_32)
SET(BUILD_TOOLS "false")
ELSEIF (TD_ARM_64)
SET(BUILD_TOOLS "false")
ELSE ()
SET(BUILD_TOOLS "false")
ENDIF ()
ELSEIF (TD_DARWIN)
SET(BUILD_TOOLS "false")
ELSE ()
SET(BUILD_TOOLS "false")
ENDIF ()
ENDIF ()
IF ("${BUILD_TOOLS}" MATCHES "false")
MESSAGE("${Yellow} Will _not_ build taos_tools! ${ColourReset}")
SET(TD_TAOS_TOOLS FALSE)
ELSE ()
MESSAGE("")
MESSAGE("${Green} Will build taos_tools! ${ColourReset}")
MESSAGE("")
SET(TD_TAOS_TOOLS TRUE)
ENDIF ()
set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
include(${CMAKE_SUPPORT_DIR}/cmake.options)

4
include/client/taos.h
View File

@ -257,6 +257,10 @@ DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co
void tmqShowMsg(tmq_message_t *tmq_message);
int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT* stmt);
#ifdef __cplusplus
}
#endif

2
include/common/taosdef.h
View File

@ -63,6 +63,8 @@ typedef enum {
extern char *qtypeStr[];
#define TSDB_PORT_HTTP 11
#ifdef __cplusplus
}
#endif

146
include/common/tmsg.h
View File

@ -1129,18 +1129,34 @@ typedef struct {
char* sql;
char* physicalPlan;
char* logicalPlan;
} SMCreateTopicReq;
} SCMCreateStreamReq;
int32_t tSerializeMCreateTopicReq(void* buf, int32_t bufLen, const SMCreateTopicReq* pReq);
int32_t tDeserializeSMCreateTopicReq(void* buf, int32_t bufLen, SMCreateTopicReq* pReq);
void tFreeSMCreateTopicReq(SMCreateTopicReq* pReq);
typedef struct {
int64_t streamId;
} SCMCreateStreamRsp;
int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateStreamReq* pReq);
int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq);
void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq);
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
int8_t igExists;
char* sql;
char* physicalPlan;
char* logicalPlan;
} SCMCreateTopicReq;
int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq);
int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq);
void tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq);
typedef struct {
int64_t topicId;
} SMCreateTopicRsp;
} SCMCreateTopicRsp;
int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp);
int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp);
int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
typedef struct {
int32_t topicNum;
@ -1877,19 +1893,27 @@ typedef enum {
TD_TIME_UNIT_MICROSEC = 9,
TD_TIME_UNIT_NANOSEC = 10
} ETDTimeUnit;
typedef struct {
uint8_t version; // for compatibility
uint8_t intervalUnit;
uint8_t slidingUnit;
char indexName[TSDB_INDEX_NAME_LEN + 1];
col_id_t numOfColIds;
uint16_t numOfFuncIds;
uint64_t tableUid; // super/common table uid
int64_t interval;
int64_t sliding;
col_id_t* colIds; // sorted column ids
uint16_t* funcIds; // sorted sma function ids
} STSma; // Time-range-wise SMA
uint16_t funcId;
uint16_t nColIds;
col_id_t* colIds; // sorted colIds
} SFuncColIds;
typedef struct {
uint8_t version; // for compatibility
uint8_t intervalUnit;
uint8_t slidingUnit;
char indexName[TSDB_INDEX_NAME_LEN];
char timezone[TD_TIMEZONE_LEN];
uint16_t nFuncColIds;
uint16_t tagsFilterLen;
tb_uid_t tableUid; // super/common table uid
int64_t interval;
int64_t sliding;
SFuncColIds* funcColIds; // sorted funcIds
char* tagsFilter;
} STSma; // Time-range-wise SMA
typedef struct {
int64_t ver; // use a general definition
@ -1898,13 +1922,13 @@ typedef struct {
typedef struct {
int8_t type; // 0 status report, 1 update data
char indexName[TSDB_INDEX_NAME_LEN + 1]; //
char indexName[TSDB_INDEX_NAME_LEN]; //
STimeWindow windows;
} STSmaMsg;
typedef struct {
int64_t ver; // use a general definition
char indexName[TSDB_INDEX_NAME_LEN + 1];
char indexName[TSDB_INDEX_NAME_LEN];
} SVDropTSmaReq;
typedef struct {
} SVCreateTSmaRsp, SVDropTSmaRsp;
@ -1955,8 +1979,14 @@ typedef struct {
static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
if (pSma) {
tfree(pSma->colIds);
tfree(pSma->funcIds);
if (pSma->funcColIds != NULL) {
for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
tfree((pSma->funcColIds + i)->colIds);
}
tfree(pSma->funcColIds);
}
tfree(pSma->tagsFilter);
}
}
@ -1978,18 +2008,24 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
tlen += taosEncodeFixedU8(buf, pSma->intervalUnit);
tlen += taosEncodeFixedU8(buf, pSma->slidingUnit);
tlen += taosEncodeString(buf, pSma->indexName);
tlen += taosEncodeFixedU16(buf, pSma->numOfColIds);
tlen += taosEncodeFixedU16(buf, pSma->numOfFuncIds);
tlen += taosEncodeFixedU64(buf, pSma->tableUid);
tlen += taosEncodeString(buf, pSma->timezone);
tlen += taosEncodeFixedU16(buf, pSma->nFuncColIds);
tlen += taosEncodeFixedU16(buf, pSma->tagsFilterLen);
tlen += taosEncodeFixedI64(buf, pSma->tableUid);
tlen += taosEncodeFixedI64(buf, pSma->interval);
tlen += taosEncodeFixedI64(buf, pSma->sliding);
for (col_id_t i = 0; i < pSma->numOfColIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->colIds + i));
for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
SFuncColIds* funcColIds = pSma->funcColIds + i;
tlen += taosEncodeFixedU16(buf, funcColIds->funcId);
tlen += taosEncodeFixedU16(buf, funcColIds->nColIds);
for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
tlen += taosEncodeFixedU16(buf, *(funcColIds->colIds + j));
}
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
tlen += taosEncodeFixedU16(buf, *(pSma->funcIds + i));
if (pSma->tagsFilterLen > 0) {
tlen += taosEncodeString(buf, pSma->tagsFilter);
}
return tlen;
@ -2010,34 +2046,52 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
buf = taosDecodeFixedU8(buf, &pSma->intervalUnit);
buf = taosDecodeFixedU8(buf, &pSma->slidingUnit);
buf = taosDecodeStringTo(buf, pSma->indexName);
buf = taosDecodeFixedU16(buf, &pSma->numOfColIds);
buf = taosDecodeFixedU16(buf, &pSma->numOfFuncIds);
buf = taosDecodeFixedU64(buf, &pSma->tableUid);
buf = taosDecodeStringTo(buf, pSma->timezone);
buf = taosDecodeFixedU16(buf, &pSma->nFuncColIds);
buf = taosDecodeFixedU16(buf, &pSma->tagsFilterLen);
buf = taosDecodeFixedI64(buf, &pSma->tableUid);
buf = taosDecodeFixedI64(buf, &pSma->interval);
buf = taosDecodeFixedI64(buf, &pSma->sliding);
if (pSma->numOfColIds > 0) {
pSma->colIds = (col_id_t*)calloc(pSma->numOfColIds, sizeof(STSma));
if (pSma->colIds == NULL) {
if (pSma->nFuncColIds > 0) {
pSma->funcColIds = (SFuncColIds*)calloc(pSma->nFuncColIds, sizeof(SFuncColIds));
if (pSma->funcColIds == NULL) {
tdDestroyTSma(pSma);
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfColIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->colIds + i);
for (uint16_t i = 0; i < pSma->nFuncColIds; ++i) {
SFuncColIds* funcColIds = pSma->funcColIds + i;
buf = taosDecodeFixedU16(buf, &funcColIds->funcId);
buf = taosDecodeFixedU16(buf, &funcColIds->nColIds);
if (funcColIds->nColIds > 0) {
funcColIds->colIds = (col_id_t*)calloc(funcColIds->nColIds, sizeof(col_id_t));
if (funcColIds->colIds != NULL) {
for (uint16_t j = 0; j < funcColIds->nColIds; ++j) {
buf = taosDecodeFixedU16(buf, funcColIds->colIds + j);
}
} else {
tdDestroyTSma(pSma);
return NULL;
}
} else {
funcColIds->colIds = NULL;
}
}
} else {
pSma->colIds = NULL;
pSma->funcColIds = NULL;
}
if (pSma->numOfFuncIds > 0) {
pSma->funcIds = (uint16_t*)calloc(pSma->numOfFuncIds, sizeof(STSma));
if (pSma->funcIds == NULL) {
if (pSma->tagsFilterLen > 0) {
pSma->tagsFilter = (char*)calloc(pSma->tagsFilterLen, 1);
if (pSma->tagsFilter != NULL) {
buf = taosDecodeStringTo(buf, pSma->tagsFilter);
} else {
tdDestroyTSma(pSma);
return NULL;
}
for (uint16_t i = 0; i < pSma->numOfFuncIds; ++i) {
buf = taosDecodeFixedU16(buf, pSma->funcIds + i);
}
} else {
pSma->funcIds = NULL;
pSma->tagsFilter = NULL;
}
return buf;

3
include/common/tmsgdef.h
View File

@ -150,6 +150,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
// Requests handled by VNODE
TD_NEW_MSG_SEG(TDMT_VND_MSG)

19
include/dnode/mnode/sdb/sdb.h
View File

@ -113,15 +113,16 @@ typedef enum {
SDB_USER = 7,
SDB_AUTH = 8,
SDB_ACCT = 9,
SDB_OFFSET = 10,
SDB_SUBSCRIBE = 11,
SDB_CONSUMER = 12,
SDB_TOPIC = 13,
SDB_VGROUP = 14,
SDB_STB = 15,
SDB_DB = 16,
SDB_FUNC = 17,
SDB_MAX = 18
SDB_STREAM = 10,
SDB_OFFSET = 11,
SDB_SUBSCRIBE = 12,
SDB_CONSUMER = 13,
SDB_TOPIC = 14,
SDB_VGROUP = 15,
SDB_STB = 16,
SDB_DB = 17,
SDB_FUNC = 18,
SDB_MAX = 19
} ESdbType;
typedef struct SSdb SSdb;

30
include/libs/sync/sync.h
View File

@ -24,6 +24,7 @@ extern "C" {
#include <tdatablock.h>
#include "taosdef.h"
#include "trpc.h"
#include "wal.h"
typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId;
@ -87,25 +88,22 @@ typedef struct SSyncFSM {
} SSyncFSM;
struct SSyncRaftEntry;
typedef struct SSyncRaftEntry SSyncRaftEntry;
// abstract definition of log store in raft
// SWal implements it
typedef struct SSyncLogStore {
void* data;
// append one log entry
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SRpcMsg* pBuf);
int32_t (*appendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
// get one log entry, user need to free pBuf->data
int32_t (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SRpcMsg* pBuf);
// get one log entry, user need to free pEntry->pCont
SSyncRaftEntry* (*getEntry)(struct SSyncLogStore* pLogStore, SyncIndex index);
// update log store commit index with "index"
int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
// truncate log with index, entries after the given index (>index) will be deleted
int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex index);
// return commit index of log
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
// truncate log with index, entries after the given index (>=index) will be deleted
int32_t (*truncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
// return index of last entry
SyncIndex (*getLastIndex)(struct SSyncLogStore* pLogStore);
@ -113,6 +111,12 @@ typedef struct SSyncLogStore {
// return term of last entry
SyncTerm (*getLastTerm)(struct SSyncLogStore* pLogStore);
// update log store commit index with "index"
int32_t (*updateCommitIndex)(struct SSyncLogStore* pLogStore, SyncIndex index);
// return commit index of log
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
} SSyncLogStore;
// raft need to persist two variables in storage: currentTerm, voteFor
@ -134,7 +138,7 @@ typedef struct SSyncInfo {
SyncGroupId vgId;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
char walPath[TSDB_FILENAME_LEN];
SWal* pWal;
SSyncFSM* pFsm;
void* rpcClient;
@ -153,7 +157,7 @@ void syncCleanUp();
int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);

10
include/os/osSleep.h
View File

@ -20,7 +20,17 @@
extern "C" {
#endif
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define Sleep SLEEP_FUNC_TAOS_FORBID
#define sleep SLEEP_FUNC_TAOS_FORBID
#define usleep USLEEP_FUNC_TAOS_FORBID
#define nanosleep NANOSLEEP_FUNC_TAOS_FORBID
#endif
void taosSsleep(int32_t s);
void taosMsleep(int32_t ms);
void taosUsleep(int32_t us);
#ifdef __cplusplus
}

1
include/os/osSysinfo.h
View File

@ -47,7 +47,6 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes);
int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes);
int32_t taosSystem(const char *cmd);
void taosKillSystem();
int32_t taosGetSystemUUID(char *uid, int32_t uidlen);
char *taosGetCmdlineByPID(int32_t pid);

12
include/os/osSystem.h
View File

@ -20,11 +20,23 @@
extern "C" {
#endif
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define popen POPEN_FUNC_TAOS_FORBID
#define pclose PCLOSE_FUNC_TAOS_FORBID
#define tcsetattr TCSETATTR_FUNC_TAOS_FORBID
#define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
#endif
int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize);
void* taosLoadDll(const char* filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle);
int32_t taosSetConsoleEcho(bool on);
void setTerminalMode();
int32_t getOldTerminalMode();
void resetTerminalMode();
#ifdef __cplusplus
}

6
include/util/taoserror.h
View File

@ -267,9 +267,11 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8)
#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9)
#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA)
#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0)
// mnode-stream
#define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0)
#define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1)
#define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2)
// dnode
#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)

3
include/util/tdef.h
View File

@ -207,10 +207,11 @@ typedef enum ELogicConditionType {
#define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_MAX_RETRIEVE 1024
#define TSDB_INDEX_NAME_LEN 32
#define TSDB_INDEX_NAME_LEN 33 // 32 + 1 '\0'
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_STREAM_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20)
#define TSDB_COL_NAME_LEN 65

2
source/client/src/clientHb.c
View File

@ -450,7 +450,7 @@ static void hbStopThread() {
}
while (2 != atomic_load_8(&clientHbMgr.threadStop)) {
usleep(10);
taosUsleep(10);
}
tscDebug("hb thread stopped");

74
source/client/src/clientMain.c
View File

@ -343,3 +343,77 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
// TODO
}
TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) {
// TODO
return NULL;
}
TAOS_RES *taos_consume(TAOS_SUB *tsub) {
// TODO
return NULL;
}
void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
// TODO
}
TAOS_STMT* taos_stmt_init(TAOS* taos) {
// TODO
return NULL;
}
int taos_stmt_close(TAOS_STMT* stmt) {
// TODO
return -1;
}
int taos_stmt_execute(TAOS_STMT* stmt) {
// TODO
return -1;
}
char *taos_stmt_errstr(TAOS_STMT *stmt) {
// TODO
return NULL;
}
int taos_stmt_affected_rows(TAOS_STMT* stmt) {
// TODO
return -1;
}
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
// TODO
return NULL;
}
int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) {
// TODO
return -1;
}
int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
// TODO
return -1;
}
int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
// TODO
return -1;
}
int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
// TODO
return -1;
}
int taos_stmt_add_batch(TAOS_STMT* stmt) {
// TODO
return -1;
}
int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
// TODO
return -1;
}

16
source/client/src/tmq.c
View File

@ -511,7 +511,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB);
tNameFromString(&name, topicName, T_NAME_TABLE);
SMCreateTopicReq req = {
SCMCreateTopicReq req = {
.igExists = 1,
.physicalPlan = (char*)pStr,
.sql = (char*)sql,
@ -519,13 +519,13 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i
};
tNameExtractFullName(&name, req.name);
int tlen = tSerializeMCreateTopicReq(NULL, 0, &req);
int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req);
void* buf = malloc(tlen);
if (buf == NULL) {
goto _return;
}
tSerializeMCreateTopicReq(buf, tlen, &req);
tSerializeSCMCreateTopicReq(buf, tlen, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL};
@ -1146,13 +1146,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
if (taosArrayGetSize(tmq->clientTopics) == 0) {
tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
/*printf("over1\n");*/
usleep(blocking_time * 1000);
taosMsleep(blocking_time);
return NULL;
}
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
if (taosArrayGetSize(pTopic->vgs) == 0) {
/*printf("over2\n");*/
usleep(blocking_time * 1000);
taosMsleep(blocking_time);
return NULL;
}
@ -1165,14 +1165,14 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
if (pReq == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
taosMsleep(blocking_time);
return NULL;
}
SMqPollCbParam* param = malloc(sizeof(SMqPollCbParam));
if (param == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
taosMsleep(blocking_time);
return NULL;
}
param->tmq = tmq;
@ -1204,7 +1204,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
if (tmq_message == NULL) {
if (beginVgIdx == pTopic->nextVgIdx) {
usleep(blocking_time * 1000);
taosMsleep(blocking_time);
} else {
continue;
}

50
source/common/src/tmsg.c
View File

@ -1988,7 +1988,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR
return 0;
}
int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopicReq *pReq) {
int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTopicReq *pReq) {
int32_t sqlLen = 0;
int32_t physicalPlanLen = 0;
int32_t logicalPlanLen = 0;
@ -2016,7 +2016,7 @@ int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopic
return tlen;
}
int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq *pReq) {
int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicReq *pReq) {
int32_t sqlLen = 0;
int32_t physicalPlanLen = 0;
int32_t logicalPlanLen = 0;
@ -2045,13 +2045,13 @@ int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq
return 0;
}
void tFreeSMCreateTopicReq(SMCreateTopicReq *pReq) {
void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) {
tfree(pReq->sql);
tfree(pReq->physicalPlan);
tfree(pReq->logicalPlan);
}
int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) {
int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -2064,7 +2064,7 @@ int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopi
return tlen;
}
int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) {
int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
@ -2655,3 +2655,43 @@ void *tDeserializeSVDropTSmaReq(void *buf, SVDropTSmaReq *pReq) {
return buf;
}
int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->sql) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->physicalPlan) < 0) return -1;
if (tDecodeCStr(&decoder, (const char **)&pReq->logicalPlan) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
tfree(pReq->sql);
tfree(pReq->physicalPlan);
tfree(pReq->logicalPlan);
}

19
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -84,6 +84,7 @@ typedef enum {
TRN_TYPE_SUBSCRIBE = 1016,
TRN_TYPE_REBALANCE = 1017,
TRN_TYPE_COMMIT_OFFSET = 1018,
TRN_TYPE_CREATE_STREAM = 1019,
TRN_TYPE_BASIC_SCOPE_END,
TRN_TYPE_GLOBAL_SCOPE = 2000,
TRN_TYPE_CREATE_DNODE = 2001,
@ -679,7 +680,23 @@ static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pCons
}
typedef struct {
} SStreamScheduler;
char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
int64_t uid;
int64_t dbUid;
int32_t version;
SRWLatch lock;
int8_t status;
// int32_t sqlLen;
char* sql;
char* logicalPlan;
char* physicalPlan;
} SStreamObj;
int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);
int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj);
typedef struct SMnodeMsg {
char user[TSDB_USER_LEN];

38
source/dnode/mnode/impl/inc/mndStream.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_STREAM_H_
#define _TD_MND_STREAM_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_STREAM_H_*/

46
source/dnode/mnode/impl/src/mndDef.c Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndDef.h"
int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
return pEncoder->pos;
}
int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->sql) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->logicalPlan) < 0) return -1;
if (tDecodeCStr(pDecoder, (const char **)&pObj->physicalPlan) < 0) return -1;
return 0;
}

444
source/dnode/mnode/impl/src/mndStream.c Normal file
View File

@ -0,0 +1,444 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndStream.h"
#include "mndAuth.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tname.h"
#define MND_STREAM_VER_NUMBER 1
#define MND_STREAM_RESERVE_SIZE 64
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq);
/*static int32_t mndProcessDropStreamReq(SMnodeMsg *pReq);*/
/*static int32_t mndProcessDropStreamInRsp(SMnodeMsg *pRsp);*/
static int32_t mndProcessStreamMetaReq(SMnodeMsg *pReq);
static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextStream(SMnode *pMnode, void *pIter);
int32_t mndInitStream(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_STREAM,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndStreamActionEncode,
.decodeFp = (SdbDecodeFp)mndStreamActionDecode,
.insertFp = (SdbInsertFp)mndStreamActionInsert,
.updateFp = (SdbUpdateFp)mndStreamActionUpdate,
.deleteFp = (SdbDeleteFp)mndStreamActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq);
/*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
/*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TP, mndGetStreamMeta);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TP, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TP, mndCancelGetNextStream);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupStream(SMnode *pMnode) {}
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
if (tEncodeSStreamObj(NULL, pStream) < 0) {
tCoderClear(&encoder);
goto STREAM_ENCODE_OVER;
}
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
int32_t size = sizeof(int32_t) + tlen + MND_STREAM_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STREAM, MND_STREAM_VER_NUMBER, size);
if (pRaw == NULL) goto STREAM_ENCODE_OVER;
buf = malloc(tlen);
if (buf == NULL) goto STREAM_ENCODE_OVER;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
if (tEncodeSStreamObj(NULL, pStream) < 0) {
tCoderClear(&encoder);
goto STREAM_ENCODE_OVER;
}
tCoderClear(&encoder);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, STREAM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, STREAM_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, STREAM_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
STREAM_ENCODE_OVER:
tfree(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("stream:%s, encode to raw:%p, row:%p", pStream->name, pRaw, pStream);
return pRaw;
}
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
if (sver != MND_STREAM_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto STREAM_DECODE_OVER;
}
int32_t size = sizeof(SStreamObj);
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto STREAM_DECODE_OVER;
SStreamObj *pStream = sdbGetRowObj(pRow);
if (pStream == NULL) goto STREAM_DECODE_OVER;
int32_t tlen;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, dataPos, &tlen, STREAM_DECODE_OVER);
buf = malloc(tlen + 1);
if (buf == NULL) goto STREAM_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER);
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER);
if (tDecodeSStreamObj(&decoder, pStream) < 0) {
goto STREAM_DECODE_OVER;
}
terrno = TSDB_CODE_SUCCESS;
STREAM_DECODE_OVER:
tfree(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to decode from raw:%p since %s", pStream->name, pRaw, terrstr());
tfree(pRow);
return NULL;
}
mTrace("stream:%s, decode from raw:%p, row:%p", pStream->name, pRaw, pStream);
return pRow;
}
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream) {
mTrace("stream:%s, perform insert action", pStream->name);
return 0;
}
static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) {
mTrace("stream:%s, perform delete action", pStream->name);
return 0;
}
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) {
mTrace("stream:%s, perform update action", pOldStream->name);
atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime);
atomic_exchange_32(&pOldStream->version, pNewStream->version);
taosWLockLatch(&pOldStream->lock);
// TODO handle update
taosWUnLockLatch(&pOldStream->lock);
return 0;
}
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
}
return pStream;
}
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pStream);
}
static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) {
SName name = {0};
tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
char db[TSDB_STREAM_FNAME_LEN] = {0};
tNameGetFullDbName(&name, db);
return mndAcquireDb(pMnode, db);
}
static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION;
return -1;
}
return 0;
}
static int32_t mndCreateStream(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
mDebug("stream:%s to create", pCreate->name);
SStreamObj streamObj = {0};
tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
streamObj.createTime = taosGetTimestampMs();
streamObj.updateTime = streamObj.createTime;
streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
streamObj.dbUid = pDb->uid;
streamObj.version = 1;
streamObj.sql = pCreate->sql;
streamObj.physicalPlan = pCreate->physicalPlan;
streamObj.logicalPlan = pCreate->logicalPlan;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
if (pTrans == NULL) {
mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
}
static int32_t mndProcessCreateStreamReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
int32_t code = -1;
SStreamObj *pStream = NULL;
SDbObj *pDb = NULL;
SUserObj *pUser = NULL;
SCMCreateStreamReq createStreamReq = {0};
if (tDeserializeSCMCreateStreamReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createStreamReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CREATE_STREAM_OVER;
}
mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql);
if (mndCheckCreateStreamReq(&createStreamReq) != 0) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
goto CREATE_STREAM_OVER;
}
pStream = mndAcquireStream(pMnode, createStreamReq.name);
if (pStream != NULL) {
if (createStreamReq.igExists) {
mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name);
code = 0;
goto CREATE_STREAM_OVER;
} else {
terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
goto CREATE_STREAM_OVER;
}
} else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) {
goto CREATE_STREAM_OVER;
}
pDb = mndAcquireDbByStream(pMnode, createStreamReq.name);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
goto CREATE_STREAM_OVER;
}
pUser = mndAcquireUser(pMnode, pReq->user);
if (pUser == NULL) {
goto CREATE_STREAM_OVER;
}
if (mndCheckWriteAuth(pUser, pDb) != 0) {
goto CREATE_STREAM_OVER;
}
code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
CREATE_STREAM_OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
}
mndReleaseStream(pMnode, pStream);
mndReleaseDb(pMnode, pDb);
mndReleaseUser(pMnode, pUser);
tFreeSCMCreateStreamReq(&createStreamReq);
return code;
}
static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
int32_t numOfStreams = 0;
void *pIter = NULL;
while (1) {
SStreamObj *pStream = NULL;
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
if (pStream->dbUid == pDb->uid) {
numOfStreams++;
}
sdbRelease(pSdb, pStream);
}
*pNumOfStreams = numOfStreams;
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndGetStreamMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
SMnode *pMnode = pReq->pMnode;
SSdb *pSdb = pMnode->pSdb;
if (mndGetNumOfStreams(pMnode, pShow->db, &pShow->numOfRows) != 0) {
return -1;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->pSchemas;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = pShow->bytes[cols];
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = pShow->bytes[cols];
cols++;
pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sql");
pSchema[cols].bytes = pShow->bytes[cols];
cols++;
pMeta->numOfColumns = cols;
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = sdbGetSize(pSdb, SDB_STREAM);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbName, mndShowStr(pShow->type));
return 0;
}
static int32_t mndRetrieveStream(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pReq->pMnode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SStreamObj *pStream = NULL;
int32_t cols = 0;
char *pWrite;
char prefix[TSDB_DB_FNAME_LEN] = {0};
SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return 0;
tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = (int32_t)strlen(prefix);
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
if (pShow->pIter == NULL) break;
if (pStream->dbUid != pDb->uid) {
if (strncmp(pStream->name, prefix, prefixLen) != 0) {
mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid);
}
sdbRelease(pSdb, pStream);
continue;
}
cols = 0;
char streamName[TSDB_TABLE_NAME_LEN] = {0};
tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, streamName);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pStream->createTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]);
cols++;
numOfRows++;
sdbRelease(pSdb, pStream);
}
mndReleaseDb(pMnode, pDb);
pShow->numOfReads += numOfRows;
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
return numOfRows;
}
static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}

50
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndTopic.h"
#include "mndAuth.h"
#include "mndDb.h"
@ -229,7 +228,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
return pDrop;
}
static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
return -1;
@ -237,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) {
return 0;
}
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) {
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
mDebug("topic:%s to create", pCreate->name);
SMqTopicObj topicObj = {0};
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
@ -278,14 +277,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq
}
static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
int32_t code = -1;
SMqTopicObj *pTopic = NULL;
SDbObj *pDb = NULL;
SUserObj *pUser = NULL;
SMCreateTopicReq createTopicReq = {0};
SMnode *pMnode = pReq->pMnode;
int32_t code = -1;
SMqTopicObj *pTopic = NULL;
SDbObj *pDb = NULL;
SUserObj *pUser = NULL;
SCMCreateTopicReq createTopicReq = {0};
if (tDeserializeSMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
if (tDeserializeSCMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CREATE_TOPIC_OVER;
}
@ -338,7 +337,7 @@ CREATE_TOPIC_OVER:
mndReleaseDb(pMnode, pDb);
mndReleaseUser(pMnode, pUser);
tFreeSMCreateTopicReq(&createTopicReq);
tFreeSCMCreateTopicReq(&createTopicReq);
return code;
}
@ -409,35 +408,6 @@ static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) {
return 0;
}
#if 0
static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
int32_t numOfTopics = 0;
void *pIter = NULL;
while (1) {
SMqTopicObj *pTopic = NULL;
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
if (pIter == NULL) break;
if (pTopic->dbUid == pDb->uid) {
numOfTopics++;
}
sdbRelease(pSdb, pTopic);
}
*pNumOfTopics = numOfTopics;
mndReleaseDb(pMnode, pDb);
return 0;
}
#endif
static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);

10
source/dnode/mnode/impl/test/topic/topic.cpp
View File

@ -61,16 +61,16 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
}
void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql, int32_t* pContLen) {
SMCreateTopicReq createReq = {0};
SCMCreateTopicReq createReq = {0};
strcpy(createReq.name, topicName);
createReq.igExists = 0;
createReq.sql = (char*)sql;
createReq.physicalPlan = (char*)"physicalPlan";
createReq.logicalPlan = (char*)"logicalPlan";
int32_t contLen = tSerializeMCreateTopicReq(NULL, 0, &createReq);
int32_t contLen = tSerializeSCMCreateTopicReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
tSerializeMCreateTopicReq(pReq, contLen, &createReq);
tSerializeSCMCreateTopicReq(pReq, contLen, &createReq);
*pContLen = contLen;
return pReq;
@ -100,9 +100,7 @@ TEST_F(MndTestTopic, 01_Create_Topic) {
ASSERT_EQ(pRsp->code, 0);
}
{
test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, "");
}
{ test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, ""); }
{
int32_t contLen = 0;

42
source/dnode/snode/inc/sndInt.h
View File

@ -20,6 +20,7 @@
#include "tlog.h"
#include "tmsg.h"
#include "tqueue.h"
#include "trpc.h"
#include "snode.h"
@ -28,12 +29,51 @@
extern "C" {
#endif
enum {
STREAM_STATUS__READY = 1,
STREAM_STATUS__STOPPED,
STREAM_STATUS__CREATING,
STREAM_STATUS__STOPING,
STREAM_STATUS__RESUMING,
STREAM_STATUS__DELETING,
};
enum {
STREAM_RUNNER__RUNNING = 1,
STREAM_RUNNER__STOP,
};
typedef struct SSnode {
SSnodeOpt cfg;
} SSnode;
typedef struct {
int64_t streamId;
int32_t IdxInLevel;
int32_t level;
} SStreamInfo;
typedef struct {
SStreamInfo meta;
int8_t status;
void* executor;
STaosQueue* queue;
void* stateStore;
// storage handle
} SStreamRunner;
typedef struct {
SHashObj* pHash;
} SStreamMeta;
int32_t sndCreateStream();
int32_t sndDropStream();
int32_t sndStopStream();
int32_t sndResumeStream();
#ifdef __cplusplus
}
#endif
#endif /*_TD_SNODE_INT_H_*/
#endif /*_TD_SNODE_INT_H_*/

24
source/dnode/vnode/src/inc/tsdbFS.h
View File

@ -43,12 +43,30 @@ typedef struct {
STsdbFSMeta meta; // FS meta
SArray * df; // data file array
// SArray * v2f100.tsma.index_name
// SArray * v2t100.index_name
SArray * smaf; // sma data file array v2f1900.tsma.index_name
SArray * smaf; // sma data file array v2t1900.index_name
} SFSStatus;
typedef struct {
/**
* @brief Directory structure of .tsma data files.
*
* root@cary /vnode2/tsdb $ tree .tsma/
* .tsma/
* ├── v2t100.index_name_1
* ├── v2t101.index_name_1
* ├── v2t102.index_name_1
* ├── v2t1900.index_name_3
* ├── v2t1901.index_name_3
* ├── v2t1902.index_name_3
* ├── v2t200.index_name_2
* ├── v2t201.index_name_2
* └── v2t202.index_name_2
*
* 0 directories, 9 files
*/
typedef struct {
pthread_rwlock_t lock;
SFSStatus *cstatus; // current status

7
source/dnode/vnode/src/inc/tsdbFile.h
View File

@ -56,10 +56,11 @@ typedef enum {
TSDB_FILE_SMAL, // .smal(Block-wise SMA)
TSDB_FILE_MAX, //
TSDB_FILE_META, // meta
TSDB_FILE_TSMA, // .tsma.${sma_index_name}, Time-range-wise SMA
TSDB_FILE_RSMA, // .rsma.${sma_index_name}, Time-range-wise Rollup SMA
} TSDB_FILE_T;
TSDB_FILE_TSMA, // v2t100.${sma_index_name}, Time-range-wise SMA
TSDB_FILE_RSMA, // v2r100.${sma_index_name}, Time-range-wise Rollup SMA
} E_TSDB_FILE_T;
typedef int32_t TSDB_FILE_T;
typedef enum {
TSDB_FS_VER_0 = 0,
TSDB_FS_VER_MAX,

2
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -31,7 +31,7 @@ int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData, STimeW
int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg);
int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
int32_t tsdbFreeSmaState(SSmaStat *pSmaStat);
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat);
// internal func

4
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -23,8 +23,8 @@ static const char *TSDB_FNAME_SUFFIX[] = {
"smal", // TSDB_FILE_SMAL
"", // TSDB_FILE_MAX
"meta", // TSDB_FILE_META
"tsma", // TSDB_FILE_TSMA
"rsma", // TSDB_FILE_RSMA
"sma", // TSDB_FILE_TSMA(directory name)
"sma", // TSDB_FILE_RSMA(directory name)
};
static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);

2
source/dnode/vnode/src/tsdb/tsdbMain.c
View File

@ -89,7 +89,7 @@ static STsdb *tsdbNew(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg,
static void tsdbFree(STsdb *pTsdb) {
if (pTsdb) {
tsdbFreeFS(pTsdb->fs);
tsdbFreeSmaState(pTsdb->pSmaStat);
tsdbDestroySmaState(pTsdb->pSmaStat);
tfree(pTsdb->path);
free(pTsdb);
}

30
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -128,7 +128,7 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
return pItem;
}
int32_t tsdbFreeSmaState(SSmaStat *pSmaStat) {
int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
if (pSmaStat) {
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
SSmaStatItem *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
@ -203,6 +203,23 @@ int32_t tsdbUpdateExpiredWindow(STsdb *pTsdb, char *msg) {
return TSDB_CODE_SUCCESS;
}
static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, const char *indexName, void *timeWindow) {
SSmaStatItem *pItem = NULL;
if (pTsdb->pSmaStat && pTsdb->pSmaStat->smaStatItems) {
pItem = (SSmaStatItem *)taosHashGet(pTsdb->pSmaStat->smaStatItems, indexName, strlen(indexName));
}
if (pItem != NULL) {
// TODO: reset time windows for the sma data blocks
while (true) {
TSKEY thisWindow = 0;
taosHashRemove(pItem->expiredWindows, &thisWindow, sizeof(thisWindow));
}
}
return TSDB_CODE_SUCCESS;
}
/**
* @brief Judge the tSma storage level
*
@ -387,7 +404,7 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaData *pData, i
static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
// pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSma *param, STSmaData *pData, int32_t storageLevel,
@ -495,6 +512,9 @@ int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, STSma *param, STSmaData *pData) {
return terrno;
}
// reset the SSmaStat
tsdbResetExpiredWindow(pTsdb, param->indexName, &pData->tsWindow);
return TSDB_CODE_SUCCESS;
}
@ -542,6 +562,10 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
TASSERT(0);
return TSDB_CODE_INVALID_PARA;
}
// reset the SSmaStat
tsdbResetExpiredWindow(pTsdb, param->tsma.indexName, &pData->tsWindow);
// Step 4: finish
return TSDB_CODE_SUCCESS;
}
@ -558,7 +582,7 @@ int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, SRSma *param, STSmaData *pData) {
static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, STSma *param, STSmaData *pData) {
pSmaH->pTsdb = pTsdb;
pSmaH->interval = tsdbGetIntervalByPrecision(param->interval, param->intervalUnit, REPO_CFG(pTsdb)->precision);
pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
// pSmaH->blockSize = param->numOfFuncIds * sizeof(int64_t);
}
/**

101
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -20,6 +20,7 @@
#include <metaDef.h>
#include <tmsg.h>
#include <tsdbDef.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -41,17 +42,20 @@ TEST(testCase, tSmaEncodeDecodeTest) {
tSma.slidingUnit = TD_TIME_UNIT_HOUR;
tSma.sliding = 0;
tstrncpy(tSma.indexName, "sma_index_test", TSDB_INDEX_NAME_LEN);
tstrncpy(tSma.timezone, "Asia/Shanghai", TD_TIMEZONE_LEN);
tSma.tableUid = 1234567890;
tSma.numOfColIds = 2;
tSma.numOfFuncIds = 5; // sum/min/max/avg/last
tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t));
tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t));
for (int32_t i = 0; i < tSma.numOfColIds; ++i) {
*(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) {
*(tSma.funcIds + i) = (i + 2);
tSma.nFuncColIds = 5;
tSma.funcColIds = (SFuncColIds *)calloc(tSma.nFuncColIds, sizeof(SFuncColIds));
ASSERT(tSma.funcColIds != NULL);
for (int32_t n = 0; n < tSma.nFuncColIds; ++n) {
SFuncColIds *funcColIds = tSma.funcColIds + n;
funcColIds->funcId = n;
funcColIds->nColIds = 10;
funcColIds->colIds = (col_id_t *)calloc(funcColIds->nColIds, sizeof(col_id_t));
ASSERT(funcColIds->colIds != NULL);
for (int32_t i = 0; i < funcColIds->nColIds; ++i) {
*(funcColIds->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
}
STSmaWrapper tSmaWrapper = {.number = 1, .tSma = &tSma};
@ -80,16 +84,21 @@ TEST(testCase, tSmaEncodeDecodeTest) {
EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit);
EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit);
EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName);
EXPECT_EQ(pSma->numOfColIds, qSma->numOfColIds);
EXPECT_EQ(pSma->numOfFuncIds, qSma->numOfFuncIds);
EXPECT_STRCASEEQ(pSma->timezone, qSma->timezone);
EXPECT_EQ(pSma->nFuncColIds, qSma->nFuncColIds);
EXPECT_EQ(pSma->tableUid, qSma->tableUid);
EXPECT_EQ(pSma->interval, qSma->interval);
EXPECT_EQ(pSma->sliding, qSma->sliding);
for (uint32_t j = 0; j < pSma->numOfColIds; ++j) {
EXPECT_EQ(*(col_id_t *)(pSma->colIds + j), *(col_id_t *)(qSma->colIds + j));
}
for (uint32_t j = 0; j < pSma->numOfFuncIds; ++j) {
EXPECT_EQ(*(uint16_t *)(pSma->funcIds + j), *(uint16_t *)(qSma->funcIds + j));
EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen);
EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter);
for (uint32_t j = 0; j < pSma->nFuncColIds; ++j) {
SFuncColIds *pFuncColIds = pSma->funcColIds + j;
SFuncColIds *qFuncColIds = qSma->funcColIds + j;
EXPECT_EQ(pFuncColIds->funcId, qFuncColIds->funcId);
EXPECT_EQ(pFuncColIds->nColIds, qFuncColIds->nColIds);
for (uint32_t k = 0; k < pFuncColIds->nColIds; ++k) {
EXPECT_EQ(*(pFuncColIds->colIds + k), *(qFuncColIds->colIds + k));
}
}
}
@ -99,9 +108,11 @@ TEST(testCase, tSmaEncodeDecodeTest) {
}
TEST(testCase, tSma_DB_Put_Get_Del_Test) {
const char *smaIndexName1 = "sma_index_test_1";
const char *smaIndexName2 = "sma_index_test_2";
const char *smaTestDir = "./smaTest";
const char * smaIndexName1 = "sma_index_test_1";
const char * smaIndexName2 = "sma_index_test_2";
const char * timeZone = "Asia/Shanghai";
const char * tagsFilter = "I'm tags filter";
const char * smaTestDir = "./smaTest";
const uint64_t tbUid = 1234567890;
const uint32_t nCntTSma = 2;
// encode
@ -112,21 +123,27 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
tSma.slidingUnit = TD_TIME_UNIT_HOUR;
tSma.sliding = 0;
tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN);
tstrncpy(tSma.timezone, timeZone, TD_TIMEZONE_LEN);
tSma.tableUid = tbUid;
tSma.numOfColIds = 2;
tSma.numOfFuncIds = 5; // sum/min/max/avg/last
tSma.colIds = (col_id_t *)calloc(tSma.numOfColIds, sizeof(col_id_t));
tSma.funcIds = (uint16_t *)calloc(tSma.numOfFuncIds, sizeof(uint16_t));
for (int32_t i = 0; i < tSma.numOfColIds; ++i) {
*(tSma.colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
for (int32_t i = 0; i < tSma.numOfFuncIds; ++i) {
*(tSma.funcIds + i) = (i + 2);
tSma.nFuncColIds = 5;
tSma.funcColIds = (SFuncColIds *)calloc(tSma.nFuncColIds, sizeof(SFuncColIds));
ASSERT(tSma.funcColIds != NULL);
for (int32_t n = 0; n < tSma.nFuncColIds; ++n) {
SFuncColIds *funcColIds = tSma.funcColIds + n;
funcColIds->funcId = n;
funcColIds->nColIds = 10;
funcColIds->colIds = (col_id_t *)calloc(funcColIds->nColIds, sizeof(col_id_t));
ASSERT(funcColIds->colIds != NULL);
for (int32_t i = 0; i < funcColIds->nColIds; ++i) {
*(funcColIds->colIds + i) = (i + PRIMARYKEY_TIMESTAMP_COL_ID);
}
}
tSma.tagsFilterLen = strlen(tagsFilter);
tSma.tagsFilter = (char *)calloc(tSma.tagsFilterLen + 1, 1);
tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
SMeta * pMeta = NULL;
STSma * pSmaCfg = &tSma;
STSma * pSmaCfg = &tSma;
const SMetaCfg *pMetaCfg = &defaultMetaOptions;
taosRemoveDir(smaTestDir);
@ -151,6 +168,8 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName1);
assert(qSmaCfg != NULL);
printf("name1 = %s\n", qSmaCfg->indexName);
printf("timezone1 = %s\n", qSmaCfg->timezone);
printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid);
tdDestroyTSma(qSmaCfg);
@ -159,6 +178,8 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
qSmaCfg = metaGetSmaInfoByName(pMeta, smaIndexName2);
assert(qSmaCfg != NULL);
printf("name2 = %s\n", qSmaCfg->indexName);
printf("timezone2 = %s\n", qSmaCfg->timezone);
printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
EXPECT_EQ(qSmaCfg->interval, tSma.interval);
tdDestroyTSma(qSmaCfg);
@ -169,7 +190,7 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
assert(pSmaCur != NULL);
uint32_t indexCnt = 0;
while (1) {
const char* indexName = metaSmaCursorNext(pSmaCur);
const char *indexName = metaSmaCursorNext(pSmaCur);
if (indexName == NULL) {
break;
}
@ -184,10 +205,14 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
assert(pSW != NULL);
EXPECT_EQ(pSW->number, nCntTSma);
EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1);
EXPECT_STRCASEEQ(pSW->tSma->timezone, timeZone);
EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter);
EXPECT_EQ(pSW->tSma->tableUid, tSma.tableUid);
EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2);
EXPECT_STRCASEEQ((pSW->tSma + 1)->timezone, timeZone);
EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter);
EXPECT_EQ((pSW->tSma + 1)->tableUid, tSma.tableUid);
tdDestroyTSmaWrapper(pSW);
tfree(pSW);
@ -211,9 +236,9 @@ TEST(testCase, tSma_DB_Put_Get_Del_Test) {
#if 0
TEST(testCase, tSmaInsertTest) {
STSma tSma = {0};
STSmaData* pSmaData = NULL;
STsdb tsdb = {0};
STSma tSma = {0};
STSmaData *pSmaData = NULL;
STsdb tsdb = {0};
// init
tSma.intervalUnit = TD_TIME_UNIT_DAY;
@ -226,7 +251,7 @@ TEST(testCase, tSmaInsertTest) {
int32_t dataLen = numOfColIds * numOfBlocks * blockSize;
pSmaData = (STSmaData*)malloc(sizeof(STSmaData) + dataLen);
pSmaData = (STSmaData *)malloc(sizeof(STSmaData) + dataLen);
ASSERT_EQ(pSmaData != NULL, true);
pSmaData->tableUid = 3232329230;
pSmaData->numOfColIds = numOfColIds;
@ -234,7 +259,7 @@ TEST(testCase, tSmaInsertTest) {
pSmaData->dataLen = dataLen;
pSmaData->tsWindow.skey = 1640000000;
pSmaData->tsWindow.ekey = 1645788649;
pSmaData->colIds = (col_id_t*)malloc(sizeof(col_id_t) * numOfColIds);
pSmaData->colIds = (col_id_t *)malloc(sizeof(col_id_t) * numOfColIds);
ASSERT_EQ(pSmaData->colIds != NULL, true);
for (int32_t i = 0; i < numOfColIds; ++i) {

2
source/libs/catalog/src/catalog.c
View File

@ -2705,7 +2705,7 @@ void catalogDestroy(void) {
tsem_post(&gCtgMgmt.sem);
while (CTG_IS_LOCKED(&gCtgMgmt.lock)) {
usleep(1);
taosUsleep(1);
}
CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock);

90
source/libs/catalog/test/catalogTests.cpp
View File

@ -723,7 +723,7 @@ void *ctgTestGetDbVgroupThread(void *param) {
}
if (ctgTestEnableSleep) {
usleep(taosRand() % 5);
taosUsleep(taosRand() % 5);
}
if (++n % ctgTestPrintNum == 0) {
printf("Get:%d\n", n);
@ -747,7 +747,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) {
}
if (ctgTestEnableSleep) {
usleep(taosRand() % 5);
taosUsleep(taosRand() % 5);
}
if (++n % ctgTestPrintNum == 0) {
printf("Set:%d\n", n);
@ -771,7 +771,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) {
}
if (ctgTestEnableSleep) {
usleep(taosRand() % 5);
taosUsleep(taosRand() % 5);
}
if (++n % ctgTestPrintNum == 0) {
printf("Set:%d\n", n);
@ -801,7 +801,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
tfree(tbMeta);
if (ctgTestEnableSleep) {
usleep(taosRand() % 5);
taosUsleep(taosRand() % 5);
}
if (++n % ctgTestPrintNum == 0) {
@ -838,7 +838,7 @@ void *ctgTestSetCtableMetaThread(void *param) {
}
if (ctgTestEnableSleep) {
usleep(taosRand() % 5);
taosUsleep(taosRand() % 5);
}
if (++n % ctgTestPrintNum == 0) {
printf("Set:%d\n", n);
@ -880,7 +880,7 @@ TEST(tableMeta, normalTable) {
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
usleep(50000);
taosMsleep(50);
}
ctgTestSetRspTableMeta();
@ -901,7 +901,7 @@ TEST(tableMeta, normalTable) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -949,7 +949,7 @@ TEST(tableMeta, normalTable) {
allDbNum += dbNum;
allStbNum += stbNum;
sleep(2);
taosSsleep(2);
}
ASSERT_EQ(allDbNum, 1);
@ -996,7 +996,7 @@ TEST(tableMeta, childTableCase) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1058,7 +1058,7 @@ TEST(tableMeta, childTableCase) {
allDbNum += dbNum;
allStbNum += stbNum;
sleep(2);
taosSsleep(2);
}
ASSERT_EQ(allDbNum, 1);
@ -1105,7 +1105,7 @@ TEST(tableMeta, superTableCase) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1132,7 +1132,7 @@ TEST(tableMeta, superTableCase) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (2 != n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1181,7 +1181,7 @@ TEST(tableMeta, superTableCase) {
allDbNum += dbNum;
allStbNum += stbNum;
sleep(2);
taosSsleep(2);
}
ASSERT_EQ(allDbNum, 1);
@ -1230,7 +1230,7 @@ TEST(tableMeta, rmStbMeta) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1244,7 +1244,7 @@ TEST(tableMeta, rmStbMeta) {
int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
if (n || m) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1300,7 +1300,7 @@ TEST(tableMeta, updateStbMeta) {
while (true) {
uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
if (0 == n) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1320,7 +1320,7 @@ TEST(tableMeta, updateStbMeta) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -1392,7 +1392,7 @@ TEST(refreshGetMeta, normal2normal) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1410,7 +1410,7 @@ TEST(refreshGetMeta, normal2normal) {
tfree(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
@ -1471,7 +1471,7 @@ TEST(refreshGetMeta, normal2notexist) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1489,7 +1489,7 @@ TEST(refreshGetMeta, normal2notexist) {
tfree(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
@ -1545,7 +1545,7 @@ TEST(refreshGetMeta, normal2child) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1563,7 +1563,7 @@ TEST(refreshGetMeta, normal2child) {
tfree(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
@ -1629,7 +1629,7 @@ TEST(refreshGetMeta, stable2child) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1648,7 +1648,7 @@ TEST(refreshGetMeta, stable2child) {
tfree(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
ctgTestCurrentSTableName = ctgTestSTablename;
@ -1714,7 +1714,7 @@ TEST(refreshGetMeta, stable2stable) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1733,7 +1733,7 @@ TEST(refreshGetMeta, stable2stable) {
tfree(tableMeta);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
code = catalogRefreshGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
@ -1802,7 +1802,7 @@ TEST(refreshGetMeta, child2stable) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
STableMeta *tableMeta = NULL;
@ -1819,7 +1819,7 @@ TEST(refreshGetMeta, child2stable) {
tfree(tableMeta);
while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
usleep(50000);
taosMsleep(50);
}
ctgTestCurrentSTableName = ctgTestTablename;
@ -2019,7 +2019,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
if (n > 0) {
break;
}
usleep(50000);
taosMsleep(50);
}
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
@ -2043,7 +2043,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
uint64_t n = 0;
ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
if (n != 3) {
usleep(50000);
taosMsleep(50);
} else {
break;
}
@ -2100,20 +2100,20 @@ TEST(multiThread, getSetRmSameDbVgroup) {
pthread_t thread1, thread2;
pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg);
sleep(1);
taosSsleep(1);
pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg);
while (true) {
if (ctgTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(ctgTestMTRunSec);
taosSsleep(ctgTestMTRunSec);
break;
}
}
ctgTestStop = true;
sleep(1);
taosSsleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -2152,20 +2152,20 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
pthread_t thread1, thread2;
pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg);
sleep(1);
taosSsleep(1);
pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg);
while (true) {
if (ctgTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(ctgTestMTRunSec);
taosSsleep(ctgTestMTRunSec);
break;
}
}
ctgTestStop = true;
sleep(1);
taosSsleep(1);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -2203,20 +2203,20 @@ TEST(multiThread, ctableMeta) {
pthread_t thread1, thread2;
pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg);
sleep(1);
taosSsleep(1);
pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg);
while (true) {
if (ctgTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(ctgTestMTRunSec);
taosSsleep(ctgTestMTRunSec);
break;
}
}
ctgTestStop = true;
sleep(2);
taosSsleep(2);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -2267,7 +2267,7 @@ TEST(rentTest, allRent) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
usleep(50000);
taosMsleep(50);
}
code = catalogGetExpiredDBs(pCtg, &dbs, &num);
@ -2292,7 +2292,7 @@ TEST(rentTest, allRent) {
}
printf("*************************************************\n");
sleep(2);
taosSsleep(2);
}
catalogDestroy();

6
source/libs/qcom/test/queryTest.cpp
View File

@ -63,7 +63,7 @@ int main(int argc, char** argv) {
TEST(testCase, async_task_test) {
SParam* p = (SParam*)calloc(1, sizeof(SParam));
taosAsyncExec(testPrint, p, NULL);
usleep(5000);
taosMsleep(5);
}
TEST(testCase, many_async_task_test) {
@ -73,14 +73,14 @@ TEST(testCase, many_async_task_test) {
taosAsyncExec(testPrint, p, NULL);
}
usleep(10000);
taosMsleep(10);
}
TEST(testCase, error_in_async_test) {
int32_t code = 0;
SParam* p = (SParam*) calloc(1, sizeof(SParam));
taosAsyncExec(testPrintError, p, &code);
usleep(1000);
taosMsleep(1);
printf("Error code:%d after asynchronously exec function\n", code);
}

64
source/libs/qworker/test/qworkerTests.cpp
View File

@ -308,7 +308,7 @@ int32_t qwtExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
if (qwtTestEnableSleep) {
if (runTime) {
usleep(runTime);
taosUsleep(runTime);
}
}
@ -590,7 +590,7 @@ void *queryThread(void *param) {
qwtBuildQueryReqMsg(&queryRpc);
qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
if (qwtTestEnableSleep) {
usleep(taosRand()%5);
taosUsleep(taosRand()%5);
}
if (++n % qwtTestPrintNum == 0) {
printf("query:%d\n", n);
@ -612,7 +612,7 @@ void *readyThread(void *param) {
qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
if (qwtTestEnableSleep) {
usleep(taosRand()%5);
taosUsleep(taosRand()%5);
}
if (++n % qwtTestPrintNum == 0) {
printf("ready:%d\n", n);
@ -634,7 +634,7 @@ void *fetchThread(void *param) {
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
if (qwtTestEnableSleep) {
usleep(taosRand()%5);
taosUsleep(taosRand()%5);
}
if (++n % qwtTestPrintNum == 0) {
printf("fetch:%d\n", n);
@ -656,7 +656,7 @@ void *dropThread(void *param) {
qwtBuildDropReqMsg(&dropMsg, &dropRpc);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
if (qwtTestEnableSleep) {
usleep(taosRand()%5);
taosUsleep(taosRand()%5);
}
if (++n % qwtTestPrintNum == 0) {
printf("drop:%d\n", n);
@ -678,7 +678,7 @@ void *statusThread(void *param) {
qwtBuildStatusReqMsg(&statusMsg, &statusRpc);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
if (qwtTestEnableSleep) {
usleep(taosRand()%5);
taosUsleep(taosRand()%5);
}
if (++n % qwtTestPrintNum == 0) {
printf("status:%d\n", n);
@ -696,7 +696,7 @@ void *qwtclientThread(void *param) {
void *mockPointer = (void *)0x1;
SRpcMsg queryRpc = {0};
sleep(1);
taosSsleep(1);
while (!qwtTestStop) {
qwtTestCaseFinished = false;
@ -705,7 +705,7 @@ void *qwtclientThread(void *param) {
qwtPutReqToQueue((void *)0x1, &queryRpc);
while (!qwtTestCaseFinished) {
usleep(1);
taosUsleep(1);
}
@ -751,7 +751,7 @@ void *queryQueueThread(void *param) {
int32_t delay = taosRand() % qwtTestReqMaxDelayUsec;
if (delay) {
usleep(delay);
taosUsleep(delay);
}
}
@ -807,7 +807,7 @@ void *fetchQueueThread(void *param) {
int32_t delay = taosRand() % qwtTestReqMaxDelayUsec;
if (delay) {
usleep(delay);
taosUsleep(delay);
}
}
@ -982,21 +982,21 @@ TEST(seqTest, randCase) {
qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
if (qwtTestEnableSleep) {
usleep(1);
taosUsleep(1);
}
} else if (r >= maxr * 2/5 && r < maxr* 3/5) {
printf("Fetch,%d\n", t++);
qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
if (qwtTestEnableSleep) {
usleep(1);
taosUsleep(1);
}
} else if (r >= maxr * 3/5 && r < maxr * 4/5) {
printf("Drop,%d\n", t++);
qwtBuildDropReqMsg(&dropMsg, &dropRpc);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
if (qwtTestEnableSleep) {
usleep(1);
taosUsleep(1);
}
} else if (r >= maxr * 4/5 && r < maxr-1) {
printf("Status,%d\n", t++);
@ -1004,7 +1004,7 @@ TEST(seqTest, randCase) {
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
if (qwtTestEnableSleep) {
usleep(1);
taosUsleep(1);
}
} else {
printf("QUIT RAND NOW");
@ -1042,15 +1042,15 @@ TEST(seqTest, multithreadRand) {
while (true) {
if (qwtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(qwtTestMTRunSec);
taosSsleep(qwtTestMTRunSec);
break;
}
}
qwtTestStop = true;
sleep(3);
taosSsleep(3);
qWorkerDestroy(&mgmt);
}
@ -1099,9 +1099,9 @@ TEST(rcTest, shortExecshortDelay) {
while (true) {
if (qwtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(qwtTestMTRunSec);
taosSsleep(qwtTestMTRunSec);
break;
}
}
@ -1113,14 +1113,14 @@ TEST(rcTest, shortExecshortDelay) {
break;
}
sleep(1);
taosSsleep(1);
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
usleep(10);
taosUsleep(10);
}
}
@ -1180,9 +1180,9 @@ TEST(rcTest, longExecshortDelay) {
while (true) {
if (qwtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(qwtTestMTRunSec);
taosSsleep(qwtTestMTRunSec);
break;
}
}
@ -1195,14 +1195,14 @@ TEST(rcTest, longExecshortDelay) {
break;
}
sleep(1);
taosSsleep(1);
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
usleep(10);
taosUsleep(10);
}
}
@ -1263,9 +1263,9 @@ TEST(rcTest, shortExeclongDelay) {
while (true) {
if (qwtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(qwtTestMTRunSec);
taosSsleep(qwtTestMTRunSec);
break;
}
}
@ -1278,14 +1278,14 @@ TEST(rcTest, shortExeclongDelay) {
break;
}
sleep(1);
taosSsleep(1);
if (qwtTestCaseFinished) {
if (qwtTestQuitThreadNum < 3) {
tsem_post(&qwtTestQuerySem);
tsem_post(&qwtTestFetchSem);
usleep(10);
taosUsleep(10);
}
}
@ -1342,15 +1342,15 @@ TEST(rcTest, dropTest) {
while (true) {
if (qwtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(qwtTestMTRunSec);
taosSsleep(qwtTestMTRunSec);
break;
}
}
qwtTestStop = true;
sleep(3);
taosSsleep(3);
qWorkerDestroy(&mgmt);
}

14
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -345,7 +345,7 @@ void *schtSendRsp(void *param) {
break;
}
usleep(1000);
taosMsleep(1);
}
pJob = schAcquireJob(job);
@ -370,7 +370,7 @@ void *schtCreateFetchRspThread(void *param) {
int64_t job = *(int64_t *)param;
SSchJob* pJob = schAcquireJob(job);
sleep(1);
taosSsleep(1);
int32_t code = 0;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)calloc(1, sizeof(SRetrieveTableRsp));
@ -394,7 +394,7 @@ void *schtFetchRspThread(void *aa) {
continue;
}
usleep(1);
taosUsleep(1);
param = (SSchCallbackParam *)calloc(1, sizeof(*param));
@ -599,7 +599,7 @@ void* schtRunJobThread(void *aa) {
void* schtFreeJobThread(void *aa) {
while (!schtTestStop) {
usleep(taosRand() % 100);
taosUsleep(taosRand() % 100);
schtFreeQueryJob(1);
}
}
@ -861,15 +861,15 @@ TEST(multiThread, forceFree) {
while (true) {
if (schtTestDeadLoop) {
sleep(1);
taosSsleep(1);
} else {
sleep(schtTestMTRunSec);
taosSsleep(schtTestMTRunSec);
break;
}
}
schtTestStop = true;
sleep(3);
taosSsleep(3);
}
int main(int argc, char** argv) {

1
source/libs/sync/inc/syncAppendEntries.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
// TLA+ Spec

1
source/libs/sync/inc/syncAppendEntriesReply.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
// TLA+ Spec

10
source/libs/sync/inc/syncInt.h
View File

@ -116,7 +116,8 @@ typedef struct SSyncNode {
SyncGroupId vgId;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
char walPath[TSDB_FILENAME_LEN];
char raftStorePath[TSDB_FILENAME_LEN * 2];
SWal* pWal;
void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
void* queue;
@ -195,8 +196,6 @@ typedef struct SSyncNode {
SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
void syncNodeClose(SSyncNode* pSyncNode);
cJSON* syncNode2Json(const SSyncNode* pSyncNode);
char* syncNode2Str(const SSyncNode* pSyncNode);
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
@ -213,6 +212,11 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms);
int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
// for debug
cJSON* syncNode2Json(const SSyncNode* pSyncNode);
char* syncNode2Str(const SSyncNode* pSyncNode);
void syncNodePrint(char* s, const SSyncNode* pSyncNode);
#ifdef __cplusplus
}
#endif

16
source/libs/sync/inc/syncMessage.h
View File

@ -24,8 +24,7 @@ extern "C" {
#include <stdio.h>
#include <stdlib.h>
#include "cJSON.h"
#include "sync.h"
#include "syncRaftEntry.h"
#include "syncInt.h"
#include "taosdef.h"
// encode as uint32
@ -46,6 +45,7 @@ typedef enum ESyncMessageType {
// ---------------------------------------------
cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg);
cJSON* syncRpcUnknownMsg2Json();
char* syncRpcMsg2Str(SRpcMsg* pRpcMsg);
// ---------------------------------------------
typedef enum ESyncTimeoutType {
@ -123,12 +123,22 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId);
typedef struct SyncClientRequest {
uint32_t bytes;
uint32_t msgType;
int64_t seqNum;
uint32_t originalRpcType;
uint64_t seqNum;
bool isWeak;
uint32_t dataLen;
char data[];
} SyncClientRequest;
SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
void syncClientRequestDestroy(SyncClientRequest* pMsg);
void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);
void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg);
SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak);
// ---------------------------------------------
typedef struct SyncClientRequestReply {
uint32_t bytes;

1
source/libs/sync/inc/syncOnMessage.h
View File

@ -23,7 +23,6 @@ extern "C" {
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncRaft.h"
#include "taosdef.h"
#ifdef __cplusplus

93
source/libs/sync/inc/syncRaft.h
View File

@ -1,93 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_RAFT_H
#define _TD_LIBS_SYNC_RAFT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "syncMessage.h"
#include "taosdef.h"
#if 0
typedef struct SRaftId {
SyncNodeId addr;
SyncGroupId vgId;
} SRaftId;
typedef struct SRaft {
SRaftId id;
SSyncFSM* pFsm;
int32_t (*FpPing)(struct SRaft* ths, const RaftPing* pMsg);
int32_t (*FpOnPing)(struct SRaft* ths, RaftPing* pMsg);
int32_t (*FpOnPingReply)(struct SRaft* ths, RaftPingReply* pMsg);
int32_t (*FpRequestVote)(struct SRaft* ths, const RaftRequestVote* pMsg);
int32_t (*FpOnRequestVote)(struct SRaft* ths, RaftRequestVote* pMsg);
int32_t (*FpOnRequestVoteReply)(struct SRaft* ths, RaftRequestVoteReply* pMsg);
int32_t (*FpAppendEntries)(struct SRaft* ths, const RaftAppendEntries* pMsg);
int32_t (*FpOnAppendEntries)(struct SRaft* ths, RaftAppendEntries* pMsg);
int32_t (*FpOnAppendEntriesReply)(struct SRaft* ths, RaftAppendEntriesReply* pMsg);
} SRaft;
SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm);
void raftClose(SRaft* pRaft);
static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg);
static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg);
static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg);
static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg);
static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg);
static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg);
static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg);
static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg);
static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg);
int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak);
static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft);
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_RAFT_H*/

24
source/libs/sync/inc/syncRaftEntry.h
View File

@ -24,15 +24,31 @@ extern "C" {
#include <stdio.h>
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "taosdef.h"
typedef struct SSyncRaftEntry {
SyncTerm term;
SyncIndex index;
SSyncBuffer data;
int8_t flag;
uint32_t bytes;
uint32_t msgType;
uint32_t originalRpcType;
uint64_t seqNum;
bool isWeak;
SyncTerm term;
SyncIndex index;
uint32_t dataLen;
char data[];
} SSyncRaftEntry;
SSyncRaftEntry* syncEntryBuild(uint32_t dataLen);
SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);
void syncEntryDestory(SSyncRaftEntry* pEntry);
char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len);
SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len);
cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry);
char* syncEntry2Str(const SSyncRaftEntry* pEntry);
void syncEntryPrint(const SSyncRaftEntry* pEntry);
void syncEntryPrint2(char *s, const SSyncRaftEntry* pEntry);
#ifdef __cplusplus
}
#endif

42
source/libs/sync/inc/syncRaftLog.h
View File

@ -24,27 +24,47 @@ extern "C" {
#include <stdio.h>
#include <stdlib.h>
#include "syncInt.h"
#include "syncRaftEntry.h"
#include "taosdef.h"
int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf);
typedef struct SSyncLogStoreData {
SSyncNode* pSyncNode;
SWal* pWal;
} SSyncLogStoreData;
// get one log entry, user need to free pBuf->data
int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf);
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode);
// update log store commit index with "index"
int32_t raftLogUpdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index);
void logStoreDestory(SSyncLogStore* pLogStore);
// truncate log with index, entries after the given index (>index) will be deleted
int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index);
// append one log entry
int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
// return commit index of log
SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore);
// get one log entry, user need to free pEntry->pCont
SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index);
// truncate log with index, entries after the given index (>=index) will be deleted
int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex);
// return index of last entry
SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore);
SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore);
// return term of last entry
SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore);
SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore);
// update log store commit index with "index"
int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index);
// return commit index of log
SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore);
SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore);
cJSON* logStore2Json(SSyncLogStore* pLogStore);
char* logStore2Str(SSyncLogStore* pLogStore);
// for debug
void logStorePrint(SSyncLogStore* pLogStore);
#ifdef __cplusplus
}

1
source/libs/sync/inc/syncRaftStore.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "cJSON.h"
#include "syncInt.h"
#include "syncRaft.h"
#include "taosdef.h"
#define RAFT_STORE_BLOCK_SIZE 512

1
source/libs/sync/inc/syncRequestVote.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
// TLA+ Spec

1
source/libs/sync/inc/syncRequestVoteReply.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
// TLA+ Spec

1
source/libs/sync/inc/syncSnapshot.h
View File

@ -24,7 +24,6 @@ extern "C" {
#include <stdio.h>
#include <stdlib.h>
#include "syncInt.h"
#include "syncRaft.h"
#include "taosdef.h"
int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);

1
source/libs/sync/inc/syncTimeout.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include <stdlib.h>
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
// TLA+ Spec

3
source/libs/sync/inc/syncUtil.h
View File

@ -49,6 +49,9 @@ cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p);
cJSON* syncUtilRaftId2Json(const SRaftId* p);
char* syncUtilRaftId2Str(const SRaftId* p);
const char* syncUtilState2String(ESyncState state);
bool syncUtilCanPrint(char c);
char* syncUtilprintBin(char* ptr, uint32_t len);
char* syncUtilprintBin2(char* ptr, uint32_t len);
#ifdef __cplusplus
}

19
source/libs/sync/src/syncIO.c
View File

@ -15,7 +15,7 @@
#include "syncIO.h"
#include <tdatablock.h>
#include "syncOnMessage.h"
#include "syncMessage.h"
#include "tglobal.h"
#include "ttimer.h"
#include "tutil.h"
@ -220,12 +220,17 @@ static void *syncIOConsumerFunc(void *param) {
while (1) {
int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL);
sTrace("syncIOConsumerFunc %d msgs are received", numOfMsgs);
if (numOfMsgs <= 0) break;
if (numOfMsgs <= 0) {
break;
}
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
char *s = syncRpcMsg2Str(pRpcMsg);
sTrace("syncIOConsumerFunc get item from queue: msgType:%d contLen:%d msg:%s", pRpcMsg->msgType, pRpcMsg->contLen,
(char *)(pRpcMsg->pCont));
s);
free(s);
if (pRpcMsg->msgType == SYNC_PING) {
if (io->FpOnSyncPing != NULL) {
@ -247,7 +252,7 @@ static void *syncIOConsumerFunc(void *param) {
}
} else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) {
if (io->FpOnSyncRequestVote) {
if (io->FpOnSyncRequestVote != NULL) {
SyncRequestVote *pSyncMsg;
pSyncMsg = syncRequestVoteBuild(pRpcMsg->contLen);
syncRequestVoteFromRpcMsg(pRpcMsg, pSyncMsg);
@ -256,7 +261,7 @@ static void *syncIOConsumerFunc(void *param) {
}
} else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) {
if (io->FpOnSyncRequestVoteReply) {
if (io->FpOnSyncRequestVoteReply != NULL) {
SyncRequestVoteReply *pSyncMsg;
pSyncMsg = SyncRequestVoteReplyBuild();
syncRequestVoteReplyFromRpcMsg(pRpcMsg, pSyncMsg);
@ -265,7 +270,7 @@ static void *syncIOConsumerFunc(void *param) {
}
} else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) {
if (io->FpOnSyncAppendEntries) {
if (io->FpOnSyncAppendEntries != NULL) {
SyncAppendEntries *pSyncMsg;
pSyncMsg = syncAppendEntriesBuild(pRpcMsg->contLen);
syncAppendEntriesFromRpcMsg(pRpcMsg, pSyncMsg);
@ -274,7 +279,7 @@ static void *syncIOConsumerFunc(void *param) {
}
} else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) {
if (io->FpOnSyncAppendEntriesReply) {
if (io->FpOnSyncAppendEntriesReply != NULL) {
SyncAppendEntriesReply *pSyncMsg;
pSyncMsg = syncAppendEntriesReplyBuild();
syncAppendEntriesReplyFromRpcMsg(pRpcMsg, pSyncMsg);

68
source/libs/sync/src/syncMain.c
View File

@ -18,8 +18,9 @@
#include "syncAppendEntries.h"
#include "syncAppendEntriesReply.h"
#include "syncEnv.h"
#include "syncIndexMgr.h"
#include "syncInt.h"
#include "syncRaft.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncRequestVote.h"
#include "syncRequestVoteReply.h"
@ -59,13 +60,32 @@ int64_t syncStart(const SSyncInfo* pSyncInfo) {
return 0;
}
void syncStop(int64_t rid) {}
void syncStop(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid
syncNodeClose(pSyncNode);
}
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; }
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak) { return 0; }
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) {
SSyncNode* pSyncNode = NULL; // get pointer from rid
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, 0, isWeak);
SRpcMsg rpcMsg;
syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
syncClientRequestDestroy(pSyncMsg);
} else {
sTrace("syncForwardToPeer not leader, %s", syncUtilState2String(pSyncNode->state));
return -1; // need define err code !!
}
return 0;
}
ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; }
ESyncState syncGetMyRole(int64_t rid) {
SSyncNode* pSyncNode = NULL; // get pointer from rid
return pSyncNode->state;
}
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {}
@ -78,7 +98,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
pSyncNode->vgId = pSyncInfo->vgId;
pSyncNode->syncCfg = pSyncInfo->syncCfg;
memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath));
snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s/raft_store.json", pSyncInfo->path);
pSyncNode->pWal = pSyncInfo->pWal;
pSyncNode->rpcClient = pSyncInfo->rpcClient;
pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg;
pSyncNode->queue = pSyncInfo->queue;
@ -114,20 +135,27 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
// init life cycle
// init server vars
// init TLA+ server vars
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath);
pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath);
assert(pSyncNode->pRaftStore != NULL);
// init candidate vars
// init TLA+ candidate vars
pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode);
assert(pSyncNode->pVotesGranted != NULL);
pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode);
assert(pSyncNode->pVotesRespond != NULL);
// init leader vars
pSyncNode->pNextIndex = NULL;
pSyncNode->pMatchIndex = NULL;
// init TLA+ leader vars
pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode);
assert(pSyncNode->pNextIndex != NULL);
pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode);
assert(pSyncNode->pMatchIndex != NULL);
// init TLA+ log vars
pSyncNode->pLogStore = logStoreCreate(pSyncNode);
assert(pSyncNode->pLogStore != NULL);
pSyncNode->commitIndex = 0;
// init ping timer
pSyncNode->pPingTimer = NULL;
@ -177,7 +205,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
// init by SSyncInfo
cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId);
cJSON_AddStringToObject(pRoot, "path", pSyncNode->path);
cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient);
cJSON_AddStringToObject(pRoot, "rpcClient", u64buf);
@ -298,6 +327,13 @@ char* syncNode2Str(const SSyncNode* pSyncNode) {
return serialized;
}
void syncNodePrint(char* s, const SSyncNode* pSyncNode) {
char* ss = syncNode2Str(pSyncNode);
// sTrace("syncNodePrint: %s [len:%lu]| %s", s, strlen(ss), ss);
fprintf(stderr, "syncNodePrint: %s [len:%lu]| %s", s, strlen(ss), ss);
free(ss);
}
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilraftId2EpSet(destRaftId, &epSet);
@ -472,6 +508,8 @@ static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
}
static void syncNodeEqPingTimer(void* param, void* tmrId) {
sTrace("<-- syncNodeEqPingTimer -->");
SSyncNode* pSyncNode = (SSyncNode*)param;
if (atomic_load_64(&pSyncNode->pingTimerLogicClockUser) <= atomic_load_64(&pSyncNode->pingTimerLogicClock)) {
SyncTimeout* pSyncMsg = syncTimeoutBuild2(SYNC_TIMEOUT_PING, atomic_load_64(&pSyncNode->pingTimerLogicClock),
@ -484,7 +522,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) {
// reset timer ms
// pSyncNode->pingTimerMS += 100;
taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager,
taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
&pSyncNode->pPingTimer);
} else {
sTrace("syncNodeEqPingTimer: pingTimerLogicClock:%lu, pingTimerLogicClockUser:%lu", pSyncNode->pingTimerLogicClock,
@ -506,7 +544,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
// reset timer ms
pSyncNode->electTimerMS = syncUtilElectRandomMS();
taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, &gSyncEnv->pTimerManager,
taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager,
&pSyncNode->pPingTimer);
} else {
sTrace("syncNodeEqElectTimer: electTimerLogicClock:%lu, electTimerLogicClockUser:%lu",
@ -530,7 +568,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
// reset timer ms
// pSyncNode->heartbeatTimerMS += 100;
taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager,
taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager,
&pSyncNode->pHeartbeatTimer);
} else {
sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu",

84
source/libs/sync/src/syncMessage.c
View File

@ -14,7 +14,6 @@
*/
#include "syncMessage.h"
#include "syncRaft.h"
#include "syncUtil.h"
#include "tcoding.h"
@ -36,7 +35,8 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) {
pRoot = syncPingReply2Json(pSyncMsg);
} else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) {
pRoot = syncRpcUnknownMsg2Json();
SyncClientRequest* pSyncMsg = (SyncClientRequest*)pRpcMsg->pCont;
pRoot = syncClientRequest2Json(pSyncMsg);
} else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST_REPLY) {
pRoot = syncRpcUnknownMsg2Json();
@ -76,6 +76,13 @@ cJSON* syncRpcUnknownMsg2Json() {
return pJson;
}
char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) {
cJSON* pJson = syncRpcMsg2Json(pRpcMsg);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
// ---- message process SyncTimeout----
SyncTimeout* syncTimeoutBuild() {
uint32_t bytes = sizeof(SyncTimeout);
@ -149,6 +156,7 @@ SyncPing* syncPingBuild(uint32_t dataLen) {
pMsg->bytes = bytes;
pMsg->msgType = SYNC_PING;
pMsg->dataLen = dataLen;
return pMsg;
}
void syncPingDestroy(SyncPing* pMsg) {
@ -247,6 +255,7 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) {
pMsg->bytes = bytes;
pMsg->msgType = SYNC_PING_REPLY;
pMsg->dataLen = dataLen;
return pMsg;
}
void syncPingReplyDestroy(SyncPingReply* pMsg) {
@ -337,6 +346,73 @@ SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId)
return pMsg;
}
// ---- message process SyncClientRequest----
SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) {
uint32_t bytes = sizeof(SyncClientRequest) + dataLen;
SyncClientRequest* pMsg = malloc(bytes);
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_CLIENT_REQUEST;
pMsg->seqNum = 0;
pMsg->isWeak = false;
pMsg->dataLen = dataLen;
return pMsg;
}
void syncClientRequestDestroy(SyncClientRequest* pMsg) {
if (pMsg != NULL) {
free(pMsg);
}
}
void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) {
assert(pMsg->bytes <= bufLen);
memcpy(buf, pMsg, pMsg->bytes);
}
void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) {
memcpy(pMsg, buf, len);
assert(len == pMsg->bytes);
}
void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg) {
memset(pRpcMsg, 0, sizeof(*pRpcMsg));
pRpcMsg->msgType = pMsg->msgType;
pRpcMsg->contLen = pMsg->bytes;
pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
syncClientRequestSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
}
void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg) {
syncClientRequestDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
}
cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType);
snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum);
cJSON_AddStringToObject(pRoot, "seqNum", u64buf);
cJSON_AddNumberToObject(pRoot, "isWeak", pMsg->isWeak);
cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SyncClientRequest", pRoot);
return pJson;
}
SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak) {
SyncClientRequest* pMsg = syncClientRequestBuild(pOriginalRpcMsg->contLen);
pMsg->originalRpcType = pOriginalRpcMsg->msgType;
pMsg->seqNum = seqNum;
pMsg->isWeak = isWeak;
memcpy(pMsg->data, pOriginalRpcMsg->pCont, pOriginalRpcMsg->contLen);
return pMsg;
}
// ---- message process SyncRequestVote----
SyncRequestVote* syncRequestVoteBuild() {
uint32_t bytes = sizeof(SyncRequestVote);
@ -344,6 +420,7 @@ SyncRequestVote* syncRequestVoteBuild() {
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_REQUEST_VOTE;
return pMsg;
}
void syncRequestVoteDestroy(SyncRequestVote* pMsg) {
@ -429,6 +506,7 @@ SyncRequestVoteReply* SyncRequestVoteReplyBuild() {
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_REQUEST_VOTE_REPLY;
return pMsg;
}
void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) {
@ -512,6 +590,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen) {
pMsg->bytes = bytes;
pMsg->msgType = SYNC_APPEND_ENTRIES;
pMsg->dataLen = dataLen;
return pMsg;
}
void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) {
@ -604,6 +683,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild() {
memset(pMsg, 0, bytes);
pMsg->bytes = bytes;
pMsg->msgType = SYNC_APPEND_ENTRIES_REPLY;
return pMsg;
}
void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) {

70
source/libs/sync/src/syncRaft.c
View File

@ -1,70 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRaft.h"
#include "sync.h"
#if 0
SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm) {
SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft));
assert(pRaft != NULL);
pRaft->id = raftId;
pRaft->pFsm = pFsm;
pRaft->FpPing = doRaftPing;
pRaft->FpOnPing = onRaftPing;
pRaft->FpOnPingReply = onRaftPingReply;
pRaft->FpRequestVote = doRaftRequestVote;
pRaft->FpOnRequestVote = onRaftRequestVote;
pRaft->FpOnRequestVoteReply = onRaftRequestVoteReply;
pRaft->FpAppendEntries = doRaftAppendEntries;
pRaft->FpOnAppendEntries = onRaftAppendEntries;
pRaft->FpOnAppendEntriesReply = onRaftAppendEntriesReply;
return pRaft;
}
void raftClose(SRaft* pRaft) {
assert(pRaft != NULL);
free(pRaft);
}
static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg) { return 0; }
static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg) { return 0; }
static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg) { return 0; }
static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg) { return 0; }
static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg) { return 0; }
static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg) { return 0; }
static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg) { return 0; }
static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg) { return 0; }
static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg) { return 0; }
int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; }
static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; }
#endif

101
source/libs/sync/src/syncRaftEntry.c
View File

@ -14,3 +14,104 @@
*/
#include "syncRaftEntry.h"
#include "syncUtil.h"
SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) {
uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen;
SSyncRaftEntry* pEntry = malloc(bytes);
assert(pEntry != NULL);
memset(pEntry, 0, bytes);
pEntry->bytes = bytes;
pEntry->dataLen = dataLen;
return pEntry;
}
SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) {
SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen);
assert(pEntry != NULL);
pEntry->msgType = pMsg->msgType;
pEntry->originalRpcType = pMsg->originalRpcType;
pEntry->seqNum = pMsg->seqNum;
pEntry->isWeak = pMsg->isWeak;
pEntry->term = term;
pEntry->index = index;
pEntry->dataLen = pMsg->dataLen;
memcpy(pEntry->data, pMsg->data, pMsg->dataLen);
return pEntry;
}
void syncEntryDestory(SSyncRaftEntry* pEntry) {
if (pEntry != NULL) {
free(pEntry);
}
}
char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) {
char* buf = malloc(pEntry->bytes);
assert(buf != NULL);
memcpy(buf, pEntry, pEntry->bytes);
if (len != NULL) {
*len = pEntry->bytes;
}
return buf;
}
SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) {
uint32_t bytes = *((uint32_t*)buf);
SSyncRaftEntry* pEntry = malloc(bytes);
assert(pEntry != NULL);
memcpy(pEntry, buf, len);
assert(len == pEntry->bytes);
return pEntry;
}
cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes);
cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType);
cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType);
snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->seqNum);
cJSON_AddStringToObject(pRoot, "seqNum", u64buf);
cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak);
snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->term);
cJSON_AddStringToObject(pRoot, "term", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index);
cJSON_AddStringToObject(pRoot, "index", u64buf);
cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen);
char* s;
s = syncUtilprintBin((char*)(pEntry->data), pEntry->dataLen);
cJSON_AddStringToObject(pRoot, "data", s);
free(s);
s = syncUtilprintBin2((char*)(pEntry->data), pEntry->dataLen);
cJSON_AddStringToObject(pRoot, "data2", s);
free(s);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SSyncRaftEntry", pRoot);
return pJson;
}
char* syncEntry2Str(const SSyncRaftEntry* pEntry) {
cJSON* pJson = syncEntry2Json(pEntry);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
void syncEntryPrint(const SSyncRaftEntry* pEntry) {
char* s = syncEntry2Str(pEntry);
sTrace("%s", s);
free(s);
}
void syncEntryPrint2(char* s, const SSyncRaftEntry* pEntry) {
char* ss = syncEntry2Str(pEntry);
sTrace("%s | %s", s, ss);
free(ss);
}

182
source/libs/sync/src/syncRaftLog.c
View File

@ -14,46 +14,160 @@
*/
#include "syncRaftLog.h"
#include "wal.h"
int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; }
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
SSyncLogStore* pLogStore = malloc(sizeof(SSyncLogStore));
assert(pLogStore != NULL);
// get one log entry, user need to free pBuf->data
int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf) { return 0; }
pLogStore->data = malloc(sizeof(SSyncLogStoreData));
assert(pLogStore->data != NULL);
// TLA+ Spec
// \* Leader i advances its commitIndex.
// \* This is done as a separate step from handling AppendEntries responses,
// \* in part to minimize atomic regions, and in part so that leaders of
// \* single-server clusters are able to mark entries committed.
// AdvanceCommitIndex(i) ==
// /\ state[i] = Leader
// /\ LET \* The set of servers that agree up through index.
// Agree(index) == {i} \cup {k \in Server :
// matchIndex[i][k] >= index}
// \* The maximum indexes for which a quorum agrees
// agreeIndexes == {index \in 1..Len(log[i]) :
// Agree(index) \in Quorum}
// \* New value for commitIndex'[i]
// newCommitIndex ==
// IF /\ agreeIndexes /= {}
// /\ log[i][Max(agreeIndexes)].term = currentTerm[i]
// THEN
// Max(agreeIndexes)
// ELSE
// commitIndex[i]
// IN commitIndex' = [commitIndex EXCEPT ![i] = newCommitIndex]
// /\ UNCHANGED <<messages, serverVars, candidateVars, leaderVars, log>>
//
int32_t raftLogupdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; }
SSyncLogStoreData* pData = pLogStore->data;
pData->pSyncNode = pSyncNode;
pData->pWal = pSyncNode->pWal;
// truncate log with index, entries after the given index (>index) will be deleted
int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; }
pLogStore->appendEntry = logStoreAppendEntry;
pLogStore->getEntry = logStoreGetEntry;
pLogStore->truncate = logStoreTruncate;
pLogStore->getLastIndex = logStoreLastIndex;
pLogStore->getLastTerm = logStoreLastTerm;
pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
pLogStore->getCommitIndex = logStoreGetCommitIndex;
}
// return commit index of log
SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore) { return 0; }
void logStoreDestory(SSyncLogStore* pLogStore) {
if (pLogStore != NULL) {
free(pLogStore->data);
free(pLogStore);
}
}
// append one log entry
int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
assert(pEntry->index == logStoreLastIndex(pLogStore) + 1);
uint32_t len;
char* serialized = syncEntrySerialize(pEntry, &len);
assert(serialized != NULL);
int code;
code = walWrite(pWal, pEntry->index, pEntry->msgType, serialized, len);
assert(code == 0);
walFsync(pWal, true);
free(serialized);
}
// get one log entry, user need to free pEntry->pCont
SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
SSyncRaftEntry* pEntry;
SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
walReadWithHandle(pWalHandle, index);
pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len);
assert(pEntry != NULL);
// need to hold, do not new every time!!
walCloseReadHandle(pWalHandle);
return pEntry;
}
// truncate log with index, entries after the given index (>=index) will be deleted
int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
walRollback(pWal, fromIndex);
}
// return index of last entry
SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore) { return 0; }
SyncIndex logStoreLastIndex(SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
SyncIndex lastIndex = walGetLastVer(pWal);
return lastIndex;
}
// return term of last entry
SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore) { return 0; }
SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) {
SyncTerm lastTerm = 0;
SSyncRaftEntry* pLastEntry = logStoreGetLastEntry(pLogStore);
if (pLastEntry != NULL) {
lastTerm = pLastEntry->term;
free(pLastEntry);
}
return lastTerm;
}
// update log store commit index with "index"
int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
walCommit(pWal, index);
}
// return commit index of log
SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
return pData->pSyncNode->commitIndex;
}
SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
SyncIndex lastIndex = walGetLastVer(pWal);
SSyncRaftEntry* pEntry = NULL;
if (lastIndex > 0) {
pEntry = logStoreGetEntry(pLogStore, lastIndex);
}
return pEntry;
}
cJSON* logStore2Json(SSyncLogStore* pLogStore) {
char u64buf[128];
SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
cJSON* pRoot = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastIndex(pLogStore));
cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", logStoreLastTerm(pLogStore));
cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
cJSON* pEntries = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "pEntries", pEntries);
SyncIndex lastIndex = logStoreLastIndex(pLogStore);
for (SyncIndex i = 0; i <= lastIndex; ++i) {
SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i);
cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
syncEntryDestory(pEntry);
}
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
return pJson;
}
char* logStore2Str(SSyncLogStore* pLogStore) {
cJSON* pJson = logStore2Json(pLogStore);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
// for debug
void logStorePrint(SSyncLogStore* pLogStore) {
char* s = logStore2Str(pLogStore);
// sTrace("%s", s);
fprintf(stderr, "logStorePrint: [len:%lu]| %s \n", strlen(s), s);
free(s);
}

1
source/libs/sync/src/syncSnapshot.c
View File

@ -14,7 +14,6 @@
*/
#include "syncSnapshot.h"
#include "syncRaft.h"
int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; }

36
source/libs/sync/src/syncUtil.c
View File

@ -148,4 +148,40 @@ const char* syncUtilState2String(ESyncState state) {
} else {
return "TAOS_SYNC_STATE_UNKNOWN";
}
}
bool syncUtilCanPrint(char c) {
if (c >= 32 && c <= 126) {
return true;
} else {
return false;
}
}
char* syncUtilprintBin(char* ptr, uint32_t len) {
char* s = malloc(len + 1);
assert(s != NULL);
memset(s, 0, len + 1);
memcpy(s, ptr, len);
for (int i = 0; i < len; ++i) {
if (!syncUtilCanPrint(s[i])) {
s[i] = '.';
}
}
return s;
}
char* syncUtilprintBin2(char* ptr, uint32_t len) {
uint32_t len2 = len * 4 + 1;
char* s = malloc(len2);
assert(s != NULL);
memset(s, 0, len2);
char* p = s;
for (int i = 0; i < len; ++i) {
int n = sprintf(p, "%d,", ptr[i]);
p += n;
}
return s;
}

30
source/libs/sync/test/CMakeLists.txt
View File

@ -15,6 +15,8 @@ add_executable(syncUtilTest "")
add_executable(syncVotesGrantedTest "")
add_executable(syncVotesRespondTest "")
add_executable(syncIndexMgrTest "")
add_executable(syncLogStoreTest "")
add_executable(syncEntryTest "")
target_sources(syncTest
@ -85,6 +87,14 @@ target_sources(syncIndexMgrTest
PRIVATE
"syncIndexMgrTest.cpp"
)
target_sources(syncLogStoreTest
PRIVATE
"syncLogStoreTest.cpp"
)
target_sources(syncEntryTest
PRIVATE
"syncEntryTest.cpp"
)
target_include_directories(syncTest
@ -172,6 +182,16 @@ target_include_directories(syncIndexMgrTest
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncLogStoreTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncEntryTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(syncTest
@ -242,6 +262,14 @@ target_link_libraries(syncIndexMgrTest
sync
gtest_main
)
target_link_libraries(syncLogStoreTest
sync
gtest_main
)
target_link_libraries(syncEntryTest
sync
gtest_main
)
enable_testing()
@ -249,3 +277,5 @@ add_test(
NAME sync_test
COMMAND syncTest
)

1
source/libs/sync/test/syncEnqTest.cpp
View File

@ -2,6 +2,7 @@
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncMessage.h"
#include "syncRaftStore.h"
void logTest() {

81
source/libs/sync/test/syncEntryTest.cpp Normal file
View File

@ -0,0 +1,81 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
void test1() {
SSyncRaftEntry* pEntry = syncEntryBuild(10);
assert(pEntry != NULL);
pEntry->msgType = 1;
pEntry->originalRpcType = 2;
pEntry->seqNum = 3;
pEntry->isWeak = true;
pEntry->term = 100;
pEntry->index = 200;
strcpy(pEntry->data, "test1");
syncEntryPrint(pEntry);
syncEntryDestory(pEntry);
}
void test2() {
SyncClientRequest* pSyncMsg = syncClientRequestBuild(10);
pSyncMsg->originalRpcType = 33;
pSyncMsg->seqNum = 11;
pSyncMsg->isWeak = 1;
strcpy(pSyncMsg->data, "test2");
SSyncRaftEntry* pEntry = syncEntryBuild2(pSyncMsg, 100, 200);
syncEntryPrint(pEntry);
syncClientRequestDestroy(pSyncMsg);
syncEntryDestory(pEntry);
}
void test3() {
SSyncRaftEntry* pEntry = syncEntryBuild(10);
assert(pEntry != NULL);
pEntry->msgType = 11;
pEntry->originalRpcType = 22;
pEntry->seqNum = 33;
pEntry->isWeak = true;
pEntry->term = 44;
pEntry->index = 55;
strcpy(pEntry->data, "test3");
syncEntryPrint(pEntry);
uint32_t len;
char* serialized = syncEntrySerialize(pEntry, &len);
assert(serialized != NULL);
SSyncRaftEntry* pEntry2 = syncEntryDeserialize(serialized, len);
syncEntryPrint(pEntry2);
free(serialized);
syncEntryDestory(pEntry2);
syncEntryDestory(pEntry);
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
test1();
test2();
test3();
return 0;
}

31
source/libs/sync/test/syncEnvTest.cpp
View File

@ -3,6 +3,7 @@
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "ttime.h"
void logTest() {
sTrace("--- sync log test: trace");
@ -13,24 +14,13 @@ void logTest() {
sFatal("--- sync log test: fatal");
}
void doSync() {
SSyncInfo syncInfo;
syncInfo.vgId = 1;
void *pTimer = NULL;
void *pTimerMgr = NULL;
int g = 300;
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->replicaNum = 3;
pCfg->nodeInfo[0].nodePort = 7010;
taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
pCfg->nodeInfo[1].nodePort = 7110;
taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
pCfg->nodeInfo[2].nodePort = 7210;
taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
static void timerFp(void *param, void *tmrId) {
printf("param:%p, tmrId:%p, pTimer:%p, pTimerMgr:%p \n", param, tmrId, pTimer, pTimerMgr);
taosTmrReset(timerFp, 1000, param, pTimerMgr, &pTimer);
}
int main() {
@ -41,13 +31,12 @@ int main() {
logTest();
// ret = syncIOStart();
// assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
// doSync();
// timer
pTimerMgr = taosTmrInit(1000, 50, 10000, "SYNC-ENV-TEST");
taosTmrStart(timerFp, 1000, &g, pTimerMgr);
while (1) {
taosMsleep(1000);

4
source/libs/sync/test/syncIOSendMsgClientTest.cpp
View File

@ -39,11 +39,11 @@ int main() {
rpcMsg.msgType = 77;
syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg);
sleep(1);
taosSsleep(1);
}
while (1) {
sleep(1);
taosSsleep(1);
}
return 0;

2
source/libs/sync/test/syncIOSendMsgServerTest.cpp
View File

@ -26,7 +26,7 @@ int main() {
assert(ret == 0);
while (1) {
sleep(1);
taosSsleep(1);
}
return 0;

4
source/libs/sync/test/syncIOSendMsgTest.cpp
View File

@ -39,11 +39,11 @@ int main() {
rpcMsg.msgType = 77;
syncIOSendMsg(gSyncIO->clientRpc, &epSet, &rpcMsg);
sleep(1);
taosSsleep(1);
}
while (1) {
sleep(1);
taosSsleep(1);
}
return 0;

2
source/libs/sync/test/syncIOTickPingTest.cpp
View File

@ -29,7 +29,7 @@ int main() {
assert(ret == 0);
while (1) {
sleep(1);
taosSsleep(1);
}
return 0;
}

2
source/libs/sync/test/syncIOTickQTest.cpp
View File

@ -29,7 +29,7 @@ int main() {
assert(ret == 0);
while (1) {
sleep(1);
taosSsleep(1);
}
return 0;
}

3
source/libs/sync/test/syncIndexMgrTest.cpp
View File

@ -33,8 +33,7 @@ SSyncNode* syncNodeInit() {
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;

12
source/libs/sync/test/syncInitTest.cpp
View File

@ -30,8 +30,7 @@ SSyncNode* syncNodeInit() {
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
@ -54,6 +53,7 @@ SSyncNode* syncNodeInit() {
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
@ -89,11 +89,11 @@ int main(int argc, char** argv) {
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
char* serialized = syncNode2Str(pSyncNode);
printf("%s\n", serialized);
free(serialized);
syncNodePrint((char*)"syncInitTest", pSyncNode);
initRaftId(pSyncNode);
//--------------------------------------------------------------
return 0;
}
}

141
source/libs/sync/test/syncLogStoreTest.cpp Normal file
View File

@ -0,0 +1,141 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 1;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SWal* pWal;
SSyncNode* pSyncNode;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
int code = walInit();
assert(code == 0);
SWalCfg walCfg;
memset(&walCfg, 0, sizeof(SWalCfg));
walCfg.vgId = syncInfo.vgId;
walCfg.fsyncPeriod = 1000;
walCfg.retentionPeriod = 1000;
walCfg.rollPeriod = 1000;
walCfg.retentionSize = 1000;
walCfg.segSize = 1000;
walCfg.level = TAOS_WAL_FSYNC;
pWal = walOpen("./wal_test", &walCfg);
assert(pWal != NULL);
syncInfo.pWal = pWal;
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode* syncInitTest() { return syncNodeInit(); }
void logStoreTest() {
logStorePrint(pSyncNode->pLogStore);
for (int i = 0; i < 5; ++i) {
int32_t dataLen = 10;
SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
assert(pEntry != NULL);
pEntry->msgType = 1;
pEntry->originalRpcType = 2;
pEntry->seqNum = 3;
pEntry->isWeak = true;
pEntry->term = 100;
pEntry->index = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore) + 1;
snprintf(pEntry->data, dataLen, "value%d", i);
//syncEntryPrint2((char*)"write entry:", pEntry);
pSyncNode->pLogStore->appendEntry(pSyncNode->pLogStore, pEntry);
syncEntryDestory(pEntry);
}
logStorePrint(pSyncNode->pLogStore);
pSyncNode->pLogStore->truncate(pSyncNode->pLogStore, 3);
logStorePrint(pSyncNode->pLogStore);
}
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
//syncNodePrint((char*)"syncLogStoreTest", pSyncNode);
//initRaftId(pSyncNode);
logStoreTest();
return 0;
}

85
source/libs/sync/test/syncPingTest.cpp
View File

@ -1,8 +1,10 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
@ -13,59 +15,65 @@ void logTest() {
sFatal("--- sync log test: fatal");
}
uint16_t ports[3] = {7010, 7110, 7210};
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SSyncNode* doSync(int myIndex) {
SSyncFSM* pFsm;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SSyncInfo syncInfo;
syncInfo.vgId = 1;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = 3;
pCfg->replicaNum = replicaNum;
pCfg->nodeInfo[0].nodePort = ports[0];
snprintf(pCfg->nodeInfo[0].nodeFqdn, sizeof(pCfg->nodeInfo[0].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
pCfg->nodeInfo[1].nodePort = ports[1];
snprintf(pCfg->nodeInfo[1].nodeFqdn, sizeof(pCfg->nodeInfo[1].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
pCfg->nodeInfo[2].nodePort = ports[2];
snprintf(pCfg->nodeInfo[2].nodeFqdn, sizeof(pCfg->nodeInfo[2].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
void timerPingAll(void* param, void* tmrId) {
SSyncNode* pSyncNode = (SSyncNode*)param;
syncNodePingAll(pSyncNode);
SSyncNode* syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char*)"syncPingTest.log", 100000, 10);
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
logTest();
int myIndex = 0;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
@ -76,30 +84,45 @@ int main(int argc, char** argv) {
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = doSync(myIndex);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout;
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
syncNodePrint((char*)"----1", pSyncNode);
initRaftId(pSyncNode);
//---------------------------
sTrace("syncNodeStartPingTimer ...");
ret = syncNodeStartPingTimer(pSyncNode);
assert(ret == 0);
syncNodePrint((char*)"----2", pSyncNode);
sTrace("sleep ...");
taosMsleep(10000);
sTrace("syncNodeStopPingTimer ...");
ret = syncNodeStopPingTimer(pSyncNode);
assert(ret == 0);
syncNodePrint((char*)"----3", pSyncNode);
taosMsleep(10000);
sTrace("sleep ...");
taosMsleep(5000);
sTrace("syncNodeStartPingTimer ...");
ret = syncNodeStartPingTimer(pSyncNode);
assert(ret == 0);
syncNodePrint((char*)"----4", pSyncNode);
sTrace("sleep ...");
taosMsleep(10000);
sTrace("syncNodeStopPingTimer ...");
ret = syncNodeStopPingTimer(pSyncNode);
assert(ret == 0);
syncNodePrint((char*)"----5", pSyncNode);
while (1) {
sTrace("while 1 sleep ...");
taosMsleep(1000);
}

3
source/libs/sync/test/syncVotesGrantedTest.cpp
View File

@ -32,8 +32,7 @@ SSyncNode* syncNodeInit() {
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;

3
source/libs/sync/test/syncVotesRespondTest.cpp
View File

@ -32,8 +32,7 @@ SSyncNode* syncNodeInit() {
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;

4
source/libs/transport/test/pushClient.c
View File

@ -107,7 +107,7 @@ static void *sendRequest(void *param) {
tDebug("recv response succefully");
// usleep(100000000);
// taosSsleep(100);
}
tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000);
@ -223,7 +223,7 @@ int main(int argc, char *argv[]) {
}
do {
usleep(1);
taosUsleep(1);
} while (tcount < appThreads);
gettimeofday(&systemTime, NULL);

4
source/libs/transport/test/pushServer.c
View File

@ -77,7 +77,7 @@ void processShellMsg() {
taosFreeQitem(pRpcMsg);
{
// sleep(1);
// taosSsleep(1);
SRpcMsg nRpcMsg = {0};
nRpcMsg.pCont = rpcMallocCont(msgSize);
nRpcMsg.contLen = msgSize;
@ -176,7 +176,7 @@ int main(int argc, char *argv[]) {
tError("failed to start RPC server");
return -1;
}
// sleep(5);
// taosSsleep(5);
tInfo("RPC server is running, ctrl-c to exit");

4
source/libs/transport/test/rclient.c
View File

@ -84,7 +84,7 @@ static void *sendRequest(void *param) {
tDebug("recv response succefully");
// usleep(100000000);
// taosSsleep(100);
}
tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000);
@ -200,7 +200,7 @@ int main(int argc, char *argv[]) {
}
do {
usleep(1);
taosUsleep(1);
} while (tcount < appThreads);
gettimeofday(&systemTime, NULL);

2
source/libs/transport/test/rsclient.c
View File

@ -178,7 +178,7 @@ int main(int argc, char *argv[]) {
}
do {
usleep(1);
taosUsleep(1);
} while ( tcount < appThreads);
gettimeofday(&systemTime, NULL);

4
source/libs/transport/test/syncClient.c
View File

@ -85,7 +85,7 @@ static void *sendRequest(void *param) {
tDebug("recv response succefully");
// usleep(100000000);
// taosSsleep(100);
}
tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000);
@ -201,7 +201,7 @@ int main(int argc, char *argv[]) {
}
do {
usleep(1);
taosUsleep(1);
} while (tcount < appThreads);
gettimeofday(&systemTime, NULL);

9
source/libs/wal/test/walMetaTest.cpp
View File

@ -124,8 +124,13 @@ class WalRetentionEnv : public ::testing::Test {
void SetUp() override {
SWalCfg cfg;
cfg.rollPeriod = -1, cfg.segSize = -1, cfg.retentionPeriod = -1, cfg.retentionSize = 0, cfg.rollPeriod = 0,
cfg.vgId = 0, cfg.level = TAOS_WAL_FSYNC;
cfg.rollPeriod = -1;
cfg.segSize = -1;
cfg.retentionPeriod = -1;
cfg.retentionSize = 0;
cfg.rollPeriod = 0;
cfg.vgId = 0;
cfg.level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, &cfg);
ASSERT(pWal != NULL);
}

54
source/os/src/osSleep.c
View File

@ -13,47 +13,35 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
void taosMsleep(int32_t ms) { Sleep(ms); }
#else
#if !(defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32))
#include <unistd.h>
#endif
/*
to make taosMsleep work,
signal SIGALRM shall be blocked in the calling thread,
sigset_t set;
sigemptyset(&set);
sigaddset(&set, SIGALRM);
pthread_sigmask(SIG_BLOCK, &set, NULL);
*/
void taosMsleep(int32_t mseconds) {
#if 1
usleep(mseconds * 1000);
void taosSsleep(int32_t s) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
Sleep(1000 * s);
#else
struct timeval timeout;
int32_t seconds, useconds;
seconds = mseconds / 1000;
useconds = (mseconds % 1000) * 1000;
timeout.tv_sec = seconds;
timeout.tv_usec = useconds;
/* sigset_t set; */
/* sigemptyset(&set); */
/* sigaddset(&set, SIGALRM); */
/* pthread_sigmask(SIG_BLOCK, &set, NULL); */
select(0, NULL, NULL, NULL, &timeout);
/* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
sleep(s);
#endif
}
void taosMsleep(int32_t ms) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
Sleep(ms);
#else
usleep(ms * 1000);
#endif
}
void taosUsleep(int32_t us) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
nanosleep(1000 * us);
#else
usleep(us);
#endif
}

7
source/os/src/osString.c
View File

@ -291,11 +291,10 @@ int32_t twcslen(const wchar_t *wcs) {
return n;
}
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes) {
for (int32_t i = 0; i < bytes; ++i) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4);
int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4);
for (int32_t i = 0; i < bytes; i += TSDB_NCHAR_SIZE) {
int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i);
int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i);
if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
return f1 - f2;

946
source/os/src/osSysinfo.c
View File

File diff suppressed because it is too large Load Diff

279
source/os/src/osSystem.c
View File

@ -13,20 +13,126 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#define _DEFAULT_SOURCE
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#elif defined(_TD_DARWIN_64)
#else
#include <dlfcn.h>
#include <termios.h>
#include <unistd.h>
#endif
/*
* windows implementation
*/
struct termios oldtio;
void* taosLoadDll(const char* filename) { return NULL; }
void* taosLoadSym(void* handle, char* name) { return NULL; }
void taosCloseDll(void* handle) {}
int32_t taosSystem(const char *cmd, char *buf, int32_t bufSize) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
FILE *fp;
if (cmd == NULL) {
// printf("taosSystem cmd is NULL!");
return -1;
}
if ((fp = _popen(cmd, "r")) == NULL) {
// printf("popen cmd:%s error: %s", cmd, strerror(errno));
return -1;
} else {
while (fgets(buf, bufSize, fp)) {
// printf("popen result:%s", buf);
}
if (!_pclose(fp)) {
// printf("close popen file pointer fp error!");
return -1;
} else {
// printf("popen res is :%d", res);
}
return 0;
#elif defined(_TD_DARWIN_64)
printf("no support funtion");
return -1;
#else
FILE *fp;
int32_t res;
if (cmd == NULL) {
// printf("taosSystem cmd is NULL!");
return -1;
}
if ((fp = popen(cmd, "r")) == NULL) {
// printf("popen cmd:%s error: %s", cmd, strerror(errno));
return -1;
} else {
while (fgets(buf, bufSize, fp)) {
// printf("popen result:%s", buf);
}
if ((res = pclose(fp)) == -1) {
// printf("close popen file pointer fp error!");
} else {
// printf("popen res is :%d", res);
}
return res;
}
#endif
}
void* taosLoadDll(const char* filename) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return NULL;
#elif defined(_TD_DARWIN_64)
return NULL;
#else
void* handle = dlopen(filename, RTLD_LAZY);
if (!handle) {
//printf("load dll:%s failed, error:%s", filename, dlerror());
return NULL;
}
//printf("dll %s loaded", filename);
return handle;
#endif
}
void* taosLoadSym(void* handle, char* name) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return NULL;
#elif defined(_TD_DARWIN_64)
return NULL;
#else
void* sym = dlsym(handle, name);
char* error = NULL;
if ((error = dlerror()) != NULL) {
//printf("load sym:%s failed, error:%s", name, dlerror());
return NULL;
}
//printf("sym %s loaded", name);
return sym;
#endif
}
void taosCloseDll(void* handle) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
return;
#elif defined(_TD_DARWIN_64)
return;
#else
if (handle) {
dlclose(handle);
}
#endif
}
int taosSetConsoleEcho(bool on) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE);
DWORD mode = 0;
GetConsoleMode(hStdin, &mode);
@ -38,19 +144,7 @@ int taosSetConsoleEcho(bool on) {
SetConsoleMode(hStdin, mode);
return 0;
}
#elif defined(_TD_DARWIN_64)
/*
* darwin implementation
*/
void* taosLoadDll(const char* filename) { return NULL; }
void* taosLoadSym(void* handle, char* name) { return NULL; }
void taosCloseDll(void* handle) {}
int taosSetConsoleEcho(bool on) {
#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL)
int err;
struct termios term;
@ -72,51 +166,7 @@ int taosSetConsoleEcho(bool on) {
}
return 0;
}
#else
/*
* linux implementation
*/
#include <dlfcn.h>
#include <termios.h>
#include <unistd.h>
void* taosLoadDll(const char* filename) {
void* handle = dlopen(filename, RTLD_LAZY);
if (!handle) {
//printf("load dll:%s failed, error:%s", filename, dlerror());
return NULL;
}
//printf("dll %s loaded", filename);
return handle;
}
void* taosLoadSym(void* handle, char* name) {
void* sym = dlsym(handle, name);
char* error = NULL;
if ((error = dlerror()) != NULL) {
//printf("load sym:%s failed, error:%s", name, dlerror());
return NULL;
}
//printf("sym %s loaded", name);
return sym;
}
void taosCloseDll(void* handle) {
if (handle) {
dlclose(handle);
}
}
int taosSetConsoleEcho(bool on) {
#define ECHOFLAGS (ECHO | ECHOE | ECHOK | ECHONL)
int err;
struct termios term;
@ -138,6 +188,111 @@ int taosSetConsoleEcho(bool on) {
}
return 0;
#endif
}
void setTerminalMode() {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#elif defined(_TD_DARWIN_64)
struct termios newtio;
/* if (atexit() != 0) { */
/* fprintf(stderr, "Error register exit function!\n"); */
/* exit(EXIT_FAILURE); */
/* } */
memcpy(&newtio, &oldtio, sizeof(oldtio));
// Set new terminal attributes.
newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP);
newtio.c_iflag |= IGNBRK;
// newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET);
newtio.c_oflag |= OPOST;
newtio.c_oflag |= ONLCR;
newtio.c_oflag &= ~(OCRNL | ONLRET);
newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG);
newtio.c_cc[VMIN] = 1;
newtio.c_cc[VTIME] = 0;
if (tcsetattr(0, TCSANOW, &newtio) != 0) {
fprintf(stderr, "Fail to set terminal properties!\n");
exit(EXIT_FAILURE);
}
#else
struct termios newtio;
/* if (atexit() != 0) { */
/* fprintf(stderr, "Error register exit function!\n"); */
/* exit(EXIT_FAILURE); */
/* } */
memcpy(&newtio, &oldtio, sizeof(oldtio));
// Set new terminal attributes.
newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP);
newtio.c_iflag |= IGNBRK;
// newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET);
newtio.c_oflag |= OPOST;
newtio.c_oflag |= ONLCR;
newtio.c_oflag &= ~(OCRNL | ONLRET);
newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG);
newtio.c_cc[VMIN] = 1;
newtio.c_cc[VTIME] = 0;
if (tcsetattr(0, TCSANOW, &newtio) != 0) {
fprintf(stderr, "Fail to set terminal properties!\n");
exit(EXIT_FAILURE);
}
#endif
}
int32_t getOldTerminalMode() {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#elif defined(_TD_DARWIN_64)
/* Make sure stdin is a terminal. */
if (!isatty(STDIN_FILENO)) {
return -1;
}
// Get the parameter of current terminal
if (tcgetattr(0, &oldtio) != 0) {
return -1;
}
return 1;
#else
/* Make sure stdin is a terminal. */
if (!isatty(STDIN_FILENO)) {
return -1;
}
// Get the parameter of current terminal
if (tcgetattr(0, &oldtio) != 0) {
return -1;
}
return 1;
#endif
}
void resetTerminalMode() {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#elif defined(_TD_DARWIN_64)
if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
fprintf(stderr, "Fail to reset the terminal properties!\n");
exit(EXIT_FAILURE);
}
#else
if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
fprintf(stderr, "Fail to reset the terminal properties!\n");
exit(EXIT_FAILURE);
}
#endif
}

4
source/util/test/cacheTest.cpp
View File

@ -14,7 +14,7 @@ TEST(cacheTest, client_cache_test) {
char data1[] = "test11";
char* cachedObj = (char*) taosCachePut(tscMetaCache, key1, strlen(key1), data1, strlen(data1)+1, 1);
sleep(REFRESH_TIME_IN_SEC+1);
taosSsleep(REFRESH_TIME_IN_SEC+1);
printf("obj is still valid: %s\n", cachedObj);
@ -37,7 +37,7 @@ TEST(cacheTest, client_cache_test) {
taosCacheRelease(tscMetaCache, (void**) &cachedObj2, false);
sleep(3);
taosSsleep(3);
char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3));
assert(d == NULL);

10
source/util/test/encodeTest.cpp
View File

@ -174,8 +174,8 @@ TEST(td_encode_test, encode_decode_variant_len_integer) {
}
TEST(td_encode_test, encode_decode_cstr) {
uint8_t * buf = new uint8_t[1024 * 1024];
char * cstr = new char[1024 * 1024];
uint8_t *buf = new uint8_t[1024 * 1024];
char *cstr = new char[1024 * 1024];
const char *dcstr;
SCoder encoder;
SCoder decoder;
@ -208,7 +208,7 @@ TEST(td_encode_test, encode_decode_cstr) {
typedef struct {
int32_t A_a;
int64_t A_b;
char * A_c;
char *A_c;
} SStructA_v1;
static int32_t tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) {
@ -240,7 +240,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) {
typedef struct {
int32_t A_a;
int64_t A_b;
char * A_c;
char *A_c;
// -------------------BELOW FEILDS ARE ADDED IN A NEW VERSION--------------
int16_t A_d;
int16_t A_e;
@ -437,4 +437,4 @@ TEST(td_encode_test, compound_struct_encode_test) {
tCoderClear(&decoder);
}
#endif
#pragma GCC diagnostic pop
#pragma GCC diagnostic pop

6
source/util/test/trefTest.c
View File

@ -42,7 +42,7 @@ void *addRef(void *param) {
pSpace->p[id] = malloc(128);
pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
}
usleep(100);
taosUsleep(100);
}
return NULL;
@ -60,7 +60,7 @@ void *removeRef(void *param) {
if (code == 0) pSpace->rid[id] = 0;
}
usleep(100);
taosUsleep(100);
}
return NULL;
@ -76,7 +76,7 @@ void *acquireRelease(void *param) {
id = random() % pSpace->refNum;
void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]);
if (p) {
usleep(id % 5 + 1);
taosUsleep(id % 5 + 1);
taosReleaseRef(pSpace->rsetId, (int64_t) pSpace->p[id]);
}
}

2
tests/script/api/batchprepare.c
View File

@ -5089,7 +5089,7 @@ int main(int argc, char *argv[])
//pthread_create(&(pThreadList[3]), &thattr, runcase, (void *)&par[3]);
while(1) {
sleep(1);
taosSsleep(1);
}
return 0;
}

11
tools/CMakeLists.txt
View File

@ -1 +1,10 @@
add_subdirectory(shell)
IF (TD_TAOS_TOOLS)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/tools/taos_tools/deps/avro/lang/c/src)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/client)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/common)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/util)
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include/os)
ADD_SUBDIRECTORY(taos-tools)
ENDIF ()
add_subdirectory(shell)

4
tools/shell/inc/shell.h
View File

@ -86,10 +86,6 @@ extern char PROMPT_HEADER[];
extern char CONTINUE_PROMPT[];
extern int prompt_size;
extern SShellHistory history;
extern struct termios oldtio;
extern void set_terminal_mode();
extern int get_old_terminal_mode(struct termios* tio);
extern void reset_terminal_mode();
extern SShellArguments args;
extern int64_t result;

60
tools/shell/src/backup/shellDarwin.c
View File

@ -358,7 +358,7 @@ int32_t shellReadCommand(TAOS *con, char *command) {
void *shellLoopQuery(void *arg) {
if (indicator) {
get_old_terminal_mode(&oldtio);
getOldTerminalMode();
indicator = 0;
}
@ -379,12 +379,12 @@ void *shellLoopQuery(void *arg) {
do {
// Read command from shell.
memset(command, 0, MAX_COMMAND_SIZE);
set_terminal_mode();
setTerminalMode();
err = shellReadCommand(con, command);
if (err) {
break;
}
reset_terminal_mode();
resetTerminalMode();
} while (shellRunCommand(con, command) == 0);
tfree(command);
@ -395,56 +395,6 @@ void *shellLoopQuery(void *arg) {
return NULL;
}
int get_old_terminal_mode(struct termios *tio) {
/* Make sure stdin is a terminal. */
if (!isatty(STDIN_FILENO)) {
return -1;
}
// Get the parameter of current terminal
if (tcgetattr(0, &oldtio) != 0) {
return -1;
}
return 1;
}
void reset_terminal_mode() {
if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
fprintf(stderr, "Fail to reset the terminal properties!\n");
exit(EXIT_FAILURE);
}
}
void set_terminal_mode() {
struct termios newtio;
/* if (atexit(reset_terminal_mode) != 0) { */
/* fprintf(stderr, "Error register exit function!\n"); */
/* exit(EXIT_FAILURE); */
/* } */
memcpy(&newtio, &oldtio, sizeof(oldtio));
// Set new terminal attributes.
newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP);
newtio.c_iflag |= IGNBRK;
// newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET);
newtio.c_oflag |= OPOST;
newtio.c_oflag |= ONLCR;
newtio.c_oflag &= ~(OCRNL | ONLRET);
newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG);
newtio.c_cc[VMIN] = 1;
newtio.c_cc[VTIME] = 0;
if (tcsetattr(0, TCSANOW, &newtio) != 0) {
fprintf(stderr, "Fail to set terminal properties!\n");
exit(EXIT_FAILURE);
}
}
void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); }
void clearScreen(int ecmd_pos, int cursor_pos) {
@ -541,9 +491,9 @@ void showOnScreen(Command *cmd) {
fflush(stdout);
}
void cleanup_handler(void *arg) { tcsetattr(0, TCSANOW, &oldtio); }
void cleanup_handler(void *arg) { resetTerminalMode(); }
void exitShell() {
tcsetattr(0, TCSANOW, &oldtio);
resetTerminalMode();
exit(EXIT_SUCCESS);
}

15
tools/shell/src/backup/shellImport.c
View File

@ -39,14 +39,14 @@ static int shellGetFilesNum(const char *directoryName, const char *prefix)
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
FILE *fp = popen(cmd, "r");
if (fp == NULL) {
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
if (fscanf(fp, "%d", &fileNum) != 1) {
if (sscanf(buf, "%d", &fileNum) != 1) {
fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
exit(0);
}
@ -56,7 +56,6 @@ static int shellGetFilesNum(const char *directoryName, const char *prefix)
exit(0);
}
pclose(fp);
return fileNum;
}
@ -65,14 +64,14 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c
char cmd[1024] = { 0 };
sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
FILE *fp = popen(cmd, "r");
if (fp == NULL) {
char buf[1024] = { 0 };
if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
exit(0);
}
int fileNum = 0;
while (fscanf(fp, "%128s", fileArray[fileNum++])) {
while (sscanf(buf, "%128s", fileArray[fileNum++])) {
if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
fileNum--;
}
@ -85,8 +84,6 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c
fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
exit(0);
}
pclose(fp);
}
static void shellCheckTablesSQLFile(const char *directoryName)

60
tools/shell/src/shellLinux.c
View File

@ -388,7 +388,7 @@ int32_t shellReadCommand(TAOS *con, char *command) {
void *shellLoopQuery(void *arg) {
if (indicator) {
get_old_terminal_mode(&oldtio);
getOldTerminalMode();
indicator = 0;
}
@ -409,12 +409,12 @@ void *shellLoopQuery(void *arg) {
do {
// Read command from shell.
memset(command, 0, MAX_COMMAND_SIZE);
set_terminal_mode();
setTerminalMode();
err = shellReadCommand(con, command);
if (err) {
break;
}
reset_terminal_mode();
resetTerminalMode();
} while (shellRunCommand(con, command) == 0);
tfree(command);
@ -425,56 +425,6 @@ void *shellLoopQuery(void *arg) {
return NULL;
}
int get_old_terminal_mode(struct termios *tio) {
/* Make sure stdin is a terminal. */
if (!isatty(STDIN_FILENO)) {
return -1;
}
// Get the parameter of current terminal
if (tcgetattr(0, &oldtio) != 0) {
return -1;
}
return 1;
}
void reset_terminal_mode() {
if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
fprintf(stderr, "Fail to reset the terminal properties!\n");
exit(EXIT_FAILURE);
}
}
void set_terminal_mode() {
struct termios newtio;
/* if (atexit(reset_terminal_mode) != 0) { */
/* fprintf(stderr, "Error register exit function!\n"); */
/* exit(EXIT_FAILURE); */
/* } */
memcpy(&newtio, &oldtio, sizeof(oldtio));
// Set new terminal attributes.
newtio.c_iflag &= ~(IXON | IXOFF | ICRNL | INLCR | IGNCR | IMAXBEL | ISTRIP);
newtio.c_iflag |= IGNBRK;
// newtio.c_oflag &= ~(OPOST|ONLCR|OCRNL|ONLRET);
newtio.c_oflag |= OPOST;
newtio.c_oflag |= ONLCR;
newtio.c_oflag &= ~(OCRNL | ONLRET);
newtio.c_lflag &= ~(IEXTEN | ICANON | ECHO | ECHOE | ECHONL | ECHOCTL | ECHOPRT | ECHOKE | ISIG);
newtio.c_cc[VMIN] = 1;
newtio.c_cc[VTIME] = 0;
if (tcsetattr(0, TCSANOW, &newtio) != 0) {
fprintf(stderr, "Fail to set terminal properties!\n");
exit(EXIT_FAILURE);
}
}
void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); }
void clearScreen(int ecmd_pos, int cursor_pos) {
@ -571,10 +521,10 @@ void showOnScreen(Command *cmd) {
fflush(stdout);
}
void cleanup_handler(void *arg) { tcsetattr(0, TCSANOW, &oldtio); }
void cleanup_handler(void *arg) { resetTerminalMode(); }
void exitShell() {
/*int32_t ret =*/ tcsetattr(STDIN_FILENO, TCSANOW, &oldtio);
/*int32_t ret =*/ resetTerminalMode();
taos_cleanup();
exit(EXIT_SUCCESS);
}

4
tools/shell/src/shellMain.c
View File

@ -41,11 +41,11 @@ void *cancelHandler(void *arg) {
taosReleaseRef(tscObjRef, rid);
#endif
#else
reset_terminal_mode();
resetTerminalMode();
printf("\nReceive ctrl+c or other signal, quit shell.\n");
exit(0);
#endif
reset_terminal_mode();
resetTerminalMode();
printf("\nReceive ctrl+c or other signal, quit shell.\n");
exit(0);
}

1
tools/taos-tools Submodule

@ -0,0 +1 @@
Subproject commit f36b07f710d661dca88fdd70e73b5e3e16a960e0