xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-04-01 10:30:20 +08:00
parent b449369265 e630b623b5
commit 36dc9da4a5
53 changed files with 1422 additions and 825 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clang-format
View File

@ -5,6 +5,7 @@ AccessModifierOffset: -1
AlignAfterOpenBracket: Align AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: true AlignConsecutiveDeclarations: true
AlignConsecutiveMacros: true
AlignEscapedNewlinesLeft: true AlignEscapedNewlinesLeft: true
AlignOperands: true AlignOperands: true
AlignTrailingComments: true AlignTrailingComments: true

52
example/src/tmq.c
View File

@ -28,7 +28,7 @@ int32_t init_env() {
return -1; return -1;
} }
TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes)); printf("error in create db, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
@ -42,25 +42,33 @@ int32_t init_env() {
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); pRes =
taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)"); pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)"); pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
} }
pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)");
if (taos_errno(pRes) != 0) {
printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes); taos_free_result(pRes);
return 0; return 0;
} }
@ -82,12 +90,40 @@ int32_t create_topic() {
/*const char* sql = "select * from tu1";*/ /*const char* sql = "select * from tu1";*/
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1"); pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1 from ct1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
#if 0
pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')");
if (taos_errno(pRes) != 0) {
printf("failed to insert, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')");
if (taos_errno(pRes) != 0) {
printf("failed to insert, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')");
if (taos_errno(pRes) != 0) {
printf("failed to insert, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')");
if (taos_errno(pRes) != 0) {
printf("failed to insert, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
#endif
taos_close(pConn); taos_close(pConn);
return 0; return 0;
} }
@ -115,7 +151,7 @@ tmq_t* build_consumer() {
tmq_list_t* build_topic_list() { tmq_list_t* build_topic_list() {
tmq_list_t* topic_list = tmq_list_new(); tmq_list_t* topic_list = tmq_list_new();
tmq_list_append(topic_list, "test_stb_topic_1"); tmq_list_append(topic_list, "topic_ctb_column");
return topic_list; return topic_list;
} }
@ -215,8 +251,8 @@ int main(int argc, char* argv[]) {
if (argc > 1) { if (argc > 1) {
printf("env init\n"); printf("env init\n");
code = init_env(); code = init_env();
}
create_topic(); create_topic();
}
tmq_t* tmq = build_consumer(); tmq_t* tmq = build_consumer();
tmq_list_t* topic_list = build_topic_list(); tmq_list_t* topic_list = build_topic_list();
/*perf_loop(tmq, topic_list);*/ /*perf_loop(tmq, topic_list);*/

4
example/src/tstream.c
View File

@ -20,7 +20,7 @@
#include "taos.h" #include "taos.h"
int32_t init_env() { int32_t init_env() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) { if (pConn == NULL) {
return -1; return -1;
} }
@ -65,7 +65,7 @@ int32_t init_env() {
int32_t create_stream() { int32_t create_stream() {
printf("create stream\n"); printf("create stream\n");
TAOS_RES* pRes; TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) { if (pConn == NULL) {
return -1; return -1;
} }

13
include/common/tcommon.h
View File

@ -37,6 +37,14 @@ enum {
TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__EP_RSP,
}; };
enum {
STREAM_TRIGGER__AT_ONCE = 1,
STREAM_TRIGGER__WINDOW_CLOSE,
STREAM_TRIGGER__BY_COUNT,
STREAM_TRIGGER__BY_BATCH_COUNT,
STREAM_TRIGGER__BY_EVENT_TIME,
};
typedef struct { typedef struct {
uint32_t numOfTables; uint32_t numOfTables;
SArray* pGroupList; SArray* pGroupList;
@ -59,7 +67,10 @@ typedef struct SDataBlockInfo {
int32_t rowSize; int32_t rowSize;
int16_t numOfCols; int16_t numOfCols;
int16_t hasVarCol; int16_t hasVarCol;
union {int64_t uid; int64_t blockId;}; union {
int64_t uid;
int64_t blockId;
};
int64_t groupId; // no need to serialize int64_t groupId; // no need to serialize
} SDataBlockInfo; } SDataBlockInfo;

2
include/libs/wal/wal.h
View File

@ -157,6 +157,7 @@ typedef struct SWalReadHandle {
int64_t curVersion; int64_t curVersion;
int64_t capacity; int64_t capacity;
int64_t status; // if cursor valid int64_t status; // if cursor valid
TdThreadMutex mutex;
SWalHead *pHead; SWalHead *pHead;
} SWalReadHandle; } SWalReadHandle;
#pragma pack(pop) #pragma pack(pop)
@ -191,6 +192,7 @@ int32_t walEndSnapshot(SWal *);
SWalReadHandle *walOpenReadHandle(SWal *); SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *); void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
// deprecated // deprecated
#if 0 #if 0

5
source/client/src/clientImpl.c
View File

@ -181,6 +181,11 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
} }
int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
// drop table if exists not_exists_table
if (NULL == pQuery->pCmdMsg) {
return TSDB_CODE_SUCCESS;
}
SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg;
pRequest->type = pMsgInfo->msgType; pRequest->type = pMsgInfo->msgType;
pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL};

15
source/client/src/tmq.c
View File

@ -255,6 +255,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
break; break;
} }
msg = NULL;
taosReadAllQitems(tmq->mqueue, tmq->qall); taosReadAllQitems(tmq->mqueue, tmq->qall);
while (1) { while (1) {
taosGetQitem(tmq->qall, (void**)&msg); taosGetQitem(tmq->qall, (void**)&msg);
@ -787,7 +788,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
static bool noPrintSchema; static bool noPrintSchema;
char pBuf[128]; char pBuf[128];
SMqPollRsp* pRsp = &tmq_message->msg; SMqPollRsp* pRsp = &tmq_message->msg;
int32_t colNum = pRsp->schema->nCols; int32_t colNum = 2;
if (!noPrintSchema) { if (!noPrintSchema) {
printf("|"); printf("|");
for (int32_t i = 0; i < colNum; i++) { for (int32_t i = 0; i < colNum; i++) {
@ -838,6 +839,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
int32_t tmqEpoch = atomic_load_32(&tmq->epoch); int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) { if (msgEpoch < tmqEpoch) {
/*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
tsem_post(&tmq->rspSem); tsem_post(&tmq->rspSem);
tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
return 0; return 0;
@ -886,6 +888,9 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto WRITE_QUEUE_FAIL; goto WRITE_QUEUE_FAIL;
} }
tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
pRsp->msg.rspOffset);
pRsp->vg = pParam->pVg; pRsp->vg = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp); taosWriteQitem(tmq->mqueue, pRsp);
atomic_add_fetch_32(&tmq->readyRequest, 1); atomic_add_fetch_32(&tmq->readyRequest, 1);
@ -902,6 +907,7 @@ WRITE_QUEUE_FAIL:
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
/*printf("call update ep %d\n", epoch);*/ /*printf("call update ep %d\n", epoch);*/
/*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/
bool set = false; bool set = false;
int32_t topicNumGet = taosArrayGetSize(pRsp->topics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
@ -932,6 +938,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
for (int32_t k = 0; k < vgNumCur; k++) { for (int32_t k = 0; k < vgNumCur; k++) {
SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
/*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/
taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
} }
break; break;
@ -945,9 +952,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
int64_t offset = pVgEp->offset; int64_t offset = pVgEp->offset;
/*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/
if (pOffset != NULL) { if (pOffset != NULL) {
offset = *pOffset; offset = *pOffset;
/*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/
} }
/*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/
SMqClientVg clientVg = { SMqClientVg clientVg = {
.pollCnt = 0, .pollCnt = 0,
.currentOffset = offset, .currentOffset = offset,
@ -1195,6 +1205,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
if (vgStatus != TMQ_VG_STATUS__IDLE) { if (vgStatus != TMQ_VG_STATUS__IDLE) {
/*printf("skip vg %d\n", pVg->vgId);*/
continue; continue;
} }
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
@ -1238,6 +1249,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
int64_t transporterId = 0; int64_t transporterId = 0;
/*printf("send poll\n");*/ /*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1); atomic_add_fetch_32(&tmq->waitingRequest, 1);
/*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++; pVg->pollCnt++;
tmq->pollCnt++; tmq->pollCnt++;

2
source/dnode/mgmt/dm/src/dmMsg.c
View File

@ -53,7 +53,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
tSerializeSStatusReq(pHead, contLen, &req); tSerializeSStatusReq(pHead, contLen, &req);
taosArrayDestroy(req.pVloads); taosArrayDestroy(req.pVloads);
SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527};
pMgmt->statusSent = 1; pMgmt->statusSent = 1;
dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);

23
source/dnode/mgmt/main/exe/dndMain.c
View File

@ -30,18 +30,13 @@ static struct {
} global = {0}; } global = {0};
static void dndStopDnode(int signum, void *info, void *ctx) { static void dndStopDnode(int signum, void *info, void *ctx) {
dInfo("signal:%d is received", signum); dInfo("system signal:%d received", signum);
SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
if (pDnode != NULL) { if (pDnode != NULL) {
dndHandleEvent(pDnode, DND_EVENT_STOP); dndHandleEvent(pDnode, DND_EVENT_STOP);
} }
} }
static void dndHandleChild(int signum, void *info, void *ctx) {
dInfo("signal:%d is received", signum);
dndHandleEvent(global.pDnode, DND_EVENT_CHILD);
}
static void dndSetSignalHandle() { static void dndSetSignalHandle() {
taosSetSignal(SIGTERM, dndStopDnode); taosSetSignal(SIGTERM, dndStopDnode);
taosSetSignal(SIGHUP, dndStopDnode); taosSetSignal(SIGHUP, dndStopDnode);
@ -50,8 +45,8 @@ static void dndSetSignalHandle() {
taosSetSignal(SIGBREAK, dndStopDnode); taosSetSignal(SIGBREAK, dndStopDnode);
if (!tsMultiProcess) { if (!tsMultiProcess) {
} else if (global.ntype == DNODE) { } else if (global.ntype == DNODE || global.ntype == NODE_MAX) {
taosSetSignal(SIGCHLD, dndHandleChild); taosIgnSignal(SIGCHLD);
} else { } else {
taosKillChildOnParentStopped(); taosKillChildOnParentStopped();
} }
@ -74,14 +69,14 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
tstrncpy(global.apolloUrl, argv[++i], PATH_MAX); tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-e") == 0) { } else if (strcmp(argv[i], "-e") == 0) {
tstrncpy(global.envFile, argv[++i], PATH_MAX); tstrncpy(global.envFile, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-n") == 0) { } else if (strcmp(argv[i], "-n") == 0) {
global.ntype = atoi(argv[++i]); global.ntype = atoi(argv[++i]);
if (global.ntype <= DNODE || global.ntype > NODE_MAX) { if (global.ntype <= DNODE || global.ntype > NODE_MAX) {
printf("'-n' range is [1-5], default is 0\n"); printf("'-n' range is [1-5], default is 0\n");
return -1; return -1;
} }
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-C") == 0) { } else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true; global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) { } else if (strcmp(argv[i], "-V") == 0) {
@ -139,7 +134,7 @@ static int32_t dndInitLog() {
static void dndSetProcInfo(int32_t argc, char **argv) { static void dndSetProcInfo(int32_t argc, char **argv) {
taosSetProcPath(argc, argv); taosSetProcPath(argc, argv);
if (global.ntype != DNODE) { if (global.ntype != DNODE && global.ntype != NODE_MAX) {
const char *name = dndNodeProcStr(global.ntype); const char *name = dndNodeProcStr(global.ntype);
taosSetProcName(argc, argv, name); taosSetProcName(argc, argv, name);
} }
@ -147,14 +142,14 @@ static void dndSetProcInfo(int32_t argc, char **argv) {
static int32_t dndRunDnode() { static int32_t dndRunDnode() {
if (dndInit() != 0) { if (dndInit() != 0) {
dError("failed to initialize environment since %s", terrstr()); dError("failed to init environment since %s", terrstr());
return -1; return -1;
} }
SDnodeOpt option = dndGetOpt(); SDnodeOpt option = dndGetOpt();
SDnode *pDnode = dndCreate(&option); SDnode *pDnode = dndCreate(&option);
if (pDnode == NULL) { if (pDnode == NULL) {
dError("failed to to create dnode object since %s", terrstr()); dError("failed to to create dnode since %s", terrstr());
return -1; return -1;
} else { } else {
global.pDnode = pDnode; global.pDnode = pDnode;
@ -184,7 +179,6 @@ int main(int argc, char const *argv[]) {
return -1; return -1;
} }
dndSetProcInfo(argc, (char **)argv);
if (global.generateGrant) { if (global.generateGrant) {
dndGenerateGrant(); dndGenerateGrant();
return 0; return 0;
@ -212,5 +206,6 @@ int main(int argc, char const *argv[]) {
return 0; return 0;
} }
dndSetProcInfo(argc, (char **)argv);
return dndRunDnode(); return dndRunDnode();
} }

1
source/dnode/mgmt/main/inc/dnd.h
View File

@ -136,6 +136,7 @@ typedef struct SDnode {
const char *dndNodeLogStr(ENodeType ntype); const char *dndNodeLogStr(ENodeType ntype);
const char *dndNodeProcStr(ENodeType ntype); const char *dndNodeProcStr(ENodeType ntype);
const char *dndEventStr(EDndEvent ev);
EDndStatus dndGetStatus(SDnode *pDnode); EDndStatus dndGetStatus(SDnode *pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus stat); void dndSetStatus(SDnode *pDnode, EDndStatus stat);
void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);

96
source/dnode/mgmt/main/src/dndEnv.c Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndInt.h"
#include "wal.h"
static int8_t once = DND_ENV_INIT;
int32_t dndInit() {
dDebug("start to init dnode env");
if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
terrno = TSDB_CODE_REPEAT_INIT;
dError("failed to init dnode env since %s", terrstr());
return -1;
}
taosIgnSIGPIPE();
taosBlockSIGPIPE();
taosResolveCRC();
SMonCfg monCfg = {0};
monCfg.maxLogs = tsMonitorMaxLogs;
monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) {
dError("failed to init monitor since %s", terrstr());
return -1;
}
dInfo("dnode env is initialized");
return 0;
}
void dndCleanup() {
dDebug("start to cleanup dnode env");
if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
dError("dnode env is already cleaned up");
return;
}
monCleanup();
walCleanUp();
taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up");
}
void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
}
EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
void dndSetStatus(SDnode *pDnode, EDndStatus status) {
if (pDnode->status != status) {
dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
pDnode->status = status;
}
}
void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
SStartupReq *pStartup = &pDnode->startup;
tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
pStartup->finished = 0;
}
void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
}
void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
dDebug("startup req is received");
SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
dndGetStartup(pDnode, pStartup);
dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
SRpcMsg rpcRsp = {
.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
rpcSendResponse(&rpcRsp);
}

19
source/dnode/mgmt/main/src/dndExec.c
View File

@ -128,7 +128,7 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
} }
pWrapper->procId = pid; pWrapper->procId = pid;
dInfo("node:%s, run in new process, pid:%d", pWrapper->name, pid); dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
return 0; return 0;
} }
@ -150,7 +150,7 @@ static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
} }
static int32_t dndRunInSingleProcess(SDnode *pDnode) { static int32_t dndRunInSingleProcess(SDnode *pDnode) {
dInfo("dnode start to run in single process"); dInfo("dnode run in single process");
for (ENodeType n = DNODE; n < NODE_MAX; ++n) { for (ENodeType n = DNODE; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
@ -189,7 +189,7 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
} }
static int32_t dndRunInParentProcess(SDnode *pDnode) { static int32_t dndRunInParentProcess(SDnode *pDnode) {
dInfo("dnode start to run in parent process"); dInfo("dnode run in parent process");
SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE]; SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE];
if (dndOpenNode(pDWrapper) != 0) { if (dndOpenNode(pDWrapper) != 0) {
dError("node:%s, failed to start since %s", pDWrapper->name, terrstr()); dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
@ -201,12 +201,13 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
pWrapper->required = dndRequireNode(pWrapper); pWrapper->required = dndRequireNode(pWrapper);
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
int64_t shmsize = 1024 * 1024 * 2; // size will be a configuration item int32_t shmsize = 1024 * 1024 * 2; // size will be a configuration item
if (taosCreateShm(&pWrapper->shm, shmsize) != 0) { if (taosCreateShm(&pWrapper->shm, shmsize) != 0) {
terrno = TAOS_SYSTEM_ERROR(terrno); terrno = TAOS_SYSTEM_ERROR(terrno);
dError("node:%s, failed to create shm size:%" PRId64 " since %s", pWrapper->name, shmsize, terrstr()); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
return -1; return -1;
} }
dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize);
SProcCfg cfg = dndGenProcCfg(pWrapper); SProcCfg cfg = dndGenProcCfg(pWrapper);
cfg.isChild = false; cfg.isChild = false;
@ -262,21 +263,21 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (pDnode->ntype == NODE_MAX) continue; if (pDnode->ntype == NODE_MAX) continue;
if (pWrapper->procId != 0 && !taosProcExists(pWrapper->procId)) { if (pWrapper->procId <= 0 || !taosProcExists(pWrapper->procId)) {
dInfo("node:%s, process not exist, pid:%d", pWrapper->name, pWrapper->procId); dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
dndNewProc(pWrapper, n); dndNewProc(pWrapper, n);
} }
}
taosMsleep(100); taosMsleep(100);
} }
}
return 0; return 0;
} }
static int32_t dndRunInChildProcess(SDnode *pDnode) { static int32_t dndRunInChildProcess(SDnode *pDnode) {
dInfo("dnode start to run in child process");
SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
dInfo("%s run in child process", pWrapper->name);
SMsgCb msgCb = dndCreateMsgcb(pWrapper); SMsgCb msgCb = dndCreateMsgcb(pWrapper);
tmsgSetDefaultMsgCb(&msgCb); tmsgSetDefaultMsgCb(&msgCb);

6
source/dnode/mgmt/main/src/dndFile.c
View File

@ -179,7 +179,7 @@ int32_t dndReadShmFile(SDnode *pDnode) {
} }
} }
if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == DNODE) { if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) { for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (pWrapper->shm.id >= 0) { if (pWrapper->shm.id >= 0) {
@ -194,10 +194,10 @@ int32_t dndReadShmFile(SDnode *pDnode) {
dError("shmid:%d, failed to attach shm since %s", pWrapper->shm.id, terrstr()); dError("shmid:%d, failed to attach shm since %s", pWrapper->shm.id, terrstr());
goto _OVER; goto _OVER;
} }
dDebug("shmid:%d, is attached, size:%d", pWrapper->shm.id, pWrapper->shm.size); dInfo("node:%s, shmid:%d is attached, size:%d", pWrapper->name, pWrapper->shm.id, pWrapper->shm.size);
} }
dDebug("successed to open %s", file); dDebug("successed to load %s", file);
code = 0; code = 0;
_OVER: _OVER:

208
source/dnode/mgmt/main/src/dndInt.c
View File

@ -15,82 +15,186 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dndInt.h" #include "dndInt.h"
#include "wal.h"
static int8_t once = DND_ENV_INIT; static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
pDnode->serverPort = pOption->serverPort;
pDnode->dataDir = strdup(pOption->dataDir);
pDnode->localEp = strdup(pOption->localEp);
pDnode->localFqdn = strdup(pOption->localFqdn);
pDnode->firstEp = strdup(pOption->firstEp);
pDnode->secondEp = strdup(pOption->secondEp);
pDnode->disks = pOption->disks;
pDnode->numOfDisks = pOption->numOfDisks;
pDnode->ntype = pOption->ntype;
pDnode->rebootTime = taosGetTimestampMs();
int32_t dndInit() { if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
dDebug("start to init dnode env"); pDnode->secondEp == NULL) {
if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_OUT_OF_MEMORY;
terrno = TSDB_CODE_REPEAT_INIT;
dError("failed to init dnode env since %s", terrstr());
return -1; return -1;
} }
taosIgnSIGPIPE(); if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
taosBlockSIGPIPE(); pDnode->lockfile = dndCheckRunning(pDnode->dataDir);
taosResolveCRC(); if (pDnode->lockfile == NULL) {
SMonCfg monCfg = {0};
monCfg.maxLogs = tsMonitorMaxLogs;
monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) {
dError("failed to init monitor since %s", terrstr());
return -1; return -1;
} }
}
dInfo("dnode env is initialized");
return 0; return 0;
} }
void dndCleanup() { static void dndClearVars(SDnode *pDnode) {
dDebug("start to cleanup dnode env"); for (ENodeType n = 0; n < NODE_MAX; ++n) {
if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
dError("dnode env is already cleaned up"); taosMemoryFreeClear(pMgmt->path);
}
if (pDnode->lockfile != NULL) {
taosUnLockFile(pDnode->lockfile);
taosCloseFile(&pDnode->lockfile);
pDnode->lockfile = NULL;
}
taosMemoryFreeClear(pDnode->localEp);
taosMemoryFreeClear(pDnode->localFqdn);
taosMemoryFreeClear(pDnode->firstEp);
taosMemoryFreeClear(pDnode->secondEp);
taosMemoryFreeClear(pDnode->dataDir);
taosMemoryFree(pDnode);
dDebug("dnode memory is cleared, data:%p", pDnode);
}
SDnode *dndCreate(const SDnodeOpt *pOption) {
dDebug("start to create dnode object");
int32_t code = -1;
char path[PATH_MAX] = {0};
SDnode *pDnode = NULL;
pDnode = taosMemoryCalloc(1, sizeof(SDnode));
if (pDnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (dndInitVars(pDnode, pOption) != 0) {
dError("failed to init variables since %s", terrstr());
goto _OVER;
}
dndSetStatus(pDnode, DND_STAT_INIT);
dmGetMgmtFp(&pDnode->wrappers[DNODE]);
mmGetMgmtFp(&pDnode->wrappers[MNODE]);
vmGetMgmtFp(&pDnode->wrappers[VNODES]);
qmGetMgmtFp(&pDnode->wrappers[QNODE]);
smGetMgmtFp(&pDnode->wrappers[SNODE]);
bmGetMgmtFp(&pDnode->wrappers[BNODE]);
for (ENodeType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = strdup(path);
pWrapper->shm.id = -1;
pWrapper->pDnode = pDnode;
if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pWrapper->procType = PROC_SINGLE;
taosInitRWLatch(&pWrapper->latch);
}
if (dndInitMsgHandle(pDnode) != 0) {
dError("failed to msg handles since %s", terrstr());
goto _OVER;
}
if (dndReadShmFile(pDnode) != 0) {
dError("failed to read shm file since %s", terrstr());
goto _OVER;
}
SMsgCb msgCb = dndCreateMsgcb(&pDnode->wrappers[0]);
tmsgSetDefaultMsgCb(&msgCb);
dInfo("dnode is created, data:%p", pDnode);
code = 0;
_OVER:
if (code != 0 && pDnode) {
dndClearVars(pDnode);
pDnode = NULL;
dError("failed to create dnode since %s", terrstr());
}
return pDnode;
}
void dndClose(SDnode *pDnode) {
if (pDnode == NULL) return;
if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
dError("dnode is shutting down, data:%p", pDnode);
return; return;
} }
monCleanup(); dInfo("start to close dnode, data:%p", pDnode);
walCleanUp(); dndSetStatus(pDnode, DND_STAT_STOPPED);
taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up"); for (ENodeType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
dndCloseNode(pWrapper);
} }
void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { dndClearVars(pDnode);
pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; dInfo("dnode is closed, data:%p", pDnode);
pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
} }
EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus status) { if (event == DND_EVENT_STOP) {
if (pDnode->status != status) { pDnode->event = event;
dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
pDnode->status = status;
} }
} }
void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
SStartupReq *pStartup = &pDnode->startup; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); SMgmtWrapper *pRetWrapper = pWrapper;
tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
pStartup->finished = 0; taosRLockLatch(&pWrapper->latch);
if (pWrapper->deployed) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
} else {
terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
pRetWrapper = NULL;
}
taosRUnLockLatch(&pWrapper->latch);
return pRetWrapper;
} }
void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); int32_t code = 0;
pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
taosRLockLatch(&pWrapper->latch);
if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
} else {
terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
code = -1;
}
taosRUnLockLatch(&pWrapper->latch);
return code;
} }
void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
dDebug("startup req is received"); if (pWrapper == NULL) return;
SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
dndGetStartup(pDnode, pStartup);
dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); taosRLockLatch(&pWrapper->latch);
SRpcMsg rpcRsp = { int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; taosRUnLockLatch(&pWrapper->latch);
rpcSendResponse(&rpcRsp); dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
} }

2
source/dnode/mgmt/main/src/dndMsg.c
View File

@ -66,7 +66,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
code = (*msgFp)(pWrapper, pMsg); code = (*msgFp)(pWrapper, pMsg);
} else if (pWrapper->procType == PROC_PARENT) { } else if (pWrapper->procType == PROC_PARENT) {
dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
pRpc->ahandle, pMsg->user); pRpc->ahandle, pMsg->user);
code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
} else { } else {

200
source/dnode/mgmt/main/src/dndObj.c
View File

@ -1,200 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndInt.h"
static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
pDnode->serverPort = pOption->serverPort;
pDnode->dataDir = strdup(pOption->dataDir);
pDnode->localEp = strdup(pOption->localEp);
pDnode->localFqdn = strdup(pOption->localFqdn);
pDnode->firstEp = strdup(pOption->firstEp);
pDnode->secondEp = strdup(pOption->secondEp);
pDnode->disks = pOption->disks;
pDnode->numOfDisks = pOption->numOfDisks;
pDnode->ntype = pOption->ntype;
pDnode->rebootTime = taosGetTimestampMs();
if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
pDnode->secondEp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
pDnode->lockfile = dndCheckRunning(pDnode->dataDir);
if (pDnode->lockfile == NULL) {
return -1;
}
}
return 0;
}
static void dndClearVars(SDnode *pDnode) {
for (ENodeType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
taosMemoryFreeClear(pMgmt->path);
}
if (pDnode->lockfile != NULL) {
taosUnLockFile(pDnode->lockfile);
taosCloseFile(&pDnode->lockfile);
pDnode->lockfile = NULL;
}
taosMemoryFreeClear(pDnode->localEp);
taosMemoryFreeClear(pDnode->localFqdn);
taosMemoryFreeClear(pDnode->firstEp);
taosMemoryFreeClear(pDnode->secondEp);
taosMemoryFreeClear(pDnode->dataDir);
taosMemoryFree(pDnode);
dDebug("dnode object memory is cleared, data:%p", pDnode);
}
SDnode *dndCreate(const SDnodeOpt *pOption) {
dDebug("start to create dnode object");
int32_t code = -1;
char path[PATH_MAX] = {0};
SDnode *pDnode = NULL;
pDnode = taosMemoryCalloc(1, sizeof(SDnode));
if (pDnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (dndInitVars(pDnode, pOption) != 0) {
dError("failed to init variables since %s", terrstr());
goto _OVER;
}
dndSetStatus(pDnode, DND_STAT_INIT);
dmGetMgmtFp(&pDnode->wrappers[DNODE]);
mmGetMgmtFp(&pDnode->wrappers[MNODE]);
vmGetMgmtFp(&pDnode->wrappers[VNODES]);
qmGetMgmtFp(&pDnode->wrappers[QNODE]);
smGetMgmtFp(&pDnode->wrappers[SNODE]);
bmGetMgmtFp(&pDnode->wrappers[BNODE]);
for (ENodeType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = strdup(path);
pWrapper->shm.id = -1;
pWrapper->pDnode = pDnode;
if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pWrapper->procType = PROC_SINGLE;
taosInitRWLatch(&pWrapper->latch);
}
if (dndInitMsgHandle(pDnode) != 0) {
dError("failed to msg handles since %s", terrstr());
goto _OVER;
}
if (dndReadShmFile(pDnode) != 0) {
dError("failed to read shm file since %s", terrstr());
goto _OVER;
}
SMsgCb msgCb = dndCreateMsgcb(&pDnode->wrappers[0]);
tmsgSetDefaultMsgCb(&msgCb);
dInfo("dnode object is created, data:%p", pDnode);
code = 0;
_OVER:
if (code != 0 && pDnode) {
dndClearVars(pDnode);
pDnode = NULL;
dError("failed to create dnode object since %s", terrstr());
}
return pDnode;
}
void dndClose(SDnode *pDnode) {
if (pDnode == NULL) return;
if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
dError("dnode is shutting down, data:%p", pDnode);
return;
}
dInfo("start to close dnode, data:%p", pDnode);
dndSetStatus(pDnode, DND_STAT_STOPPED);
for (ENodeType n = 0; n < NODE_MAX; ++n) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
dndCloseNode(pWrapper);
}
dndClearVars(pDnode);
dInfo("dnode object is closed, data:%p", pDnode);
}
void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
dInfo("dnode object receive event %d, data:%p", event, pDnode);
if (event == DND_EVENT_STOP) {
pDnode->event = event;
}
}
SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SMgmtWrapper *pRetWrapper = pWrapper;
taosRLockLatch(&pWrapper->latch);
if (pWrapper->deployed) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
} else {
terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
pRetWrapper = NULL;
}
taosRUnLockLatch(&pWrapper->latch);
return pRetWrapper;
}
int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
int32_t code = 0;
taosRLockLatch(&pWrapper->latch);
if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) {
int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
} else {
terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
code = -1;
}
taosRUnLockLatch(&pWrapper->latch);
return code;
}
void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
if (pWrapper == NULL) return;
taosRLockLatch(&pWrapper->latch);
int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
taosRUnLockLatch(&pWrapper->latch);
dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
}

13
source/dnode/mgmt/main/src/dndStr.c
View File

@ -62,3 +62,16 @@ const char *dndNodeProcStr(ENodeType ntype) {
return "taosd"; return "taosd";
} }
} }
const char *dndEventStr(EDndEvent ev) {
switch (ev) {
case DND_EVENT_START:
return "start";
case DND_EVENT_STOP:
return "stop";
case DND_EVENT_CHILD:
return "child";
default:
return "UNKNOWN";
}
}

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -735,6 +735,9 @@ typedef struct {
int8_t createdBy; // STREAM_CREATED_BY__USER or SMA int8_t createdBy; // STREAM_CREATED_BY__USER or SMA
int32_t fixedSinkVgId; // 0 for shuffle int32_t fixedSinkVgId; // 0 for shuffle
int64_t smaId; // 0 for unused int64_t smaId; // 0 for unused
int8_t trigger;
int32_t triggerParam;
int64_t waterMark;
char* sql; char* sql;
char* logicalPlan; char* logicalPlan;
char* physicalPlan; char* physicalPlan;

2
source/dnode/vnode/src/inc/vnd.h
View File

@ -194,7 +194,7 @@ void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
int32_t tqProcessRebReq(STQ* pTq, char* msg); int32_t tqProcessRebReq(STQ* pTq, char* msg);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);

2
source/dnode/vnode/src/meta/metaTbUid.c
View File

@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */
tb_uid_t metaGenerateUid(SMeta *pMeta) { tb_uid_t metaGenerateUid(SMeta *pMeta) {
// Generate a new table UID // Generate a new table UID
return ++(pMeta->uidGnrt.nextUid); return tGenIdPI32();
} }

48
source/dnode/vnode/src/tq/tq.c
View File

@ -250,7 +250,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
return 0; return 0;
} }
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
SMqPollReq* pReq = pMsg->pCont; SMqPollReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId; int64_t consumerId = pReq->consumerId;
int64_t fetchOffset; int64_t fetchOffset;
@ -264,6 +264,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
fetchOffset = pReq->currentOffset + 1; fetchOffset = pReq->currentOffset + 1;
} }
/*printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);*/
SMqPollRsp rsp = { SMqPollRsp rsp = {
/*.consumerId = consumerId,*/ /*.consumerId = consumerId,*/
.numOfTopics = 0, .numOfTopics = 0,
@ -288,37 +290,47 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
rsp.reqOffset = pReq->currentOffset; rsp.reqOffset = pReq->currentOffset;
rsp.skipLogNum = 0; rsp.skipLogNum = 0;
SWalHead* pHead;
while (1) { while (1) {
/*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { SWalReadHead* pHead;
if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
// TODO: no more log, set timer to wait blocking time // TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and // if data inserted during waiting, launch query and
// response to user // response to user
break; break;
} }
int8_t pos = fetchOffset % TQ_BUFFER_SIZE; /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType,
pHead = pTopic->pReadhandle->pHead; * pReq->epoch);*/
if (pHead->head.msgType == TDMT_VND_SUBMIT) { /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; /*pHead = pTopic->pReadhandle->pHead;*/
qTaskInfo_t task = pTopic->buffer.output[pos].task; if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
/*printf("from topic %s from consumer\n", pTopic->topicName, consumerId);*/
qTaskInfo_t task = pTopic->buffer.output[workerId].task;
ASSERT(task);
qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
while (1) { while (1) {
SSDataBlock* pDataBlock; SSDataBlock* pDataBlock = NULL;
uint64_t ts; uint64_t ts;
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(false); ASSERT(false);
} }
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
fetchOffset++; /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
pos = fetchOffset % TQ_BUFFER_SIZE;
rsp.skipLogNum++;
break; break;
} }
taosArrayPush(pRes, pDataBlock); taosArrayPush(pRes, pDataBlock);
rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; }
if (taosArrayGetSize(pRes) == 0) {
fetchOffset++;
rsp.skipLogNum++;
taosArrayDestroy(pRes);
continue;
}
rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper;
rsp.rspOffset = fetchOffset; rsp.rspOffset = fetchOffset;
rsp.numOfTopics = 1; rsp.numOfTopics = 1;
@ -328,6 +340,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
void* buf = rpcMallocCont(tlen); void* buf = rpcMallocCont(tlen);
if (buf == NULL) { if (buf == NULL) {
pMsg->code = -1; pMsg->code = -1;
taosMemoryFree(pHead);
return -1; return -1;
} }
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
@ -340,10 +353,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->pCont = buf; pMsg->pCont = buf;
pMsg->contLen = tlen; pMsg->contLen = tlen;
pMsg->code = 0; pMsg->code = 0;
/*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset,
* pHead->msgType,*/
/*pReq->epoch);*/
tmsgSendRsp(pMsg); tmsgSendRsp(pMsg);
taosMemoryFree(pHead);
return 0; return 0;
}
} else { } else {
taosMemoryFree(pHead);
fetchOffset++; fetchOffset++;
rsp.skipLogNum++; rsp.skipLogNum++;
} }
@ -368,6 +385,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->contLen = tlen; pMsg->contLen = tlen;
pMsg->code = 0; pMsg->code = 0;
tmsgSendRsp(pMsg); tmsgSendRsp(pMsg);
/*printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch);*/
/*}*/ /*}*/
return 0; return 0;
@ -432,7 +450,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
}; };
pTopic->buffer.output[i].pReadHandle = pReadHandle; pTopic->buffer.output[i].pReadHandle = pReadHandle;
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
ASSERT(pTopic->buffer.output[i].task);
} }
printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId);
taosArrayPush(pConsumer->topics, pTopic); taosArrayPush(pConsumer->topics, pTopic);
tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
tqHandleCommit(pTq->tqMeta, req.consumerId); tqHandleCommit(pTq->tqMeta, req.consumerId);

1
source/dnode/vnode/src/tq/tqRead.c
View File

@ -168,6 +168,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
break; break;
} }
if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {
/*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/
taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
return NULL; return NULL;
} }

2
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -1394,7 +1394,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64
" numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast);
return 0; return 0;

2
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -66,7 +66,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
case TDMT_VND_TABLE_META: case TDMT_VND_TABLE_META:
return vnodeGetTableMeta(pVnode, pMsg); return vnodeGetTableMeta(pVnode, pMsg);
case TDMT_VND_CONSUME: case TDMT_VND_CONSUME:
return tqProcessPollReq(pVnode->pTq, pMsg); return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId);
case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_PIPE_EXEC:
case TDMT_VND_TASK_MERGE_EXEC: case TDMT_VND_TASK_MERGE_EXEC:
return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0);

1
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -165,6 +165,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// } // }
break; break;
case TDMT_VND_SUBMIT: case TDMT_VND_SUBMIT:
/*printf("vnode %d write data %ld\n", pVnode->vgId, ver);*/
if (pVnode->config.streamMode == 0) { if (pVnode->config.streamMode == 0) {
if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) { if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) {
// TODO: handle error // TODO: handle error

7
source/libs/index/inc/indexFstRegex.h
View File

@ -63,9 +63,10 @@ typedef struct {
FstRegex *regexCreate(const char *str); FstRegex *regexCreate(const char *str);
void regexSetup(FstRegex *regex, uint32_t size, const char *str); uint32_t regexAutomStart(FstRegex *regex);
bool regexAutomIsMatch(FstRegex *regex, uint32_t state);
// uint32_t regexStart() bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null);
bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result);
#ifdef __cplusplus #ifdef __cplusplus
} }

4
source/libs/index/inc/indexFstSparse.h
View File

@ -23,8 +23,8 @@ extern "C" {
#endif #endif
typedef struct FstSparseSet { typedef struct FstSparseSet {
SArray *dense; uint32_t *dense;
SArray *sparse; uint32_t *sparse;
int32_t size; int32_t size;
} FstSparseSet; } FstSparseSet;

2
source/libs/index/src/indexFstCommon.c
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com> YAML:9:25: error: unknown key 'AlignConsecutiveMacros' * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
* *
* This program is free software: you can use, redistribute, and/or modify * This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3 * it under the terms of the GNU Affero General Public License, version 3

33
source/libs/index/src/indexFstRegex.c
View File

@ -14,6 +14,7 @@
*/ */
#include "indexFstRegex.h" #include "indexFstRegex.h"
#include "indexFstDfa.h"
#include "indexFstSparse.h" #include "indexFstSparse.h"
FstRegex *regexCreate(const char *str) { FstRegex *regexCreate(const char *str) {
@ -26,9 +27,35 @@ FstRegex *regexCreate(const char *str) {
memcpy(orig, str, sz); memcpy(orig, str, sz);
regex->orig = orig; regex->orig = orig;
// construct insts based on str
SArray *insts = NULL;
FstDfaBuilder *builder = dfaBuilderCreate(insts);
regex->dfa = dfaBuilderBuild(builder);
return regex;
} }
void regexSetup(FstRegex *regex, uint32_t size, const char *str) { uint32_t regexAutomStart(FstRegex *regex) {
// return ///// no nothing
// return; return 0;
}
bool regexAutomIsMatch(FstRegex *regex, uint32_t state) {
if (regex->dfa != NULL && dfaIsMatch(regex->dfa, state)) {
return true;
} else {
return false;
}
}
bool regexAutomCanMatch(FstRegex *regex, uint32_t state, bool null) {
// make frame happy
return null;
}
bool regexAutomAccept(FstRegex *regex, uint32_t state, uint8_t byte, uint32_t *result) {
if (regex->dfa == NULL) {
return false;
}
return dfaAccept(regex->dfa, state, byte, result);
} }

37
source/libs/index/src/indexSparse.c → source/libs/index/src/indexFstSparse.c
View File

@ -21,47 +21,44 @@ FstSparseSet *sparSetCreate(int32_t sz) {
return NULL; return NULL;
} }
ss->dense = taosArrayInit(sz, sizeof(uint32_t)); ss->dense = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
ss->sparse = taosArrayInit(sz, sizeof(uint32_t)); ss->sparse = (uint32_t *)taosMemoryCalloc(sz, sizeof(uint32_t));
ss->size = sz; ss->size = 0;
return ss; return ss;
} }
void sparSetDestroy(FstSparseSet *ss) { void sparSetDestroy(FstSparseSet *ss) {
if (ss == NULL) { if (ss == NULL) {
return; return;
} }
taosArrayDestroy(ss->dense); taosMemoryFree(ss->dense);
taosArrayDestroy(ss->sparse); taosMemoryFree(ss->sparse);
taosMemoryFree(ss); taosMemoryFree(ss);
} }
uint32_t sparSetLen(FstSparseSet *ss) { return ss == NULL ? 0 : ss->size; } uint32_t sparSetLen(FstSparseSet *ss) {
// Get occupied size
return ss == NULL ? 0 : ss->size;
}
uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) { uint32_t sparSetAdd(FstSparseSet *ss, uint32_t ip) {
if (ss == NULL) { if (ss == NULL) {
return 0; return 0;
} }
uint32_t i = ss->size; uint32_t i = ss->size;
taosArraySet(ss->dense, i, &ip); ss->dense[i] = ip;
taosArraySet(ss->sparse, ip, &i); ss->sparse[ip] = i;
ss->size += 1; ss->size += 1;
return i; return i;
} }
uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) { uint32_t sparSetGet(FstSparseSet *ss, uint32_t i) {
if (i >= taosArrayGetSize(ss->dense)) { // check later
return 0; return ss->dense[i];
}
uint32_t *v = taosArrayGet(ss->dense, i);
return *v;
} }
bool sparSetContains(FstSparseSet *ss, uint32_t ip) { bool sparSetContains(FstSparseSet *ss, uint32_t ip) {
if (ip >= taosArrayGetSize(ss->sparse)) { uint32_t i = ss->sparse[ip];
if (i < ss->size && ss->dense[i] == ip) {
return true;
} else {
return false; return false;
} }
uint32_t i = *(uint32_t *)taosArrayGet(ss->sparse, ip);
if (i >= taosArrayGetSize(ss->dense)) {
return false;
}
uint32_t v = *(uint32_t *)taosArrayGet(ss->dense, i);
return v == ip;
} }
void sparSetClear(FstSparseSet *ss) { void sparSetClear(FstSparseSet *ss) {
if (ss == NULL) { if (ss == NULL) {

5
source/libs/parser/src/parTranslater.c
View File

@ -1217,6 +1217,9 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt
SName tableName; SName tableName;
int32_t code = getTableMetaImpl( int32_t code = getTableMetaImpl(
pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta);
if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) {
return TSDB_CODE_SUCCESS;
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (TSDB_SUPER_TABLE == pTableMeta->tableType) { if (TSDB_SUPER_TABLE == pTableMeta->tableType) {
code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists);
@ -2593,8 +2596,10 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
break; break;
default: default:
pQuery->directRpc = true; pQuery->directRpc = true;
if (NULL != pCxt->pCmdMsg) {
TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*); TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*);
pQuery->msgType = pQuery->pCmdMsg->msgType; pQuery->msgType = pQuery->pCmdMsg->msgType;
}
break; break;
} }

5
source/libs/planner/src/planPhysiCreater.c
View File

@ -719,6 +719,11 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc
if (NULL == pScan->pScanCols) { if (NULL == pScan->pScanCols) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (TSDB_CODE_SUCCESS == code) {
code = sortScanCols(pScan->pScanCols);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = sortScanCols(pScan->pScanCols); code = sortScanCols(pScan->pScanCols);
} }

74
source/libs/scheduler/src/scheduler.c
View File

@ -127,11 +127,13 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp case TDMT_VND_QUERY_RSP: // query_rsp may be processed later than ready_rsp
if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) { if (lastMsgType != reqMsgType && -1 != lastMsgType && TDMT_VND_FETCH != lastMsgType) {
SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); SCH_TASK_DLOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType),
TMSG_INFO(msgType));
} }
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_TASK_DLOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
} }
SCH_SET_TASK_LASTMSG_TYPE(pTask, -1); SCH_SET_TASK_LASTMSG_TYPE(pTask, -1);
@ -139,12 +141,14 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
case TDMT_VND_RES_READY_RSP: case TDMT_VND_RES_READY_RSP:
reqMsgType = TDMT_VND_QUERY; reqMsgType = TDMT_VND_QUERY;
if (lastMsgType != reqMsgType && -1 != lastMsgType) { if (lastMsgType != reqMsgType && -1 != lastMsgType) {
SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", (lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s",
(lastMsgType > 0 ? TMSG_INFO(lastMsgType) : "null"), TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
@ -152,12 +156,14 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
case TDMT_VND_FETCH_RSP: case TDMT_VND_FETCH_RSP:
if (lastMsgType != reqMsgType && -1 != lastMsgType) { if (lastMsgType != reqMsgType && -1 != lastMsgType) {
SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
@ -172,12 +178,14 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m
} }
if (lastMsgType != reqMsgType) { if (lastMsgType != reqMsgType) {
SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg type mis-match, last sent msgType:%s, rspType:%s", TMSG_INFO(lastMsgType),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { if (taskStatus != JOB_TASK_STATUS_EXECUTING && taskStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) {
SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus),
TMSG_INFO(msgType));
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }
@ -655,7 +663,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
if (SCH_IS_DATA_SRC_TASK(pTask)) { if (SCH_IS_DATA_SRC_TASK(pTask)) {
if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
*needRetry = false; *needRetry = false;
SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes,
SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} else { } else {
@ -663,7 +672,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
if ((pTask->candidateIdx + 1) >= candidateNum) { if ((pTask->candidateIdx + 1) >= candidateNum) {
*needRetry = false; *needRetry = false;
SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
pTask->candidateIdx, candidateNum);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
@ -707,9 +717,8 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) {
memcpy(&hb->trans, trans, sizeof(*trans)); memcpy(&hb->trans, trans, sizeof(*trans));
SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_WRITE, &hb->lock);
qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, instance:%p, handle:%p", schMgmt.sId,
schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->transInst, trans->transHandle);
trans->transHandle);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -966,7 +975,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
int8_t status = 0; int8_t status = 0;
if (schJobNeedToStop(pJob, &status)) { if (schJobNeedToStop(pJob, &status)) {
SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status),
rspCode);
SCH_RET(atomic_load_32(&pJob->errCode)); SCH_RET(atomic_load_32(&pJob->errCode));
} }
@ -1174,7 +1184,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) {
SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans)); SCH_ERR_RET(schUpdateHbConnection(&rsp.epId, &trans));
int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus);
qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, rsp.epId.ep.port); qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn,
rsp.epId.ep.port);
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i);
@ -1189,7 +1200,8 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) {
// TODO // TODO
SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId,
jobTaskStatusStr(taskStatus->status));
schReleaseJob(taskStatus->refId); schReleaseJob(taskStatus->refId);
} }
@ -1220,7 +1232,6 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
switch (msgType) { switch (msgType) {
case TDMT_VND_CREATE_TABLE: case TDMT_VND_CREATE_TABLE:
@ -1302,7 +1313,6 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) { int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb) {
int32_t code = 0; int32_t code = 0;
SMsgSendInfo *pMsgSendInfo = NULL; SMsgSendInfo *pMsgSendInfo = NULL;
@ -1451,7 +1461,6 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) {
int32_t code = 0; int32_t code = 0;
SSchHbTrans hb = {0}; SSchHbTrans hb = {0};
@ -1476,8 +1485,6 @@ int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) { int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHeader **pDst) {
if (pSrc->isHbParam) { if (pSrc->isHbParam) {
SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam));
@ -1569,8 +1576,8 @@ _return:
SCH_RET(code); SCH_RET(code);
} }
int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet *epSet, int32_t msgType, void *msg,
int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet* epSet, int32_t msgType, void *msg, uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) { uint32_t msgSize, bool persistHandle, SRpcCtx *ctx) {
int32_t code = 0; int32_t code = 0;
SSchTrans *trans = (SSchTrans *)transport; SSchTrans *trans = (SSchTrans *)transport;
@ -1602,9 +1609,9 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, void *transport, SEpSet*
pMsgSendInfo->msgType = msgType; pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = fp; pMsgSendInfo->fp = fp;
qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", qDebug("start to send %s msg to node[%d,%s,%d], refId:%" PRIx64 "instance:%p, handle:%p", TMSG_INFO(msgType),
TMSG_INFO(msgType), ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, ntohl(((SMsgHead *)msg)->vgId), epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, pJob->refId,
pJob->refId, trans->transInst, trans->transHandle); trans->transInst, trans->transHandle);
int64_t transporterId = 0; int64_t transporterId = 0;
code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx); code = asyncSendMsgToServerExt(trans->transInst, epSet, &transporterId, pMsgSendInfo, persistHandle, ctx);
@ -1629,13 +1636,14 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) {
SSchTrans trans = {0}; SSchTrans trans = {0};
int32_t msgType = TDMT_VND_QUERY_HEARTBEAT; int32_t msgType = TDMT_VND_QUERY_HEARTBEAT;
req.header.vgId = htonl(nodeEpId->nodeId); req.header.vgId = nodeEpId->nodeId;
req.sId = schMgmt.sId; req.sId = schMgmt.sId;
memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId));
SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId));
if (NULL == hb) { if (NULL == hb) {
qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn,
nodeEpId->ep.port);
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
@ -1690,12 +1698,13 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) {
SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SEpSet epSet = {.inUse = 0, .numOfEps = 1};
memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep)); memcpy(&epSet.eps[0], &nodeEpId->ep, sizeof(nodeEpId->ep));
qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port); qDebug("start to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d", trans.transInst, trans.transHandle,
nodeEpId->ep.fqdn, nodeEpId->ep.port);
code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx); code = asyncSendMsgToServerExt(trans.transInst, &epSet, &transporterId, pMsgSendInfo, true, &rpcCtx);
if (code) { if (code) {
qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", qError("fail to send hb msg, instance:%p, handle:%p, fqdn:%s, port:%d, error:%x - %s", trans.transInst,
trans.transInst, trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code)); trans.transHandle, nodeEpId->ep.fqdn, nodeEpId->ep.port, code, tstrerror(code));
SCH_ERR_JRET(code); SCH_ERR_JRET(code);
} }
@ -1857,7 +1866,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType); SCH_SET_TASK_LASTMSG_TYPE(pTask, msgType);
SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)}; SSchTrans trans = {.transInst = pJob->transport, .transHandle = SCH_GET_TASK_HANDLE(pTask)};
SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL))); SCH_ERR_JRET(schAsyncSendMsg(pJob, pTask, &trans, &epSet, msgType, msg, msgSize, persistHandle,
(rpcCtx.args ? &rpcCtx : NULL)));
if (msgType == TDMT_VND_QUERY) { if (msgType == TDMT_VND_QUERY) {
SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle)); SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.transHandle));

2
source/libs/stream/src/tstream.c
View File

@ -38,7 +38,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
req.taskId = pTask->fixedEpDispatcher.taskId; req.taskId = pTask->fixedEpDispatcher.taskId;
} else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
// TODO fix tbname issue // TODO use general name rule of schemaless
char ctbName[TSDB_TABLE_FNAME_LEN + 22]; char ctbName[TSDB_TABLE_FNAME_LEN + 22];
// all groupId must be the same in an array // all groupId must be the same in an array
SSDataBlock* pBlock = taosArrayGet(data, 0); SSDataBlock* pBlock = taosArrayGet(data, 0);

23
source/libs/wal/src/walRead.c
View File

@ -30,6 +30,9 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
pRead->curFileFirstVer = -1; pRead->curFileFirstVer = -1;
pRead->capacity = 0; pRead->capacity = 0;
pRead->status = 0; pRead->status = 0;
taosThreadMutexInit(&pRead->mutex, NULL);
pRead->pHead = taosMemoryMalloc(sizeof(SWalHead)); pRead->pHead = taosMemoryMalloc(sizeof(SWalHead));
if (pRead->pHead == NULL) { if (pRead->pHead == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
@ -135,6 +138,22 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
return 0; return 0;
} }
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) {
taosThreadMutexLock(&pRead->mutex);
if (walReadWithHandle(pRead, ver) < 0) {
taosThreadMutexUnlock(&pRead->mutex);
return -1;
}
*ppHead = taosMemoryMalloc(sizeof(SWalReadHead) + pRead->pHead->head.len);
if (*ppHead == NULL) {
taosThreadMutexUnlock(&pRead->mutex);
return -1;
}
memcpy(*ppHead, &pRead->pHead->head, sizeof(SWalReadHead) + pRead->pHead->head.len);
taosThreadMutexUnlock(&pRead->mutex);
return 0;
}
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int code; int code;
// TODO: check wal life // TODO: check wal life
@ -145,7 +164,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
} }
} }
if (!taosValidFile(pRead->pReadLogTFile)) return -1; if (!taosValidFile(pRead->pReadLogTFile)) {
return -1;
}
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) { if (code != sizeof(SWalHead)) {

7
source/os/src/osProc.c
View File

@ -23,6 +23,9 @@ int32_t taosNewProc(char **args) {
int32_t pid = fork(); int32_t pid = fork();
if (pid == 0) { if (pid == 0) {
args[0] = tsProcPath; args[0] = tsProcPath;
close(STDIN_FILENO);
close(STDOUT_FILENO);
close(STDERR_FILENO);
return execvp(tsProcPath, args); return execvp(tsProcPath, args);
} else { } else {
return pid; return pid;
@ -39,7 +42,7 @@ void taosSetProcName(int32_t argc, char **argv, const char *name) {
argv[i][j] = 0; argv[i][j] = 0;
} }
if (i == 0) { if (i == 0) {
tstrncpy(argv[0], name, len); tstrncpy(argv[0], name, len + 1);
} }
} }
} }
@ -48,5 +51,5 @@ void taosSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; }
bool taosProcExists(int32_t pid) { bool taosProcExists(int32_t pid) {
int32_t p = getpgid(pid); int32_t p = getpgid(pid);
return p == 0; return p >= 0;
} }

6
source/util/src/tconfig.c
View File

@ -590,12 +590,12 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
} }
int32_t cfgLoadFromEnvVar(SConfig *pConfig) { int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
uInfo("load from global env variables"); uInfo("load from global env variables not implemented yet");
return 0; return 0;
} }
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
uInfo("load from env file %s", filepath); uInfo("load from env file not implemented yet");
return 0; return 0;
} }
@ -654,6 +654,6 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
} }
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
uInfo("load from apoll url %s", url); uInfo("load from apoll url not implemented yet");
return 0; return 0;
} }

7
source/util/src/tjson.c
View File

@ -121,7 +121,8 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) {
return tjsonAddItemToArray(pJson, pJobj); return tjsonAddItemToArray(pJson, pJobj);
} }
int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num) { int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize,
int32_t num) {
if (num > 0) { if (num > 0) {
SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName);
if (NULL == pJsonArray) { if (NULL == pJsonArray) {
@ -168,7 +169,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal
} }
*pVal = strtol(p, NULL, 10); *pVal = strtol(p, NULL, 10);
return (errno == EINVAL || errno == ERANGE) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal) { int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal) {
@ -199,7 +200,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV
} }
*pVal = strtoul(p, NULL, 10); *pVal = strtoul(p, NULL, 10);
return (errno == ERANGE||errno == EINVAL) ? TSDB_CODE_FAILED:TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) { int32_t tjsonGetUIntValue(const SJson* pJson, const char* pName, uint32_t* pVal) {

26
source/util/src/tprocess.c
View File

@ -140,14 +140,16 @@ static void taosProcDestroySem(SProcQueue *pQueue) {
pQueue->sem = NULL; pQueue->sem = NULL;
} }
} }
#endif
static void taosProcCleanupQueue(SProcQueue *pQueue) { static void taosProcCleanupQueue(SProcQueue *pQueue) {
#if 0
if (pQueue != NULL) { if (pQueue != NULL) {
taosProcDestroyMutex(pQueue); taosProcDestroyMutex(pQueue);
taosProcDestroySem(pQueue); taosProcDestroySem(pQueue);
} }
}
#endif #endif
}
static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody, static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody,
int32_t rawBodyLen, ProcFuncType ftype) { int32_t rawBodyLen, ProcFuncType ftype) {
@ -222,7 +224,6 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea
taosThreadMutexLock(&pQueue->mutex); taosThreadMutexLock(&pQueue->mutex);
if (pQueue->total - pQueue->avail <= 0) { if (pQueue->total - pQueue->avail <= 0) {
taosThreadMutexUnlock(&pQueue->mutex); taosThreadMutexUnlock(&pQueue->mutex);
tsem_post(&pQueue->sem);
terrno = TSDB_CODE_OUT_OF_SHM_MEM; terrno = TSDB_CODE_OUT_OF_SHM_MEM;
return 0; return 0;
} }
@ -317,7 +318,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) {
pProc->pChildQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + cstart, csize); pProc->pChildQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + cstart, csize);
pProc->pParentQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + pstart, psize); pProc->pParentQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + pstart, psize);
if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) {
// taosProcCleanupQueue(pProc->pChildQueue); taosProcCleanupQueue(pProc->pChildQueue);
taosMemoryFree(pProc); taosMemoryFree(pProc);
return NULL; return NULL;
} }
@ -336,7 +337,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) {
pProc->parentConsumeFp = pCfg->parentConsumeFp; pProc->parentConsumeFp = pCfg->parentConsumeFp;
pProc->isChild = pCfg->isChild; pProc->isChild = pCfg->isChild;
uDebug("proc:%s, is initialized, child:%d child queue:%p parent queue:%p", pProc->name, pProc->isChild, uDebug("proc:%s, is initialized, isChild:%d child queue:%p parent queue:%p", pProc->name, pProc->isChild,
pProc->pChildQueue, pProc->pParentQueue); pProc->pChildQueue, pProc->pParentQueue);
return pProc; return pProc;
@ -376,7 +377,7 @@ static void taosProcThreadLoop(SProcObj *pProc) {
int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype, mallocHeadFp, freeHeadFp, int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype, mallocHeadFp, freeHeadFp,
mallocBodyFp, freeBodyFp); mallocBodyFp, freeBodyFp);
if (numOfMsgs == 0) { if (numOfMsgs == 0) {
uInfo("proc:%s, get no msg from queue:%p and exit the proc thread", pProc->name, pQueue); uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pProc->name, pQueue);
break; break;
} else if (numOfMsgs < 0) { } else if (numOfMsgs < 0) {
uTrace("proc:%s, get no msg from queue:%p since %s", pProc->name, pQueue, terrstr()); uTrace("proc:%s, get no msg from queue:%p since %s", pProc->name, pQueue, terrstr());
@ -399,19 +400,19 @@ int32_t taosProcRun(SProcObj *pProc) {
return -1; return -1;
} }
uDebug("proc:%s, start to consume queue:%p", pProc->name, pProc->pChildQueue); uDebug("proc:%s, start to consume queue:%p, thread:%" PRId64, pProc->name, pProc->pChildQueue, pProc->thread);
return 0; return 0;
} }
static void taosProcStop(SProcObj *pProc) { static void taosProcStop(SProcObj *pProc) {
if (!taosCheckPthreadValid(pProc->thread)) return; if (!taosCheckPthreadValid(pProc->thread)) return;
uDebug("proc:%s, start to join thread", pProc->name); uDebug("proc:%s, start to join thread:%" PRId64, pProc->name, pProc->thread);
SProcQueue *pQueue; SProcQueue *pQueue;
if (pProc->isChild) { if (pProc->isChild) {
pQueue = pProc->pParentQueue;
} else {
pQueue = pProc->pChildQueue; pQueue = pProc->pChildQueue;
} else {
pQueue = pProc->pParentQueue;
} }
tsem_post(&pQueue->sem); tsem_post(&pQueue->sem);
taosThreadJoin(pProc->thread, NULL); taosThreadJoin(pProc->thread, NULL);
@ -419,10 +420,11 @@ static void taosProcStop(SProcObj *pProc) {
void taosProcCleanup(SProcObj *pProc) { void taosProcCleanup(SProcObj *pProc) {
if (pProc != NULL) { if (pProc != NULL) {
uDebug("proc:%s, clean up", pProc->name); uDebug("proc:%s, start to clean up", pProc->name);
taosProcStop(pProc); taosProcStop(pProc);
// taosProcCleanupQueue(pProc->pChildQueue); taosProcCleanupQueue(pProc->pChildQueue);
// taosProcCleanupQueue(pProc->pParentQueue); taosProcCleanupQueue(pProc->pParentQueue);
uDebug("proc:%s, is cleaned up", pProc->name);
taosMemoryFree(pProc); taosMemoryFree(pProc);
} }
} }

1
tests/script/jenkins/basic.txt
View File

@ -5,6 +5,7 @@
./test.sh -f tsim/user/basic1.sim ./test.sh -f tsim/user/basic1.sim
# ---- db # ---- db
./test.sh -f tsim/db/alter_option.sim
./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic2.sim
./test.sh -f tsim/db/basic3.sim ./test.sh -f tsim/db/basic3.sim

172
tests/script/tsim/db/alter_option.sim
View File

@ -59,22 +59,12 @@ endi
print ============= create database print ============= create database
#database_option: { #database_option: {
# BLOCKS value [3~1000, default: 6] # BLOCKS value [3~1000, default: 6]
# | CACHE value [default: 16M]
# | CACHELAST value [0, 1, 2, 3] # | CACHELAST value [0, 1, 2, 3]
# | COMP [0 | 1 | 2]
# | DAYS value [unit is minutes]
# | FSYNC value [0 ~ 180000 ms] # | FSYNC value [0 ~ 180000 ms]
# | MAXROWS value [default: 4096]
# | MINROWS value [default: 100]
# | KEEP value [days, 365000] # | KEEP value [days, 365000]
# | PRECISION ['ms' | 'us' | 'ns']
# | QUORUM value [1 | 2] # | QUORUM value [1 | 2]
# | REPLICA value [1 | 3] # | REPLICA value [1 | 3]
# | TTL value [unit is day, min=1]
# | WAL value [1 | 2] # | WAL value [1 | 2]
# | VGROUPS value [default: 2]
# | SINGLE_STABLE [0 | 1]
# | STREAM_MODE [0 | 1]
sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1
sql show databases sql show databases
@ -187,22 +177,10 @@ sql_error alter database db quorum 4
sql_error alter database db quorum 5 sql_error alter database db quorum 5
#print ============== modify days #print ============== modify days
#sql alter database db days 480 sql_error alter database db days 480
#sql show databases sql_error alter database db days 360
#print days $data6_db
#if $data6_db != 480 then # days
# return -1
#endi
#sql alter database db days 360
#sql show databases
#print days $data6_db
#if $data6_db != 360 then # days
# return -1
#endi
sql_error alter database db days 0 sql_error alter database db days 0
#sql_error alter database db days 14400 # set over than keep sql_error alter database db days 14400 # set over than keep
print ============== modify keep print ============== modify keep
sql alter database db keep 2000 sql alter database db keep 2000
@ -230,25 +208,14 @@ sql_error alter database db keep -1
#sql_error alter database db keep 365001 #sql_error alter database db keep 365001
print ============== modify cache print ============== modify cache
#sql alter database db cache 12 sql_error alter database db cache 12
#sql show databases sql_error alter database db cache 1
#print cache $data8_db sql_error alter database db cache 60
#if $data8_db != 12 then sql_error alter database db cache 50
# return -1 sql_error alter database db cache 20
#endi sql_error alter database db cache 3
#sql alter database db cache 1 sql_error alter database db cache 129
#sql show databases sql_error alter database db cache 300
#print cache $data8_db
#if $data8_db != 6 then
# return -1
#endi
#
#sql_error alter database db cache 60
#sql_error alter database db cache 50
#sql_error alter database db cache 20
#sql_error alter database db cache 3
#sql_error alter database db cache 129
#sql_error alter database db cache 300
sql_error alter database db cache 0 sql_error alter database db cache 0
sql_error alter database db cache -1 sql_error alter database db cache -1
@ -276,45 +243,18 @@ sql_error alter database db blocks 0
sql_error alter database db blocks -1 sql_error alter database db blocks -1
sql_error alter database db blocks 10001 sql_error alter database db blocks 10001
#print ============== modify minrows print ============== modify minrows
#sql alter database db minrows 8 sql_error alter database db minrows 8
#sql show databases sql_error alter database db minrows 200
#print minrows $data10_db sql_error alter database db minrows 11
#if $data10_db != 8 then sql_error alter database db minrows 8000
# return -1 sql_error alter database db minrows 8001
#endi
#sql alter database db minrows 200
#sql show databases
#print minrows $data10_db
#if $data10_db != 200 then
# return -1
#endi
#
#sql alter database db minrows 11
#sql show databases
#print minrows $data10_db
#if $data10_db != 11 then
# return -1
#endi
#sql_error alter database db minrows 8000
#sql_error alter database db minrows 8001
#print ============== modify maxrows print ============== modify maxrows
#sql alter database db maxrows 1000 sql_error alter database db maxrows 1000
#sql show databases sql_error alter database db maxrows 2000
#print maxrows $data11_db sql_error alter database db maxrows 11 # equal minrows
#if $data11_db != 1000 then sql_error alter database db maxrows 10 # little than minrows
# return -1
#endi
#sql alter database db maxrows 2000
#sql show databases
#print maxrows $data11_db
#if $data11_db != 2000 then
# return -1
#endi
#
#sql_error alter database db maxrows 11 # equal minrows
#sql_error alter database db maxrows 10 # little than minrows
print ============== step wal print ============== step wal
sql alter database db wal 1 sql alter database db wal 1
@ -330,7 +270,7 @@ if $data12_db != 2 then
return -1 return -1
endi endi
sql_error alter database db wal 0 #sql_error alter database db wal 0 # TD-14436
sql_error alter database db wal 3 sql_error alter database db wal 3
sql_error alter database db wal 100 sql_error alter database db wal 100
sql_error alter database db wal -1 sql_error alter database db wal -1
@ -348,35 +288,20 @@ print fsync $data13_db
if $data13_db != 500 then if $data13_db != 500 then
return -1 return -1
endi endi
sql_error alter database db fsync 0 sql alter database db fsync 0
sql show databases
print fsync $data13_db
if $data13_db != 0 then
return -1
endi
sql_error alter database db fsync 180001
sql_error alter database db fsync -1 sql_error alter database db fsync -1
print ============== modify comp print ============== modify comp
sql alter database db comp 1 sql_error alter database db comp 1
sql show databases sql_error alter database db comp 2
print comp $data14_db sql_error alter database db comp 1
if $data14_db != 1 then sql_error alter database db comp 0
return -1
endi
sql alter database db comp 2
sql show databases
print comp $data14_db
if $data14_db != 2 then
return -1
endi
sql alter database db comp 1
sql show databases
print comp $data14_db
if $data14_db != 1 then
return -1
endi
sql alter database db comp 0
sql show databases
print comp $data14_db
if $data14_db != 0 then
return -1
endi
sql_error alter database db comp 3 sql_error alter database db comp 3
sql_error alter database db comp 4 sql_error alter database db comp 4
sql_error alter database db comp 5 sql_error alter database db comp 5
@ -414,30 +339,15 @@ if $data15_db != 3 then
return -1 return -1
endi endi
sql_error alter database db comp 4 sql_error alter database db cachelast 4
sql_error alter database db comp 10 sql_error alter database db cachelast 10
sql_error alter database db comp -1 sql_error alter database db cachelast -1
print ============== modify precision print ============== modify precision
sql alter database db precision 'ms' sql_error alter database db precision 'ms'
sql show databases sql_error alter database db precision 'us'
print precision $data16_db sql_error alter database db precision 'ns'
if $data16_db != ms then sql_error alter database db precision 'ys'
return -1
endi
sql alter database db precision 'us'
sql show databases
print precision $data16_db
if $data16_db != us then
return -1
endi
sql alter database db precision 'ns'
sql show databases
print precision $data16_db
if $data16_db != ns then
return -1
endi
sql_error alter database db prec 'xs' sql_error alter database db prec 'xs'
#system sh/exec.sh -n dnode1 -s stop -x SIGINT #system sh/exec.sh -n dnode1 -s stop -x SIGINT

34
tests/script/tsim/query/interval-offset.sim
View File

@ -176,6 +176,33 @@ if $data70 != 1 then
return -1 return -1
endi endi
sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from ct2 interval(1d, 2h) sliding(12h)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h)
print ===> rows: $rows
print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05
print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15
print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25
print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35
print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55
print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65
print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75
if $rows != 8 then
return -1
endi
if $data05 != 1 then
return -1
endi
if $data15 != 2 then
return -1
endi
if $data75 != 1 then
return -1
endi
if $data02 != 86400000 then
return -1
endi
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
print ===> rows: $rows print ===> rows: $rows
@ -187,7 +214,7 @@ print ===> rows4: $data40 $data41 $data42 $data43 $data44
# if $rows != 5 then # if $rows != 5 then
# return -1 # return -1
# endi # endi
#if $data00 != 1 then # f $data00 != 1 then
# return -1 # return -1
# endi # endi
# if $data40 != 1 then # if $data40 != 1 then
@ -322,9 +349,4 @@ endi
#sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d)
#system sh/exec.sh -n dnode1 -s stop -x SIGINT #system sh/exec.sh -n dnode1 -s stop -x SIGINT

28
tests/script/tsim/query/interval.sim
View File

@ -59,7 +59,7 @@ if $data04 != 1 then
return -1 return -1
endi endi
#print =============== step3 print =============== step3
#$cc = 4 * 60000 #$cc = 4 * 60000
#$ms = 1601481600000 + $cc #$ms = 1601481600000 + $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m)
@ -75,25 +75,25 @@ endi
# return -1 # return -1
#endi #endi
#print =============== step4 print =============== step4
#$cc = 40 * 60000 #$cc = 40 * 60000
#$ms = 1601481600000 + $cc #$ms = 1601481600000 + $cc
#$cc = 1 * 60000 #$cc = 1 * 60000
#$ms2 = 1601481600000 - $cc #$ms2 = 1601481600000 - $cc
#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m)
#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m)
#print ===> $rows $data01 $data05 print ===> $rows $data01 $data05
#if $rows != 20 then if $rows != $rowNum then
# return -1 return -1
#endi endi
#if $data00 != 1 then if $data05 != 1 then
# return -1 return -1
#endi endi
#if $data04 != 1 then if $data02 != 60000 then
# return -1 return -1
#endi endi
#print =============== step5 #print =============== step5
#$cc = 40 * 60000 #$cc = 40 * 60000

1
tests/script/tsim/testCaseSuite.sim
View File

@ -1,6 +1,7 @@
run tsim/user/basic1.sim run tsim/user/basic1.sim
run tsim/db/alter_option.sim
run tsim/db/basic1.sim run tsim/db/basic1.sim
run tsim/db/basic2.sim run tsim/db/basic2.sim
run tsim/db/basic3.sim run tsim/db/basic3.sim

224
tests/script/tsim/tmq/multiTopic.sim Normal file
View File

@ -0,0 +1,224 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#
######## ######## ######## ######## ######## ######## ######## ######## ######## ########
######## This test case include scene2 and scene4
######## ######## ######## ######## ######## ######## ######## ######## ######## ########
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
$loop_cnt = 0
$vgroups = 1
$dbNamme = d0
loop_vgroups:
print =============== create database $dbNamme vgroups $vgroups
sql create database $dbNamme vgroups $vgroups
sql show databases
print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
if $loop_cnt == 0 then
if $rows != 2 then
return -1
endi
if $data02 != 1 then # vgroups
print vgroups: $data02
return -1
endi
else
if $rows != 3 then
return -1
endi
if $data00 == d1 then
if $data02 != 4 then # vgroups
print vgroups: $data02
return -1
endi
else
if $data12 != 4 then # vgroups
print vgroups: $data12
return -1
endi
endi
endi
sql use $dbNamme
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
$tbPrefix = ct
$tbNum = 100
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using stb tags( $i )
$i = $i + 1
endw
print =============== create normal table
sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
print =============== create multi topics. notes: now only support:
print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
print =============== will support: * from stb
sql create topic topic_stb_column as select ts, c1, c3 from stb
#sql create topic topic_stb_all as select * from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
sql create topic topic_ctb_column as select ts, c1, c3 from ct0
sql create topic topic_ctb_all as select * from ct0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
sql create topic topic_ntb_column as select ts, c1, c3 from ntb
sql create topic topic_ntb_all as select * from ntb
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
sql show tables
if $rows != 101 then
return -1
endi
print =============== insert data
$rowNum = 100
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$x = 0
while $x < $rowNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$binary = ' . binary
$binary = $binary . $c
$binary = $binary . '
sql insert into $tb values ($tstart , $c , $x , $binary )
sql insert into ntb values ($tstart , $c , $x , $binary )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
#root@trd02 /home $ tmq_sim --help
# -c Configuration directory, default is
# -d The name of the database for cosumer, no default
# -t The topic string for cosumer, no default
# -k The key-value string for cosumer, no default
# -g showMsgFlag, default is 0
#
$totalMsgCnt = $rowNum * $tbNum
print inserted totalMsgCnt: $totalMsgCnt
# supported key:
# group.id:<xxx>
# enable.auto.commit:<true | false>
# auto.offset.reset:<earliest | latest | none>
# td.connect.ip:<fqdn | ipaddress>
# td.connect.user:root
# td.connect.pass:taosdata
# td.connect.port:6030
# td.connect.db:db
$numOfTopics = 2
$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics
$expect_result = @{consume success: @
$expect_result = $expect_result . $totalMsgCntOfmultiTopics
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2"
#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2"
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 20000, 0}@ then
if $system_content != $expect_result then
return -1
endi
$numOfTopics = 3
$totalMsgCntOfmultiTopics = $rowNum * $numOfTopics
$expect_result = @{consume success: @
$expect_result = $expect_result . $totalMsgCntOfmultiTopics
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 300, 0}@ then
if $system_content != $expect_result then
return -1
endi
$numOfTopics = 3
$totalMsgCntOfmultiTopics = $totalMsgCnt * $numOfTopics
$expect_result = @{consume success: @
$expect_result = $expect_result . $totalMsgCntOfmultiTopics
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 30000, 0}@ then
if $system_content != $expect_result then
return -1
endi
if $loop_cnt == 0 then
$loop_cnt = 1
$vgroups = 4
$dbNamme = d1
goto loop_vgroups
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

264
tests/script/tsim/tmq/oneTopic.sim Normal file
View File

@ -0,0 +1,264 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#
######## ######## ######## ######## ######## ######## ######## ######## ######## ########
######## This test case include scene1 and scene3
######## ######## ######## ######## ######## ######## ######## ######## ######## ########
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
$loop_cnt = 0
$vgroups = 1
$dbNamme = d0
loop_vgroups:
print =============== create database $dbNamme vgroups $vgroups
sql create database $dbNamme vgroups $vgroups
sql show databases
print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
if $loop_cnt == 0 then
if $rows != 2 then
return -1
endi
if $data02 != 1 then # vgroups
print vgroups: $data02
return -1
endi
else
if $rows != 3 then
return -1
endi
if $data00 == d1 then
if $data02 != 4 then # vgroups
print vgroups: $data02
return -1
endi
else
if $data12 != 4 then # vgroups
print vgroups: $data12
return -1
endi
endi
endi
sql use $dbNamme
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
$tbPrefix = ct
$tbNum = 100
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
sql create table $tb using stb tags( $i )
$i = $i + 1
endw
print =============== create normal table
sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
print =============== create multi topics. notes: now only support:
print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb
print =============== will support: * from stb
sql create topic topic_stb_column as select ts, c1, c3 from stb
#sql create topic topic_stb_all as select * from stb
sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
sql create topic topic_ctb_column as select ts, c1, c3 from ct0
sql create topic topic_ctb_all as select * from ct0
sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
sql create topic topic_ntb_column as select ts, c1, c3 from ntb
sql create topic topic_ntb_all as select * from ntb
sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
sql show tables
if $rows != 101 then
return -1
endi
print =============== insert data
$rowNum = 100
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$x = 0
while $x < $rowNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$binary = ' . binary
$binary = $binary . $c
$binary = $binary . '
sql insert into $tb values ($tstart , $c , $x , $binary )
sql insert into ntb values ($tstart , $c , $x , $binary )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
#root@trd02 /home $ tmq_sim --help
# -c Configuration directory, default is
# -d The name of the database for cosumer, no default
# -t The topic string for cosumer, no default
# -k The key-value string for cosumer, no default
# -g showMsgFlag, default is 0
#
$totalMsgCnt = $rowNum * $tbNum
print inserted totalMsgCnt: $totalMsgCnt
# supported key:
# group.id:<xxx>
# enable.auto.commit:<true | false>
# auto.offset.reset:<earliest | latest | none>
# td.connect.ip:<fqdn | ipaddress>
# td.connect.user:root
# td.connect.pass:taosdata
# td.connect.port:6030
# td.connect.db:db
$expect_result = @{consume success: @
$expect_result = $expect_result . $totalMsgCnt
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 10000, 0}@ then
if $system_content != $expect_result then
return -1
endi
#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#print cmd result----> $system_content
##if $system_content != @{consume success: 10000, 0}@ then
#if $system_content != $expect_result then
# return -1
#endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 10000, 0}@ then
if $system_content != $expect_result then
return -1
endi
$expect_result = @{consume success: @
$expect_result = $expect_result . $rowNum
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 100, 0}@ then
if $system_content != $expect_result then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 100, 0}@ then
if $system_content != $expect_result then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 100, 0}@ then
if $system_content != $expect_result then
return -1
endi
$expect_result = @{consume success: @
$expect_result = $expect_result . $totalMsgCnt
$expect_result = $expect_result . @, @
$expect_result = $expect_result . 0}
print expect_result----> $expect_result
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 10000, 0}@ then
if $system_content != $expect_result then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 10000, 0}@ then
if $system_content != $expect_result then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
print cmd result----> $system_content
#if $system_content != @{consume success: 10000, 0}@ then
if $system_content != $expect_result then
return -1
endi
if $loop_cnt == 0 then
$loop_cnt = 1
$vgroups = 4
$dbNamme = d1
goto loop_vgroups
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

2
tests/test/c/tmqSim.c
View File

@ -226,7 +226,7 @@ void loop_consume(tmq_t* tmq) {
int32_t totalRows = 0; int32_t totalRows = 0;
int32_t skipLogNum = 0; int32_t skipLogNum = 0;
while (running) { while (running) {
tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 6000); tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 3000);
if (tmqMsg) { if (tmqMsg) {
totalMsgs++; totalMsgs++;