xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-01-26 08:51:09 +08:00
parent 09b86cbf76 36cbe6ad34
commit 3aa92dec71
107 changed files with 3917 additions and 2528 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
2.0/src/query/src/queryMain.c
View File

@ -461,7 +461,7 @@ int32_t qKillQuery(qinfo_t qinfo) {
}
qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId);
setQueryKilled(pQInfo);
setTaskKilled(pQInfo);
// Wait for the query executing thread being stopped/
// Once the query is stopped, the owner of qHandle will be cleared immediately.
@ -634,7 +634,7 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId);
setQueryKilled(pQInfo);
setTaskKilled(pQInfo);
// wait query stop
int32_t loop = 0;

146
include/common/tmsg.h
View File

@ -107,7 +107,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_CLUSTER,
TSDB_MGMT_TABLE_STREAMTABLES,
TSDB_MGMT_TABLE_TP,
TSDB_MGMT_TABLE_FUNCTION,
TSDB_MGMT_TABLE_FUNC,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -533,20 +533,21 @@ typedef struct {
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int8_t igExists;
int8_t funcType;
int8_t scriptType;
int8_t align;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
int64_t sigature;
int64_t signature;
int32_t commentSize;
int32_t codeSize;
char pCont[];
} SCreateFuncReq;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
char name[TSDB_FUNC_NAME_LEN];
int8_t igNotExists;
} SDropFuncReq;
typedef struct {
@ -556,13 +557,13 @@ typedef struct {
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int8_t align;
int8_t funcType;
int8_t scriptType;
int8_t align;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
int64_t sigature;
int64_t signature;
int32_t commentSize;
int32_t codeSize;
char pCont[];
@ -1247,6 +1248,12 @@ typedef struct {
char data[];
} SVShowTablesFetchRsp;
typedef struct SMqCMGetSubEpReq {
int64_t consumerId;
int32_t epoch;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
} SMqCMGetSubEpReq;
#pragma pack(pop)
static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
@ -1521,15 +1528,17 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
}
typedef struct SMqSetCVgReq {
int32_t vgId;
int64_t oldConsumerId;
int64_t newConsumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
SSubQueryMsg msg;
int32_t vgId;
int64_t oldConsumerId;
int64_t newConsumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN];
char* sql;
char* logicalPlan;
char* physicalPlan;
uint32_t qmsgLen;
void* qmsg;
//SSubQueryMsg msg;
} SMqSetCVgReq;
static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) {
@ -1538,7 +1547,7 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg*
tlen += taosEncodeFixedU64(buf, pMsg->queryId);
tlen += taosEncodeFixedU64(buf, pMsg->taskId);
tlen += taosEncodeFixedU32(buf, pMsg->contentLen);
tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen);
//tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen);
return tlen;
}
@ -1547,7 +1556,7 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) {
buf = taosDecodeFixedU64(buf, &pMsg->queryId);
buf = taosDecodeFixedU64(buf, &pMsg->taskId);
buf = taosDecodeFixedU32(buf, &pMsg->contentLen);
buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen);
//buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen);
return buf;
}
@ -1561,7 +1570,9 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->logicalPlan);
tlen += taosEncodeString(buf, pReq->physicalPlan);
tlen += tEncodeSSubQueryMsg(buf, &pReq->msg);
tlen += taosEncodeFixedU32(buf, pReq->qmsgLen);
tlen += taosEncodeString(buf, (char*)pReq->qmsg);
//tlen += tEncodeSSubQueryMsg(buf, &pReq->msg);
return tlen;
}
@ -1574,15 +1585,18 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeString(buf, &pReq->sql);
buf = taosDecodeString(buf, &pReq->logicalPlan);
buf = taosDecodeString(buf, &pReq->physicalPlan);
buf = tDecodeSSubQueryMsg(buf, &pReq->msg);
buf = taosDecodeFixedU32(buf, &pReq->qmsgLen);
buf = taosDecodeString(buf, (char**)&pReq->qmsg);
//buf = tDecodeSSubQueryMsg(buf, &pReq->msg);
return buf;
}
typedef struct SMqSetCVgRsp {
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cGroup[TSDB_CONSUMER_GROUP_LEN];
SMsgHead header;
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqSetCVgRsp;
typedef struct SMqColData {
@ -1634,6 +1648,92 @@ typedef struct SMqConsumeReq {
char topic[TSDB_TOPIC_FNAME_LEN];
} SMqConsumeReq;
typedef struct SMqSubVgEp {
int32_t vgId;
SEpSet epSet;
} SMqSubVgEp;
typedef struct SMqSubTopicEp {
char topic[TSDB_TOPIC_FNAME_LEN];
SArray* vgs; // SArray<SMqSubVgEp>
} SMqSubTopicEp;
typedef struct SMqCMGetSubEpRsp {
int64_t consumerId;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp;
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI16(buf, pVgEp->vgId);
tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
return buf;
}
static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopicEp->topic);
int32_t sz = taosArrayGetSize(pTopicEp->vgs);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i);
tlen += tEncodeSMqSubVgEp(buf, pVgEp);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) {
buf = taosDecodeStringTo(buf, pTopicEp->topic);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp));
if (pTopicEp->vgs == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqSubVgEp vgEp;
buf = tDecodeSMqSubVgEp(buf, &vgEp);
taosArrayPush(pTopicEp->vgs, &vgEp);
}
return buf;
}
static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeString(buf, pRsp->cgroup);
int32_t sz = taosArrayGetSize(pRsp->topics);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i);
tlen += tEncodeSMqSubTopicEp(buf, pVgEp);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeStringTo(buf, pRsp->cgroup);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
if (pRsp->topics == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqSubTopicEp topicEp;
buf = tDecodeSMqSubTopicEp(buf, &topicEp);
taosArrayPush(pRsp->topics, &topicEp);
}
return buf;
}
#ifdef __cplusplus
}

7
include/common/tmsgdef.h
View File

@ -119,9 +119,9 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNCTION, "mnode-create-function", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNCTION, "mnode-retrieve-function", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNCTION, "mnode-drop-function", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "mnode-create-func", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "mnode-retrieve-func", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "mnode-drop-func", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL)
@ -140,6 +140,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-timer", SMqTmrMsg, SMqTmrMsg)
// Requests handled by VNODE

4
include/libs/executor/executor.h
View File

@ -32,7 +32,7 @@ struct SSubplan;
* @param streamReadHandle
* @return
*/
qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle);
qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle);
int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input);
@ -45,7 +45,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input);
* @param qId
* @return
*/
int32_t qCreateExecTask(void* readHandle, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
/**
* The main task execution function, including query on both table and multiple tables,

1
include/libs/parser/parser.h
View File

@ -74,6 +74,7 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid);
void columnListDestroy(SArray* pColumnList);
void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel);
void dropOneLevelExprInfo(SArray* pExprInfo);
typedef struct SSourceParam {
SArray *pExprNodeList; //Array<struct tExprNode*>

1
include/os/os.h
View File

@ -51,6 +51,7 @@ extern "C" {
#include <libgen.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include "osAtomic.h"
#include "osDef.h"

3
include/os/osDef.h
View File

@ -181,7 +181,8 @@ extern "C" {
#endif
#else
// Windows
#define setThreadName(name)
// #define setThreadName(name)
#define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0)
#endif
#if defined(_WIN32)

1
include/util/taoserror.h
View File

@ -239,6 +239,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4)
#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5)
#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6)
#define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x03C7)
// mnode-trans
#define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0)

1
include/util/tdef.h
View File

@ -173,6 +173,7 @@ do { \
#define TSDB_FUNC_BUF_SIZE 512
#define TSDB_FUNC_TYPE_SCALAR 1
#define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_MAX_RETRIEVE 1024
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)

64
include/util/tlist.h
View File

@ -34,6 +34,7 @@ extern "C" {
#define TD_SLIST_HEAD(sl) ((sl)->sl_head_)
#define TD_SLIST_NELES(sl) ((sl)->sl_neles_)
#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_)
#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_)
#define TD_SLIST_INIT(sl) \
do { \
@ -48,12 +49,25 @@ extern "C" {
TD_SLIST_NELES(sl) += 1; \
} while (0)
#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field) \
do { \
TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \
TD_SLIST_HEAD(sl) = (sln); \
TD_SLIST_NELES(sl) += 1; \
} while (0)
#define TD_SLIST_POP(sl) \
do { \
TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \
TD_SLIST_NELES(sl) -= 1; \
} while (0)
#define TD_SLIST_POP_WITH_FIELD(sl, field) \
do { \
TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \
TD_SLIST_NELES(sl) -= 1; \
} while (0)
// Double linked list ================
#define TD_DLIST_NODE(TYPE) \
struct { \
@ -70,6 +84,8 @@ extern "C" {
#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_)
#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_)
#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_)
#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_)
#define TD_DLIST_HEAD(dl) ((dl)->dl_head_)
#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_)
#define TD_DLIST_NELES(dl) ((dl)->dl_neles_)
@ -94,6 +110,20 @@ extern "C" {
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field) \
do { \
if (TD_DLIST_HEAD(dl) == NULL) { \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \
} else { \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl); \
TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln); \
TD_DLIST_TAIL(dl) = (dln); \
} \
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define TD_DLIST_PREPEND(dl, dln) \
do { \
if (TD_DLIST_HEAD(dl) == NULL) { \
@ -108,6 +138,20 @@ extern "C" {
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field) \
do { \
if (TD_DLIST_HEAD(dl) == NULL) { \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \
} else { \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL; \
TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl); \
TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln); \
TD_DLIST_HEAD(dl) = (dln); \
} \
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define TD_DLIST_POP(dl, dln) \
do { \
if (TD_DLIST_HEAD(dl) == (dln)) { \
@ -126,6 +170,26 @@ extern "C" {
TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
} while (0)
#define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \
do { \
if (TD_DLIST_HEAD(dl) == (dln)) { \
TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \
} \
if (TD_DLIST_TAIL(dl) == (dln)) { \
TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \
} \
if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) { \
TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) = \
TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \
} \
if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) { \
TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) = \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \
} \
TD_DLIST_NELES(dl) -= 1; \
TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \
} while (0)
// General double linked list
typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T;

16
include/util/tqueue.h
View File

@ -15,6 +15,7 @@
#ifndef _TD_UTIL_QUEUE_H
#define _TD_UTIL_QUEUE_H
#include "os.h"
#ifdef __cplusplus
extern "C" {
@ -40,13 +41,13 @@ shall be used to set up the protection.
typedef struct STaosQueue STaosQueue;
typedef struct STaosQset STaosQset;
typedef struct STaosQall STaosQall;
typedef void (*FProcessItem)(void *ahandle, void *pItem);
typedef void (*FProcessItems)(void *ahandle, STaosQall *qall, int32_t numOfItems);
typedef void (*FItem)(void *ahandle, void *pItem);
typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems);
STaosQueue *taosOpenQueue();
void taosCloseQueue(STaosQueue *queue);
void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp);
void *taosAllocateQitem(int32_t size);
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
void * taosAllocateQitem(int32_t size);
void taosFreeQitem(void *pItem);
int32_t taosWriteQitem(STaosQueue *queue, void *pItem);
int32_t taosReadQitem(STaosQueue *queue, void **ppItem);
@ -66,8 +67,11 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle);
void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue);
int32_t taosGetQueueNumber(STaosQset *qset);
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp);
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp);
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp);
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId);
void taosResetQsetThread(STaosQset *qset, void *pItem);
int32_t taosGetQueueItemsNumber(STaosQueue *queue);
int32_t taosGetQsetItemsNumber(STaosQset *qset);

64
include/util/tworker.h
View File

@ -15,57 +15,61 @@
#ifndef _TD_UTIL_WORKER_H
#define _TD_UTIL_WORKER_H
#include "tqueue.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SWorkerPool SWorkerPool;
typedef struct SMWorkerPool SMWorkerPool;
typedef struct SQWorkerPool SQWorkerPool;
typedef struct SWWorkerPool SWWorkerPool;
typedef struct SWorker {
int32_t id; // worker ID
pthread_t thread; // thread
SWorkerPool *pool;
} SWorker;
typedef struct SQWorker {
int32_t id; // worker ID
pthread_t thread; // thread
SQWorkerPool *pool;
} SQWorker, SFWorker;
typedef struct SWorkerPool {
typedef struct SQWorkerPool {
int32_t max; // max number of workers
int32_t min; // min number of workers
int32_t num; // current number of workers
STaosQset *qset;
const char *name;
SWorker *workers;
STaosQset * qset;
const char * name;
SQWorker * workers;
pthread_mutex_t mutex;
} SWorkerPool;
} SQWorkerPool, SFWorkerPool;
typedef struct SMWorker {
typedef struct SWWorker {
int32_t id; // worker id
pthread_t thread; // thread
STaosQall *qall;
STaosQset *qset; // queue set
SMWorkerPool *pool;
} SMWorker;
STaosQall * qall;
STaosQset * qset; // queue set
SWWorkerPool *pool;
} SWWorker;
typedef struct SMWorkerPool {
typedef struct SWWorkerPool {
int32_t max; // max number of workers
int32_t nextId; // from 0 to max-1, cyclic
const char *name;
SMWorker *workers;
const char * name;
SWWorker * workers;
pthread_mutex_t mutex;
} SMWorkerPool;
} SWWorkerPool;
int32_t tWorkerInit(SWorkerPool *pool);
void tWorkerCleanup(SWorkerPool *pool);
STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp);
void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue);
int32_t tQWorkerInit(SQWorkerPool *pool);
void tQWorkerCleanup(SQWorkerPool *pool);
STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp);
void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue);
int32_t tMWorkerInit(SMWorkerPool *pool);
void tMWorkerCleanup(SMWorkerPool *pool);
STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp);
void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue);
int32_t tFWorkerInit(SFWorkerPool *pool);
void tFWorkerCleanup(SFWorkerPool *pool);
STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp);
void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue);
int32_t tWWorkerInit(SWWorkerPool *pool);
void tWWorkerCleanup(SWWorkerPool *pool);
STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp);
void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue);
#ifdef __cplusplus
}

79
packaging/release.sh Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
#
# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
set -e
#set -x
# set parameters by default value
version="3.0.0.0"
curr_dir=$(pwd)
script_dir="$(dirname $(readlink -f $0))"
top_dir="$(readlink -f ${script_dir}/..)"
echo "=======================new version number: ${verNumber}======================================"
build_time=$(date +"%F %R")
echo "top_dir: ${top_dir}"
cd ${top_dir}
git pull || :
echo "curr_dir: ${curr_dir}"
# 2. cmake executable file
compile_dir="${top_dir}/debug"
if [ -d ${compile_dir} ]; then
rm -rf ${compile_dir}
fi
mkdir -p ${compile_dir}
cd ${compile_dir}
echo "compile_dir: ${compile_dir}"
cmake ..
make -j32
release_dir="${top_dir}/release"
if [ -d ${release_dir} ]; then
rm -rf ${release_dir}
fi
mkdir -p ${release_dir}
cd ${release_dir}
install_dir="${release_dir}/TDengine-server-${version}"
mkdir -p ${install_dir}
mkdir -p ${install_dir}/lib
bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table"
cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || :
cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/
cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/
cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/
pkg_name=${install_dir}-Linux-x64
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :

105
source/client/src/clientImpl.c
View File

@ -218,6 +218,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag,
if (pQueryNode->type == TSDB_SQL_SELECT) {
setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols);
tfree(pSchema);
pRequest->type = TDMT_VND_QUERY;
} else {
tfree(pSchema);
@ -262,7 +263,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList)
typedef struct SMqClientVg {
// statistics
int64_t consumeCnt;
int64_t pollCnt;
// offset
int64_t committedOffset;
int64_t currentOffset;
@ -313,10 +314,10 @@ tmq_conf_t* tmq_conf_new() {
}
int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) {
if (strcmp(key, "group.id")) {
if (strcmp(key, "group.id") == 0) {
strcpy(conf->groupId, value);
}
if (strcmp(key, "client.id")) {
if (strcmp(key, "client.id") == 0) {
strcpy(conf->clientId, value);
}
return 0;
@ -344,6 +345,7 @@ tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t err
strcpy(pTmq->groupId, conf->groupId);
pTmq->commit_cb = conf->commit_cb;
pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
return pTmq;
}
@ -364,7 +366,7 @@ tmq_list_t* tmq_list_new() {
int32_t tmq_list_append(tmq_list_t* ptr, char* src) {
if (ptr->cnt >= ptr->tot-1) return -1;
ptr->elems[ptr->cnt] = src;
ptr->elems[ptr->cnt] = strdup(src);
ptr->cnt++;
return 0;
}
@ -376,8 +378,23 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
int32_t sz = topic_list->cnt;
tmq->clientTopics = taosArrayInit(sz, sizeof(void*));
for (int i = 0; i < sz; i++) {
char* topicName = strdup(topic_list->elems[i]);
taosArrayPush(tmq->clientTopics, &topicName);
char* topicName = topic_list->elems[i];
SName name = {0};
char* dbName = getDbOfConnection(tmq->pTscObj);
tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName));
tNameFromString(&name, topicName, T_NAME_TABLE);
char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN);
if (topicFname == NULL) {
}
tNameExtractFullName(&name, topicFname);
tscDebug("subscribe topic: %s", topicFname);
taosArrayPush(tmq->clientTopics, &topicFname);
/*SMqClientTopic topic = {*/
/*.*/
/*};*/
}
SCMSubscribeReq req;
req.topicNum = taosArrayGetSize(tmq->clientTopics);
@ -395,13 +412,12 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tSerializeSCMSubscribeReq(&abuf, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TSDB_SQL_SELECT);
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE);
if (pRequest == NULL) {
tscError("failed to malloc sqlObj");
}
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
@ -529,7 +545,7 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql,
SCMCreateTopicReq req = {
.name = (char*) topicFname,
.igExists = 0,
.igExists = 1,
.physicalPlan = (char*) pStr,
.sql = (char*) sql,
.logicalPlan = "no logic plan",
@ -575,6 +591,37 @@ struct tmq_message_t {
SMqConsumeRsp rsp;
};
int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) {
return 0;
}
int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = (tmq_t*)param;
SMqCMGetSubEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp);
int32_t sz = taosArrayGetSize(rsp.topics);
// TODO: lock
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
for (int32_t i = 0; i < sz; i++) {
SMqClientTopic topic = {0};
SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i);
topic.topicName = strdup(pTopicEp->topic);
int32_t vgSz = taosArrayGetSize(pTopicEp->vgs);
topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg));
for (int32_t j = 0; j < vgSz; j++) {
SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
SMqClientVg clientVg = {
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet
};
taosArrayPush(topic.vgs, &clientVg);
}
taosArrayPush(tmq->clientTopics, &topic);
}
// unlock
return 0;
}
tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) {
return NULL;
@ -584,9 +631,38 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
req.reqType = 1;
req.blockingTime = blocking_time;
req.consumerId = tmq->consumerId;
tmq_message_t* tmq_message = NULL;
strcpy(req.cgroup, tmq->groupId);
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
if (taosArrayGetSize(tmq->clientTopics) == 0) {
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* buf = malloc(tlen);
if (buf == NULL) {
tscError("failed to malloc get subscribe ep buf");
}
buf->consumerId = htobe64(buf->consumerId);
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP);
if (pRequest == NULL) {
tscError("failed to malloc subscribe ep request");
}
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = tmq;
sendInfo->fp = tmq_ask_ep_cb;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
}
SMqClientTopic* pTopic = taosArrayGetP(tmq->clientTopics, tmq->nextTopicIdx);
tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
strcpy(req.topic, pTopic->topicName);
int32_t nextVgIdx = pTopic->nextVgIdx;
@ -597,14 +673,17 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) };
pRequest->type = TDMT_VND_CONSUME;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = &tmq_message;
sendInfo->fp = tmq_poll_cb_inner;
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, body);
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
return (tmq_message_t*)pRequest->body.resInfo.pData;
return tmq_message;
/*tsem_wait(&pRequest->body.rspSem);*/

1097
source/client/test/clientTests.cpp
View File

File diff suppressed because it is too large Load Diff

12
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -31,8 +31,8 @@ typedef struct {
SDnode *pDnode;
STaosQueue *queue;
union {
SWorkerPool pool;
SMWorkerPool mpool;
SQWorkerPool pool;
SWWorkerPool mpool;
};
} SDnodeWorker;
@ -109,10 +109,10 @@ typedef struct {
int32_t openVnodes;
int32_t totalVnodes;
SRWLatch latch;
SWorkerPool queryPool;
SWorkerPool fetchPool;
SMWorkerPool syncPool;
SMWorkerPool writePool;
SQWorkerPool queryPool;
SFWorkerPool fetchPool;
SWWorkerPool syncPool;
SWWorkerPool writePool;
} SVnodesMgmt;
typedef struct {

2
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -487,6 +487,8 @@ static void *dnodeThreadRoutine(void *param) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
int32_t ms = pDnode->cfg.statusInterval * 1000;
setThreadName("dnode-hb");
while (true) {
pthread_testcancel();
taosMsleep(ms);

58
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -25,9 +25,9 @@
#include "dndMnode.h"
#include "dndVnodes.h"
#define INTERNAL_USER "_internal"
#define INTERNAL_USER "_dnd"
#define INTERNAL_CKEY "_key"
#define INTERNAL_SECRET "_secret"
#define INTERNAL_SECRET "_pwd"
static void dndInitMsgFp(STransMgmt *pMgmt) {
// Requests handled by DNODE
@ -90,9 +90,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNCTION)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNCTION)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNCTION)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg;
@ -112,11 +112,15 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
/*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg;
// Requests handled by VNODE
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
@ -142,6 +146,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg;
}
static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
@ -152,17 +158,18 @@ static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
if (pRsp == NULL || pRsp->pCont == NULL) return;
dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pRsp->handle, TMSG_INFO(msgType));
dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
rpcFreeCont(pRsp->pCont);
return;
}
DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
if (fp != NULL) {
dTrace("RPC %p, rsp:%s will be processed, code:0x%x", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF);
dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF,
pRsp->ahandle);
(*fp)(pDnode, pRsp, pEpSet);
} else {
dError("RPC %p, rsp:%s not processed", pRsp->handle, TMSG_INFO(msgType));
dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle);
rpcFreeCont(pRsp->pCont);
}
}
@ -189,7 +196,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
pMgmt->clientRpc = rpcOpen(&rpcInit);
if (pMgmt->clientRpc == NULL) {
dError("failed to init rpc client");
dError("failed to init dnode rpc client");
return -1;
}
@ -212,40 +219,39 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
tmsg_t msgType = pReq->msgType;
if (msgType == TDMT_DND_NETWORK_TEST) {
dTrace("RPC %p, network test req, app:%p will be processed, code:0x%x", pReq->handle, pReq->ahandle, pReq->code);
dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle);
dndProcessStartupReq(pDnode, pReq);
return;
}
if (dndGetStat(pDnode) == DND_STAT_STOPPED) {
dError("RPC %p, req:%s app:%p is ignored since dnode exiting", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE};
dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle};
rpcSendResponse(&rspMsg);
rpcFreeCont(pReq->pCont);
return;
} else if (dndGetStat(pDnode) != DND_STAT_RUNNING) {
dError("RPC %p, req:%s app:%p is ignored since dnode not running", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY};
dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle};
rpcSendResponse(&rspMsg);
rpcFreeCont(pReq->pCont);
return;
}
if (pReq->pCont == NULL) {
dTrace("RPC %p, req:%s app:%p not processed since content is null", pReq->handle, TMSG_INFO(msgType),
pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN};
dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle};
rpcSendResponse(&rspMsg);
return;
}
DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)];
if (fp != NULL) {
dTrace("RPC %p, req:%s app:%p will be processed", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
(*fp)(pDnode, pReq, pEpSet);
} else {
dError("RPC %p, req:%s app:%p is not processed since no handle", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED};
dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle);
SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle};
rpcSendResponse(&rspMsg);
rpcFreeCont(pReq->pCont);
}
@ -285,24 +291,24 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
SDnode *pDnode = parent;
if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) {
dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) {
dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
if (terrno != TSDB_CODE_APP_NOT_READY) {
dTrace("failed to get user auth from internal mnode since %s", terrstr());
dTrace("failed to get user auth from mnode since %s", terrstr());
return -1;
}
SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq));
tstrncpy(pReq->user, user, TSDB_USER_LEN);
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH};
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};
SRpcMsg rpcRsp = {0};
dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt);
dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp);
@ -346,7 +352,7 @@ static int32_t dndInitServer(SDnode *pDnode) {
pMgmt->serverRpc = rpcOpen(&rpcInit);
if (pMgmt->serverRpc == NULL) {
dError("failed to init rpc server");
dError("failed to init dnode rpc server");
return -1;
}

78
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -253,7 +253,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
}
for (int32_t i = 0; i < vnodesNum; ++i) {
cJSON *vnode = cJSON_GetArrayItem(vnodes, i);
cJSON * vnode = cJSON_GetArrayItem(vnodes, i);
SWrapperCfg *pCfg = &pCfgs[i];
cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) {
dndReportStartup(pDnode, "open-vnodes", stepDesc);
SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId};
SVnode *pImpl = vnodeOpen(pCfg->path, &cfg);
SVnode * pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->failed++;
@ -737,15 +737,9 @@ int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
return 0;
}
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) {
SRpcMsg *pRsp = NULL;
vnodeProcessQueryReq(pVnode->pImpl, pMsg, &pRsp);
}
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessQueryMsg(pVnode->pImpl, pMsg); }
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) {
SRpcMsg *pRsp = NULL;
vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp);
}
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchMsg(pVnode->pImpl, pMsg); }
static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) {
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
@ -916,27 +910,27 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) {
int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1);
int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1);
SWorkerPool *pPool = &pMgmt->queryPool;
pPool->name = "vnode-query";
pPool->min = minQueryThreads;
pPool->max = maxQueryThreads;
if (tWorkerInit(pPool) != 0) return -1;
SQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query";
pQPool->min = minQueryThreads;
pQPool->max = maxQueryThreads;
if (tQWorkerInit(pQPool) != 0) return -1;
pPool = &pMgmt->fetchPool;
pPool->name = "vnode-fetch";
pPool->min = minFetchThreads;
pPool->max = maxFetchThreads;
if (tWorkerInit(pPool) != 0) return -1;
SFWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch";
pFPool->min = minFetchThreads;
pFPool->max = maxFetchThreads;
if (tFWorkerInit(pFPool) != 0) return -1;
SMWorkerPool *pMPool = &pMgmt->writePool;
pMPool->name = "vnode-write";
pMPool->max = maxWriteThreads;
if (tMWorkerInit(pMPool) != 0) return -1;
SWWorkerPool *pWPool = &pMgmt->writePool;
pWPool->name = "vnode-write";
pWPool->max = maxWriteThreads;
if (tWWorkerInit(pWPool) != 0) return -1;
pMPool = &pMgmt->syncPool;
pMPool->name = "vnode-sync";
pMPool->max = maxSyncThreads;
if (tMWorkerInit(pMPool) != 0) return -1;
pWPool = &pMgmt->syncPool;
pWPool->name = "vnode-sync";
pWPool->max = maxSyncThreads;
if (tWWorkerInit(pWPool) != 0) return -1;
dDebug("vnode workers is initialized");
return 0;
@ -944,21 +938,21 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) {
static void dndCleanupVnodeWorkers(SDnode *pDnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
tWorkerCleanup(&pMgmt->fetchPool);
tWorkerCleanup(&pMgmt->queryPool);
tMWorkerCleanup(&pMgmt->writePool);
tMWorkerCleanup(&pMgmt->syncPool);
tFWorkerCleanup(&pMgmt->fetchPool);
tQWorkerCleanup(&pMgmt->queryPool);
tWWorkerCleanup(&pMgmt->writePool);
tWWorkerCleanup(&pMgmt->syncPool);
dDebug("vnode workers is closed");
}
static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue);
pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue);
pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue);
pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue);
pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue);
pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue);
pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue);
pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue);
pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue);
pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue);
if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL ||
pVnode->pQueryQ == NULL) {
@ -971,11 +965,11 @@ static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) {
static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
pVnode->pWriteQ = NULL;
pVnode->pApplyQ = NULL;
pVnode->pSyncQ = NULL;

20
source/dnode/mgmt/impl/src/dndWorker.c
View File

@ -31,28 +31,28 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c
pWorker->pDnode = pDnode;
if (pWorker->type == DND_WORKER_SINGLE) {
SWorkerPool *pPool = &pWorker->pool;
SQWorkerPool *pPool = &pWorker->pool;
pPool->name = name;
pPool->min = minNum;
pPool->max = maxNum;
if (tWorkerInit(pPool) != 0) {
if (tQWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp);
pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else if (pWorker->type == DND_WORKER_MULTI) {
SMWorkerPool *pPool = &pWorker->mpool;
SWWorkerPool *pPool = &pWorker->mpool;
pPool->name = name;
pPool->max = maxNum;
if (tMWorkerInit(pPool) != 0) {
if (tWWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp);
pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -70,11 +70,11 @@ void dndCleanupWorker(SDnodeWorker *pWorker) {
}
if (pWorker->type == DND_WORKER_SINGLE) {
tWorkerCleanup(&pWorker->pool);
tWorkerFreeQueue(&pWorker->pool, pWorker->queue);
tQWorkerCleanup(&pWorker->pool);
tQWorkerFreeQueue(&pWorker->pool, pWorker->queue);
} else if (pWorker->type == DND_WORKER_MULTI) {
tMWorkerCleanup(&pWorker->mpool);
tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue);
tWWorkerCleanup(&pWorker->mpool);
tWWorkerFreeQueue(&pWorker->mpool, pWorker->queue);
} else {
}
}

9
source/dnode/mgmt/impl/test/sut/inc/sut.h
View File

@ -96,6 +96,15 @@ class Testbase {
#define CheckBinary(val, len) \
{ EXPECT_STREQ(test.GetShowBinary(len), val); }
#define CheckBinaryByte(b, len) \
{ \
char* bytes = (char*)calloc(1, len); \
for (int32_t i = 0; i < len - 1; ++i) { \
bytes[i] = b; \
} \
EXPECT_STREQ(test.GetShowBinary(len), bytes); \
}
#define CheckInt8(val) \
{ EXPECT_EQ(test.GetShowInt8(), val); }

2
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -25,7 +25,7 @@ extern "C" {
int32_t mndInitConsumer(SMnode *pMnode);
void mndCleanupConsumer(SMnode *pMnode);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);

23
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -314,7 +314,7 @@ typedef struct {
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
int64_t sigature;
int64_t signature;
int32_t commentSize;
int32_t codeSize;
char* pComment;
@ -363,8 +363,9 @@ typedef struct SMqConsumerEp {
int64_t consumerId; // -1 for unassigned
int64_t lastConsumerHbTs;
int64_t lastVgHbTs;
int32_t execLen;
SSubQueryMsg qExec;
uint32_t qmsgLen;
char* qmsg;
//SSubQueryMsg qExec;
} SMqConsumerEp;
static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
@ -373,7 +374,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon
tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
//tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen);
tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen);
return tlen;
}
@ -382,8 +385,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu
buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen;
//buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen);
buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen);
return buf;
}
@ -402,11 +406,12 @@ typedef struct SMqSubscribeObj {
static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj));
pSub->key[0] = 0;
pSub->epoch = 0;
if (pSub == NULL) {
return NULL;
}
pSub->key[0] = 0;
pSub->epoch = 0;
pSub->availConsumer = taosArrayInit(0, sizeof(int64_t));
if (pSub->availConsumer == NULL) {
free(pSub);
@ -433,7 +438,7 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
free(pSub);
return NULL;
}
return NULL;
return pSub;
}
static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) {

101
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -30,24 +30,23 @@
#define MND_CONSUMER_VER_NUMBER 1
#define MND_CONSUMER_RESERVE_SIZE 64
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg);
static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg);
static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter);
int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_CONSUMER,
.keyType = SDB_KEY_BINARY,
.keyType = SDB_KEY_INT64,
.encodeFp = (SdbEncodeFp)mndConsumerActionEncode,
.decodeFp = (SdbDecodeFp)mndConsumerActionDecode,
.insertFp = (SdbInsertFp)mndConsumerActionInsert,
.updateFp = (SdbUpdateFp)mndConsumerActionUpdate,
.deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
return sdbSetTable(pMnode->pSdb, table);
}
@ -56,46 +55,20 @@ void mndCleanupConsumer(SMnode *pMnode) {}
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer);
SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, tlen);
int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size);
if (pRaw == NULL) goto CM_ENCODE_OVER;
void* buf = malloc(tlen);
void *buf = malloc(tlen);
if (buf == NULL) goto CM_ENCODE_OVER;
void* abuf = buf;
void *abuf = buf;
tEncodeSMqConsumerObj(&abuf, pConsumer);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, CM_ENCODE_OVER);
#if 0
int32_t topicNum = taosArrayGetSize(pConsumer->topics);
SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER);
int32_t len = strlen(pConsumer->cgroup);
SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CM_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, topicNum, CM_ENCODE_OVER);
for (int i = 0; i < topicNum; i++) {
int32_t len;
SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i);
len = strlen(pConsumerTopic->name);
SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len, CM_ENCODE_OVER);
int vgSize;
if (pConsumerTopic->vgroups == NULL) {
vgSize = 0;
} else {
vgSize = listNEles(pConsumerTopic->vgroups);
}
SDB_SET_INT32(pRaw, dataPos, vgSize, CM_ENCODE_OVER);
for (int j = 0; j < vgSize; j++) {
// SList* head;
/*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/
}
}
#endif
SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER);
@ -116,53 +89,35 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CONSUME_DECODE_OVER;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER;
if (sver != MND_CONSUMER_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto CONSUME_DECODE_OVER;
goto CM_DECODE_OVER;
}
SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj));
if (pRow == NULL) goto CONSUME_DECODE_OVER;
if (pRow == NULL) goto CM_DECODE_OVER;
SMqConsumerObj *pConsumer = sdbGetRowObj(pRow);
if (pConsumer == NULL) goto CONSUME_DECODE_OVER;
if (pConsumer == NULL) goto CM_DECODE_OVER;
int32_t dataPos = 0;
int32_t len;
SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER);
void* buf = malloc(len);
if (buf == NULL) goto CONSUME_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, len, CONSUME_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER);
void *buf = malloc(len);
if (buf == NULL) goto CM_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER);
tDecodeSMqConsumerObj(buf, pConsumer);
SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER);
if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) {
goto CM_DECODE_OVER;
}
terrno = TSDB_CODE_SUCCESS;
#if 0
SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER);
for (int i = 0; i < topicNum; i++) {
int32_t topicLen;
SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic));
if (pConsumerTopic == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
// TODO
return NULL;
}
/*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/
SDB_GET_INT32(pRaw, dataPos, &topicLen, CONSUME_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER);
int32_t vgSize;
SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER);
}
#endif
CONSUME_DECODE_OVER:
if (terrno != 0) {
CM_DECODE_OVER:
if (terrno != TSDB_CODE_SUCCESS) {
mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
tfree(pRow);
return NULL;
@ -191,7 +146,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
return 0;
}
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) {
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) {
SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (pConsumer == NULL) {

309
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -25,14 +25,14 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc);
static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw);
static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc);
static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc);
static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc);
static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate);
static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc);
static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg);
static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg);
static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew);
static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate);
static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc);
static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq);
static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq);
static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq);
static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter);
int32_t mndInitFunc(SMnode *pMnode) {
@ -44,13 +44,13 @@ int32_t mndInitFunc(SMnode *pMnode) {
.updateFp = (SdbUpdateFp)mndFuncActionUpdate,
.deleteFp = (SdbDeleteFp)mndFuncActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNCTION, mndProcessCreateFuncMsg);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNCTION, mndProcessDropFuncMsg);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg);
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq);
mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNC, mndProcessRetrieveFuncReq);
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc);
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndGetFuncMeta);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndRetrieveFuncs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndCancelGetNextFunc);
return sdbSetTable(pMnode->pSdb, table);
}
@ -73,7 +73,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) {
SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, FUNC_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, FUNC_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, FUNC_ENCODE_OVER)
SDB_SET_INT64(pRaw, dataPos, pFunc->sigature, FUNC_ENCODE_OVER)
SDB_SET_INT64(pRaw, dataPos, pFunc->signature, FUNC_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, FUNC_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER)
@ -104,13 +104,11 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
goto FUNC_DECODE_OVER;
}
int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN;
SSdbRow *pRow = sdbAllocRow(size);
SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj));
if (pRow == NULL) goto FUNC_DECODE_OVER;
SFuncObj *pFunc = sdbGetRowObj(pRow);
if (pFunc == NULL) goto FUNC_DECODE_OVER;
char *tmp = (char *)pFunc + sizeof(SFuncObj);
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_DECODE_OVER)
@ -121,12 +119,18 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, FUNC_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, FUNC_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, FUNC_DECODE_OVER)
SDB_GET_INT64(pRaw, dataPos, &pFunc->sigature, FUNC_DECODE_OVER)
SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize, FUNC_DECODE_OVER)
pFunc->pComment = pFunc->pData;
pFunc->pCode = (pFunc->pData + pFunc->commentSize);
pFunc->pComment = calloc(1, pFunc->commentSize);
pFunc->pCode = calloc(1, pFunc->codeSize);
if (pFunc->pComment == NULL || pFunc->pCode == NULL) {
goto FUNC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER)
SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER)
terrno = 0;
@ -148,136 +152,136 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) {
static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) {
mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc);
tfree(pFunc->pCode);
tfree(pFunc->pComment);
return 0;
}
static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) {
mTrace("func:%s, perform update action, old row:%p new row:%p", pOldFunc->name, pOldFunc, pNewFunc);
static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) {
mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew);
return 0;
}
static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate) {
SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize);
pFunc->createdTime = taosGetTimestampMs();
pFunc->funcType = pCreate->funcType;
pFunc->scriptType = pCreate->scriptType;
pFunc->outputType = pCreate->outputType;
pFunc->outputLen = pCreate->outputLen;
pFunc->bufSize = pCreate->bufSize;
pFunc->sigature = pCreate->sigature;
pFunc->commentSize = pCreate->commentSize;
pFunc->codeSize = pCreate->codeSize;
pFunc->pComment = pFunc->pData;
memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize);
pFunc->pCode = pFunc->pData + pCreate->commentSize;
memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
free(pFunc);
mError("func:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) {
SSdb *pSdb = pMnode->pSdb;
SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName);
if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST;
}
return pFunc;
}
static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pFunc);
}
static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) {
int32_t code = -1;
STrans *pTrans = NULL;
SFuncObj func = {0};
memcpy(func.name, pCreate->name, TSDB_FUNC_NAME_LEN);
func.createdTime = taosGetTimestampMs();
func.funcType = pCreate->funcType;
func.scriptType = pCreate->scriptType;
func.outputType = pCreate->outputType;
func.outputLen = pCreate->outputLen;
func.bufSize = pCreate->bufSize;
func.signature = pCreate->signature;
func.commentSize = pCreate->commentSize;
func.codeSize = pCreate->codeSize;
func.pComment = malloc(func.commentSize);
func.pCode = malloc(func.codeSize);
if (func.pCode == NULL || func.pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto CREATE_FUNC_OVER;
}
memcpy(func.pComment, pCreate->pCont, pCreate->commentSize);
memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg);
if (pTrans == NULL) goto CREATE_FUNC_OVER;
mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
free(pFunc);
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
SSdbRaw *pRedoRaw = mndFuncActionEncode(&func);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto CREATE_FUNC_OVER;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto CREATE_FUNC_OVER;
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
free(pFunc);
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pUndoRaw = mndFuncActionEncode(&func);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto CREATE_FUNC_OVER;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto CREATE_FUNC_OVER;
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
free(pFunc);
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto CREATE_FUNC_OVER;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto CREATE_FUNC_OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_FUNC_OVER;
free(pFunc);
code = 0;
CREATE_FUNC_OVER:
free(func.pCode);
free(func.pComment);
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
mError("func:%s, failed to drop since %s", pFunc->name, terrstr());
return -1;
}
static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg);
if (pTrans == NULL) goto DROP_FUNC_OVER;
mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto DROP_FUNC_OVER;
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto DROP_FUNC_OVER;
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto DROP_FUNC_OVER;
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_FUNC_OVER;
code = 0;
DROP_FUNC_OVER:
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
SCreateFuncReq *pCreate = pMsg->rpcMsg.pCont;
SCreateFuncReq *pCreate = pReq->rpcMsg.pCont;
pCreate->outputLen = htonl(pCreate->outputLen);
pCreate->bufSize = htonl(pCreate->bufSize);
pCreate->sigature = htobe64(pCreate->sigature);
pCreate->signature = htobe64(pCreate->signature);
pCreate->commentSize = htonl(pCreate->commentSize);
pCreate->codeSize = htonl(pCreate->codeSize);
mDebug("func:%s, start to create", pCreate->name);
SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name);
SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name);
if (pFunc != NULL) {
sdbRelease(pMnode->pSdb, pFunc);
terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
mError("func:%s, failed to create since %s", pCreate->name, terrstr());
mndReleaseFunc(pMnode, pFunc);
if (pCreate->igExists) {
mDebug("stb:%s, already exist, ignore exist is set", pCreate->name);
return 0;
} else {
terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
mError("func:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
} else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) {
mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
@ -305,14 +309,13 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) {
return -1;
}
if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) {
if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
mError("func:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
int32_t code = mndCreateFunc(pMnode, pMsg, pCreate);
int32_t code = mndCreateFunc(pMnode, pReq, pCreate);
if (code != 0) {
mError("func:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
@ -321,9 +324,9 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SDropFuncReq *pDrop = pMsg->rpcMsg.pCont;
static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) {
SMnode *pMnode = pReq->pMnode;
SDropFuncReq *pDrop = pReq->rpcMsg.pCont;
mDebug("func:%s, start to drop", pDrop->name);
@ -333,14 +336,20 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) {
return -1;
}
SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name);
SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name);
if (pFunc == NULL) {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST;
mError("func:%s, failed to drop since %s", pDrop->name, terrstr());
return -1;
if (pDrop->igNotExists) {
mDebug("func:%s, not exist, ignore not exist is set", pDrop->name);
return 0;
} else {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST;
mError("func:%s, failed to drop since %s", pDrop->name, terrstr());
return -1;
}
}
int32_t code = mndDropFunc(pMnode, pMsg, pFunc);
int32_t code = mndDropFunc(pMnode, pReq, pFunc);
mndReleaseFunc(pMnode, pFunc);
if (code != 0) {
mError("func:%s, failed to drop since %s", pDrop->name, terrstr());
@ -350,15 +359,26 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) {
int32_t code = -1;
SMnode *pMnode = pReq->pMnode;
SRetrieveFuncReq *pRetrieve = pMsg->rpcMsg.pCont;
SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont;
pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs);
if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE;
return -1;
}
int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384;
int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN;
int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs;
SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size);
if (pRetrieveRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FUNC_RETRIEVE_OVER;
}
pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs);
char *pOutput = pRetrieveRsp->pFuncInfos;
@ -366,37 +386,42 @@ static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) {
char funcName[TSDB_FUNC_NAME_LEN] = {0};
memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN);
SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName);
SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
if (pFunc == NULL) {
terrno = TSDB_CODE_MND_INVALID_FUNC;
mError("func:%s, failed to retrieve since %s", funcName, terrstr());
return -1;
goto FUNC_RETRIEVE_OVER;
}
SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput;
strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN);
memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN);
pFuncInfo->funcType = pFunc->funcType;
pFuncInfo->scriptType = pFunc->scriptType;
pFuncInfo->outputType = pFunc->outputType;
pFuncInfo->outputLen = htonl(pFunc->outputLen);
pFuncInfo->bufSize = htonl(pFunc->bufSize);
pFuncInfo->sigature = htobe64(pFunc->sigature);
pFuncInfo->signature = htobe64(pFunc->signature);
pFuncInfo->commentSize = htonl(pFunc->commentSize);
pFuncInfo->codeSize = htonl(pFunc->codeSize);
memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize);
pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize;
memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize);
memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize);
pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize);
mndReleaseFunc(pMnode, pFunc);
}
pMsg->pCont = pRetrieveRsp;
pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp);
pReq->pCont = pRetrieveRsp;
pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp);
return 0;
code = 0;
FUNC_RETRIEVE_OVER:
if (code != 0) rpcFreeCont(pRetrieveRsp);
return code;
}
static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
SMnode *pMnode = pMsg->pMnode;
static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
SMnode *pMnode = pReq->pMnode;
SSdb *pSdb = pMnode->pSdb;
int32_t cols = 0;
@ -454,7 +479,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *p
pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbFname, "show funcs");
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
return 0;
}
@ -477,8 +502,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le
return tDataTypes[type].name;
}
static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pMsg->pMnode;
static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pReq->pMnode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SFuncObj *pFunc = NULL;

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -296,7 +296,7 @@ char *mndShowStr(int32_t showType) {
return "show streamtables";
case TSDB_MGMT_TABLE_TP:
return "show topics";
case TSDB_MGMT_TABLE_FUNCTION:
case TSDB_MGMT_TABLE_FUNC:
return "show functions";
default:
return "undefined";

14
source/dnode/mnode/impl/src/mndStb.c
View File

@ -123,8 +123,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
goto STB_DECODE_OVER;
}
int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema);
SSdbRow *pRow = sdbAllocRow(size);
SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj));
if (pRow == NULL) goto STB_DECODE_OVER;
SStbObj *pStb = sdbGetRowObj(pRow);
@ -143,6 +142,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int32_t totalCols = pStb->numOfColumns + pStb->numOfTags;
pStb->pSchema = calloc(totalCols, sizeof(SSchema));
if (pStb->pSchema == NULL) {
goto STB_DECODE_OVER;
}
for (int32_t i = 0; i < totalCols; ++i) {
SSchema *pSchema = &pStb->pSchema[i];
@ -448,7 +450,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr
stbObj.pSchema[i].colId = i + 1;
}
int32_t code = 0;
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg);
if (pTrans == NULL) goto CREATE_STB_OVER;
@ -481,7 +483,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) {
SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name);
if (pStb != NULL) {
sdbRelease(pMnode->pSdb, pStb);
mndReleaseStb(pMnode, pStb);
if (pCreate->igExists) {
mDebug("stb:%s, already exist, ignore exist is set", pCreate->name);
return 0;
@ -492,6 +494,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) {
}
} else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
// topic should have different name with stb
@ -640,7 +643,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) {
DROP_STB_OVER:
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) {
@ -665,7 +668,6 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) {
mndReleaseStb(pMnode, pStb);
if (code != 0) {
terrno = code;
mError("stb:%s, failed to drop since %s", pDrop->name, terrstr());
return -1;
}

192
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -30,6 +30,8 @@
#define MND_SUBSCRIBE_VER_NUMBER 1
#define MND_SUBSCRIBE_RESERVE_SIZE 64
static char *mndMakeSubscribeKey(char *cgroup, char *topicName);
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
@ -41,9 +43,10 @@ static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg);
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg);
static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg);
static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg);
static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer,
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic);
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub);
int32_t mndInitSubscribe(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
@ -55,11 +58,62 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndSubActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);
return sdbSetTable(pMnode->pSdb, table);
}
static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->pCont;
SMqCMGetSubEpRsp rsp;
int64_t consumerId = be64toh(pReq->consumerId);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId);
if (pConsumer == NULL) {
/*terrno = */
return -1;
}
ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
strcpy(rsp.cgroup, pReq->cgroup);
rsp.consumerId = consumerId;
SArray *pTopics = pConsumer->topics;
int32_t sz = taosArrayGetSize(pTopics);
rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
for (int32_t i = 0; i < sz; i++) {
SMqSubTopicEp topicEp;
SMqConsumerTopic *pConsumerTopic = taosArrayGet(pTopics, i);
strcpy(topicEp.topic, pConsumerTopic->name);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, pConsumerTopic->name);
int32_t assignedSz = taosArrayGetSize(pSub->assigned);
topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp));
for (int32_t j = 0; j < assignedSz; j++) {
SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
if (pCEp->consumerId == consumerId) {
taosArrayPush(pSub->assigned, pCEp);
}
}
if (taosArrayGetSize(topicEp.vgs) != 0) {
taosArrayPush(rsp.topics, &topicEp);
}
}
int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
void *buf = malloc(tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = buf;
tEncodeSMqCMGetSubEpRsp(&abuf, &rsp);
//TODO: free rsp
pMsg->pCont = buf;
pMsg->contLen = tlen;
return 0;
}
static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) {
int i = 0;
while (key[i] != ':') {
@ -96,25 +150,28 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer);
// build msg
SMqSetCVgReq req = {
.vgId = pCEp->vgId,
.oldConsumerId = -1,
.newConsumerId = consumerId,
};
strcpy(req.cgroup, cgroup);
strcpy(req.topicName, topic);
strcpy(req.sql, pTopic->sql);
strcpy(req.logicalPlan, pTopic->logicalPlan);
strcpy(req.physicalPlan, pTopic->physicalPlan);
memcpy(&req.msg, &pCEp->qExec, pCEp->execLen);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
SMqSetCVgReq *pReq = malloc(sizeof(SMqSetCVgReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
strcpy(pReq->cgroup, cgroup);
strcpy(pReq->topicName, topic);
pReq->sql = strdup(pTopic->sql);
pReq->logicalPlan = strdup(pTopic->logicalPlan);
pReq->physicalPlan = strdup(pTopic->physicalPlan);
pReq->qmsgLen = pCEp->qmsgLen;
/*memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen);*/
pReq->qmsg = strdup(pCEp->qmsg);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq);
void *reqStr = malloc(tlen);
if (reqStr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = reqStr;
tEncodeSMqSetCVgReq(abuf, &req);
tEncodeSMqSetCVgReq(&abuf, pReq);
// persist msg
STransAction action = {0};
@ -128,6 +185,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
SSdbRaw *pRaw = mndSubActionEncode(pSub);
mndTransAppendRedolog(pTrans, pRaw);
free(pReq);
tfree(topic);
tfree(cgroup);
}
@ -143,30 +201,48 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
}
static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) {
//convert phyplan to dag
// convert phyplan to dag
SQueryDag *pDag = qStringToDag(pTopic->physicalPlan);
SArray *pArray;
SArray *pArray;
SArray *inner = taosArrayGet(pDag->pSubplans, 0);
SSubplan *plan = taosArrayGetP(inner, 0);
plan->execNode.inUse = 0;
strcpy(plan->execNode.epAddr[0].fqdn, "localhost");
plan->execNode.epAddr[0].port = 6030;
plan->execNode.nodeId = 2;
plan->execNode.numOfEps = 1;
if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) {
return -1;
}
int32_t sz = taosArrayGetSize(pArray);
//convert dag to msg
// convert dag to msg
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp CEp;
CEp.status = 0;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
STaskInfo* pTaskInfo = taosArrayGet(pArray, i);
STaskInfo *pTaskInfo = taosArrayGet(pArray, i);
tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr);
/*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1],
* CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/
CEp.vgId = pTaskInfo->addr.nodeId;
CEp.qmsg = strdup(pTaskInfo->msg->msg);
CEp.qmsgLen = strlen(CEp.qmsg) + 1;
printf("abc:\n%s\n", CEp.qmsg);
/*CEp.qmsg = malloc(CEp.qmsgLen);*/
/*if (CEp.qmsg == NULL) {*/
/*return -1;*/
/*}*/
/*memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen);*/
taosArrayPush(unassignedVg, &CEp);
}
qDestroyQueryDag(pDag);
/*qDestroyQueryDag(pDag);*/
return 0;
}
static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer,
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic) {
SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp) {
int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
for (int32_t i = 0; i < sz; i++) {
int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i);
@ -178,27 +254,35 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume
};
strcpy(req.cgroup, pConsumer->cgroup);
strcpy(req.topicName, pTopic->name);
strcpy(req.sql, pTopic->sql);
strcpy(req.logicalPlan, pTopic->logicalPlan);
strcpy(req.physicalPlan, pTopic->physicalPlan);
req.sql = pTopic->sql;
req.logicalPlan = pTopic->logicalPlan;
req.physicalPlan = pTopic->physicalPlan;
req.qmsg = strdup(pCEp->qmsg);
req.qmsgLen = strlen(req.qmsg);
int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req);
void *reqStr = malloc(tlen);
if (reqStr == NULL) {
void *buf = malloc(sizeof(SMsgHead) + tlen);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
void *abuf = reqStr;
SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen);
pMsgHead->vgId = htonl(vgId);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncodeSMqSetCVgReq(&abuf, &req);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = reqStr;
action.contLen = tlen;
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + tlen;
action.msgType = TDMT_VND_MQ_SET_CONN;
mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(reqStr);
free(buf);
return -1;
}
}
@ -208,19 +292,18 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume
void mndCleanupSubscribe(SMnode *pMnode) {}
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t tlen = tEncodeSubscribeObj(NULL, pSub);
int32_t size = tlen + MND_SUBSCRIBE_RESERVE_SIZE;
int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size);
if (pRaw == NULL) goto SUB_ENCODE_OVER;
void *buf = malloc(tlen);
if (buf == NULL) {
goto SUB_ENCODE_OVER;
}
void *abuf = buf;
if (buf == NULL) goto SUB_ENCODE_OVER;
tEncodeSubscribeObj(&buf, pSub);
void *abuf = buf;
tEncodeSubscribeObj(&abuf, pSub);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER);
@ -228,6 +311,8 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
SDB_SET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, SUB_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
SUB_ENCODE_OVER:
if (terrno != 0) {
mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr());
@ -259,9 +344,9 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
int32_t dataPos = 0;
int32_t tlen;
void *buf = malloc(tlen + 1);
if (buf == NULL) goto SUB_DECODE_OVER;
SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER);
void *buf = malloc(tlen + 1);
if (buf == NULL) goto SUB_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER);
@ -269,8 +354,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
goto SUB_DECODE_OVER;
}
terrno = TSDB_CODE_SUCCESS;
SUB_DECODE_OVER:
if (terrno != 0) {
if (terrno != TSDB_CODE_SUCCESS) {
mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr());
// TODO free subscribeobj
tfree(pRow);
@ -379,10 +466,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name;
j++;
} else if (j >= oldTopicNum) {
newTopicName = taosArrayGet(newSub, i);
newTopicName = taosArrayGetP(newSub, i);
i++;
} else {
newTopicName = taosArrayGet(newSub, i);
newTopicName = taosArrayGetP(newSub, i);
oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name;
int comp = compareLenPrefixedStr(newTopicName, oldTopicName);
@ -466,9 +553,11 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
char *key = mndMakeSubscribeKey(consumerGroup, newTopicName);
strcpy(pSub->key, key);
// set unassigned vg
mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg);
//TODO: disable alter
// TODO: disable alter
}
taosArrayPush(pSub->availConsumer, &consumerId);
@ -477,16 +566,19 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) {
ASSERT(taosArrayGetSize(pConsumerTopic->pVgInfo) == 1);
int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo);
// send setmsg to vnode
if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) {
// TODO
return -1;
int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo);
SMqConsumerEp *pCEp = taosArrayGetLast(pSub->assigned);
if (pCEp->vgId == vgId) {
if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp) < 0) {
// TODO
return -1;
}
}
// send setmsg to vnode
}
SSdbRaw *pRaw = mndSubActionEncode(pSub);
/*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pRaw);
#if 0
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen);
@ -519,8 +611,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
mndTransAppendRedolog(pTrans, pTopicRaw);
#endif
mndReleaseTopic(pMnode, pTopic);
mndReleaseSubscribe(pMnode, pSub);
/*mndReleaseTopic(pMnode, pTopic);*/
/*mndReleaseSubscribe(pMnode, pSub);*/
}
}
// part3. persist consumerObj

17
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -60,7 +60,9 @@ void mndCleanupTopic(SMnode *pMnode) {}
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE;
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
@ -74,12 +76,10 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->logicalPlan)+1, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, logicalPlanLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER);
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
@ -135,7 +135,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
pTopic->physicalPlan = calloc(len + 1, sizeof(char));
@ -144,7 +144,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
@ -231,6 +231,7 @@ static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) {
}
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
mDebug("topic:%s to create", pCreate->name);
SMqTopicObj topicObj = {0};
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
@ -273,7 +274,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
return 0;
} else {
terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
mError("db:%s, failed to create since %s", createTopicReq.name, terrstr());
mError("topic:%s, failed to create since already exists", createTopicReq.name);
return -1;
}
}

8
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -390,9 +390,11 @@ static void mndTransDropActions(SArray *pArray) {
}
void mndTransDrop(STrans *pTrans) {
mndTransDropData(pTrans);
mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans);
if (pTrans != NULL) {
mndTransDropData(pTrans);
mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans);
}
}
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) {

1
source/dnode/mnode/impl/test/CMakeLists.txt
View File

@ -12,3 +12,4 @@ add_subdirectory(dnode)
add_subdirectory(mnode)
add_subdirectory(db)
add_subdirectory(stb)
add_subdirectory(func)

11
source/dnode/mnode/impl/test/func/CMakeLists.txt Normal file
View File

@ -0,0 +1,11 @@
aux_source_directory(. FUNC_SRC)
add_executable(mnode_test_func ${FUNC_SRC})
target_link_libraries(
mnode_test_func
PUBLIC sut
)
add_test(
NAME mnode_test_func
COMMAND mnode_test_func
)

521
source/dnode/mnode/impl/test/func/func.cpp Normal file
View File

@ -0,0 +1,521 @@
/**
* @file func.cpp
* @author slguan (slguan@taosdata.com)
* @brief MNODE module func tests
* @version 1.0
* @date 2022-01-24
*
* @copyright Copyright (c) 2022
*
*/
#include "sut.h"
class MndTestFunc : public ::testing::Test {
protected:
static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); }
static void TearDownTestSuite() { test.Cleanup(); }
static Testbase test;
public:
void SetUp() override {}
void TearDown() override {}
};
Testbase MndTestFunc::test;
TEST_F(MndTestFunc, 01_Show_Func) {
test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, "");
CHECK_META("show functions", 7);
CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE, "name");
CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, PATH_MAX + VARSTR_HEADER_SIZE, "comment");
CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "aggregate");
CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE, "outputtype");
CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time");
CHECK_SCHEMA(5, TSDB_DATA_TYPE_INT, 4, "code_len");
CHECK_SCHEMA(6, TSDB_DATA_TYPE_INT, 4, "bufsize");
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 0);
}
TEST_F(MndTestFunc, 02_Create_Func) {
{
int32_t contLen = sizeof(SCreateFuncReq);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME);
}
{
int32_t contLen = sizeof(SCreateFuncReq);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT);
}
{
int32_t contLen = sizeof(SCreateFuncReq);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT);
}
{
int32_t contLen = sizeof(SCreateFuncReq);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE);
}
{
int32_t contLen = sizeof(SCreateFuncReq);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN);
pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE);
}
{
int32_t contLen = sizeof(SCreateFuncReq) + 24;
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN);
pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN);
pReq->pCont[0] = 0;
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE);
}
{
int32_t contLen = sizeof(SCreateFuncReq) + 24;
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN);
pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN);
pReq->pCont[0] = 'a';
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE);
}
{
int32_t contLen = sizeof(SCreateFuncReq) + 24;
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN);
pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN);
pReq->pCont[0] = 'a';
pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE);
}
for (int32_t i = 0; i < 3; ++i) {
int32_t contLen = sizeof(SCreateFuncReq);
int32_t commentSize = TSDB_FUNC_COMMENT_LEN;
int32_t codeSize = TSDB_FUNC_CODE_LEN;
contLen = (contLen + codeSize + commentSize);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
pReq->igExists = 0;
if (i == 2) pReq->igExists = 1;
pReq->funcType = 1;
pReq->scriptType = 2;
pReq->outputType = TSDB_DATA_TYPE_SMALLINT;
pReq->outputLen = htonl(12);
pReq->bufSize = htonl(4);
pReq->signature = htobe64(5);
pReq->commentSize = htonl(commentSize);
pReq->codeSize = htonl(codeSize);
for (int32_t i = 0; i < commentSize - 1; ++i) {
pReq->pCont[i] = 'm';
}
for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) {
pReq->pCont[i] = 'd';
}
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
if (i == 0 || i == 2) {
ASSERT_EQ(pRsp->code, 0);
} else {
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_ALREADY_EXIST);
}
}
test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, "");
CHECK_META("show functions", 7);
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 1);
CheckBinary("f1", TSDB_FUNC_NAME_LEN);
CheckBinaryByte('m', TSDB_FUNC_COMMENT_LEN);
CheckInt32(0);
CheckBinary("SMALLINT", TSDB_TYPE_STR_MAX_LEN);
CheckTimestamp();
CheckInt32(TSDB_FUNC_CODE_LEN);
CheckInt32(4);
}
TEST_F(MndTestFunc, 03_Retrieve_Func) {
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 1;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(1);
strcpy(pReq->pFuncNames, "f1");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont;
pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs);
SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos);
pFuncInfo->outputLen = htonl(pFuncInfo->outputLen);
pFuncInfo->bufSize = htonl(pFuncInfo->bufSize);
pFuncInfo->signature = htobe64(pFuncInfo->signature);
pFuncInfo->commentSize = htonl(pFuncInfo->commentSize);
pFuncInfo->codeSize = htonl(pFuncInfo->codeSize);
EXPECT_STREQ(pFuncInfo->name, "f1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN);
EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN);
char* pComment = pFuncInfo->pCont;
char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize;
char comments[TSDB_FUNC_COMMENT_LEN] = {0};
for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) {
comments[i] = 'm';
}
char codes[TSDB_FUNC_CODE_LEN] = {0};
for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) {
codes[i] = 'd';
}
EXPECT_STREQ(pComment, comments);
EXPECT_STREQ(pCode, codes);
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 0;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(numOfFuncs);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE);
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(numOfFuncs);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE);
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 1;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(numOfFuncs);
strcpy(pReq->pFuncNames, "f2");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
}
{
int32_t contLen = sizeof(SCreateFuncReq);
int32_t commentSize = 1024;
int32_t codeSize = 9527;
contLen = (contLen + codeSize + commentSize);
SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f2");
pReq->igExists = 1;
pReq->funcType = 2;
pReq->scriptType = 3;
pReq->outputType = TSDB_DATA_TYPE_BINARY;
pReq->outputLen = htonl(24);
pReq->bufSize = htonl(6);
pReq->signature = htobe64(18);
pReq->commentSize = htonl(commentSize);
pReq->codeSize = htonl(codeSize);
for (int32_t i = 0; i < commentSize - 1; ++i) {
pReq->pCont[i] = 'p';
}
for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) {
pReq->pCont[i] = 'q';
}
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, "");
CHECK_META("show functions", 7);
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 2);
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 1;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(1);
strcpy(pReq->pFuncNames, "f2");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont;
pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs);
SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos);
pFuncInfo->outputLen = htonl(pFuncInfo->outputLen);
pFuncInfo->bufSize = htonl(pFuncInfo->bufSize);
pFuncInfo->signature = htobe64(pFuncInfo->signature);
pFuncInfo->commentSize = htonl(pFuncInfo->commentSize);
pFuncInfo->codeSize = htonl(pFuncInfo->codeSize);
EXPECT_STREQ(pFuncInfo->name, "f2");
EXPECT_EQ(pFuncInfo->funcType, 2);
EXPECT_EQ(pFuncInfo->scriptType, 3);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY);
EXPECT_EQ(pFuncInfo->outputLen, 24);
EXPECT_EQ(pFuncInfo->bufSize, 6);
EXPECT_EQ(pFuncInfo->signature, 18);
EXPECT_EQ(pFuncInfo->commentSize, 1024);
EXPECT_EQ(pFuncInfo->codeSize, 9527);
char* pComment = pFuncInfo->pCont;
char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize;
char* comments = (char*)calloc(1, 1024);
for (int32_t i = 0; i < 1024 - 1; ++i) {
comments[i] = 'p';
}
char* codes = (char*)calloc(1, 9527);
for (int32_t i = 0; i < 9527 - 1; ++i) {
codes[i] = 'q';
}
EXPECT_STREQ(pComment, comments);
EXPECT_STREQ(pCode, codes);
free(comments);
free(codes);
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 2;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(numOfFuncs);
strcpy(pReq->pFuncNames, "f2");
strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont;
pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs);
{
SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos);
pFuncInfo->outputLen = htonl(pFuncInfo->outputLen);
pFuncInfo->bufSize = htonl(pFuncInfo->bufSize);
pFuncInfo->signature = htobe64(pFuncInfo->signature);
pFuncInfo->commentSize = htonl(pFuncInfo->commentSize);
pFuncInfo->codeSize = htonl(pFuncInfo->codeSize);
EXPECT_STREQ(pFuncInfo->name, "f2");
EXPECT_EQ(pFuncInfo->funcType, 2);
EXPECT_EQ(pFuncInfo->scriptType, 3);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY);
EXPECT_EQ(pFuncInfo->outputLen, 24);
EXPECT_EQ(pFuncInfo->bufSize, 6);
EXPECT_EQ(pFuncInfo->signature, 18);
EXPECT_EQ(pFuncInfo->commentSize, 1024);
EXPECT_EQ(pFuncInfo->codeSize, 9527);
char* pComment = pFuncInfo->pCont;
char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize;
char* comments = (char*)calloc(1, 1024);
for (int32_t i = 0; i < 1024 - 1; ++i) {
comments[i] = 'p';
}
char* codes = (char*)calloc(1, 9527);
for (int32_t i = 0; i < 9527 - 1; ++i) {
codes[i] = 'q';
}
EXPECT_STREQ(pComment, comments);
EXPECT_STREQ(pCode, codes);
free(comments);
free(codes);
}
{
SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527);
pFuncInfo->outputLen = htonl(pFuncInfo->outputLen);
pFuncInfo->bufSize = htonl(pFuncInfo->bufSize);
pFuncInfo->signature = htobe64(pFuncInfo->signature);
pFuncInfo->commentSize = htonl(pFuncInfo->commentSize);
pFuncInfo->codeSize = htonl(pFuncInfo->codeSize);
EXPECT_STREQ(pFuncInfo->name, "f1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN);
EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN);
char* pComment = pFuncInfo->pCont;
char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize;
char comments[TSDB_FUNC_COMMENT_LEN] = {0};
for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) {
comments[i] = 'm';
}
char codes[TSDB_FUNC_CODE_LEN] = {0};
for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) {
codes[i] = 'd';
}
EXPECT_STREQ(pComment, comments);
EXPECT_STREQ(pCode, codes);
}
}
{
int32_t contLen = sizeof(SRetrieveFuncReq);
int32_t numOfFuncs = 2;
contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN);
SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen);
pReq->numOfFuncs = htonl(numOfFuncs);
strcpy(pReq->pFuncNames, "f2");
strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
}
}
TEST_F(MndTestFunc, 04_Drop_Func) {
{
int32_t contLen = sizeof(SDropFuncReq);
SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME);
}
{
int32_t contLen = sizeof(SDropFuncReq);
SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f3");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
}
{
int32_t contLen = sizeof(SDropFuncReq);
SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f3");
pReq->igNotExists = 1;
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
{
int32_t contLen = sizeof(SDropFuncReq);
SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen);
strcpy(pReq->name, "f1");
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, "");
CHECK_META("show functions", 7);
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 1);
// restart
test.Restart();
test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, "");
CHECK_META("show functions", 7);
test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 1);
CheckBinary("f2", TSDB_FUNC_NAME_LEN);
}

2
source/dnode/vnode/inc/tq.h
View File

@ -318,7 +318,7 @@ int tqRegisterContext(STqGroup*, void* ahandle);
int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
#endif
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp);
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
#ifdef __cplusplus

6
source/dnode/vnode/inc/tsdb.h
View File

@ -75,7 +75,6 @@ typedef struct STsdbQueryCond {
} STsdbQueryCond;
typedef struct {
void *pTable;
TSKEY lastKey;
uint64_t uid;
} STableKeyInfo;
@ -104,8 +103,7 @@ typedef void* tsdbReadHandleT;
* @param qinfo query info handle from query processor
* @return
*/
tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
void *pRef);
tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId);
/**
* Get the last row of the given query time window for all the tables in STableGroupInfo object.
@ -142,7 +140,7 @@ bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle);
*/
int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId);
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId);
/**
* get num of rows in mem table
*

6
source/dnode/vnode/inc/vnode.h
View File

@ -159,20 +159,18 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
/**
* @brief Process a fetch message.
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg);
/* ------------------------ SVnodeCfg ------------------------ */
/**

4
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -587,7 +587,11 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) {
pBuf = value.data;
metaDecodeTbInfo(pBuf, &tbCfg);
if (tbCfg.type == META_SUPER_TABLE) {
free(tbCfg.name);
free(tbCfg.stbCfg.pTagSchema);
continue;
} else if (tbCfg.type == META_CHILD_TABLE) {
kvRowFree(tbCfg.ctbCfg.pTag);
}
return tbCfg.name;
} else {

19
source/dnode/vnode/src/tq/tq.c
View File

@ -81,6 +81,10 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
}
void tqClose(STQ* pTq) {
if (pTq) {
tfree(pTq->path);
free(pTq);
}
// TODO
}
@ -663,8 +667,8 @@ int tqItemSSize() {
}
#endif
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
SMqConsumeReq* pReq = pMsg->pCont;
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
SRpcMsg rpcMsg;
int64_t reqId = pReq->reqId;
int64_t consumerId = pReq->consumerId;
@ -795,9 +799,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
return -1;
}
strcpy(pTopic->topicName, req.topicName);
strcpy(pTopic->sql, req.sql);
strcpy(pTopic->logicalPlan, req.logicalPlan);
strcpy(pTopic->physicalPlan, req.physicalPlan);
pTopic->sql = strdup(req.sql);
pTopic->logicalPlan = strdup(req.logicalPlan);
pTopic->physicalPlan = strdup(req.physicalPlan);
pTopic->buffer.firstOffset = -1;
pTopic->buffer.lastOffset = -1;
@ -807,9 +811,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.msg, pReadHandle);
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle);
}
taosArrayPush(pConsumer->topics, pTopic);
terrno = TSDB_CODE_SUCCESS;
return 0;
}
@ -822,7 +827,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
pReadHandle->pMsg = NULL;
pReadHandle->ver = -1;
pReadHandle->pColIdList = NULL;
return NULL;
return pReadHandle;
}
void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) {

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -359,7 +359,7 @@ void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) {
tsdbResetFSStatus(pfs->nstatus);
pfs->nstatus->meta = pfs->cstatus->meta;
// if (pfs->cstatus->pmf == NULL) {
pfs->nstatus->meta.version = 0;
pfs->nstatus->meta.version += 1;
// } else {
// pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1;
// }

262
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -81,7 +81,6 @@ enum {
CHECKINFO_CHOSEN_BOTH = 2 //for update=2(merge case)
};
typedef struct STableCheckInfo {
uint64_t tableId;
TSKEY lastKey;
@ -132,7 +131,7 @@ typedef struct STsdbReadHandle {
bool loadExternalRow; // load time window external data rows
bool currentLoadExternalRows; // current load external rows
int32_t loadType; // block load type
uint64_t qId; // query info handle, for debug purpose
char *idStr; // query info handle, for debug purpose
int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
SDFileSet* pFileGroup;
SFSIter fileIter;
@ -209,31 +208,6 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load
return pLocalIdList;
}
static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) {
// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL);
//
// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef;
// if (pTsdbReadHandle->pMemRef->ref++ == 0) {
// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable);
// }
//
// taosArrayDestroy(psTable);
}
static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) {
// assert(pTsdbReadHandle != NULL);
// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef;
// if (pMemRef == NULL) { // it has been freed
// return;
// }
//
// if (--pMemRef->ref == 0) {
// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot));
// }
//
// pTsdbReadHandle->pMemRef = NULL;
}
//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) {
// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
//
@ -263,9 +237,9 @@ static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) {
// return rows;
//}
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) {
size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
assert(sizeOfGroup >= 1);
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) {
size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
assert(numOfGroup >= 1);
// allocate buffer in order to load data blocks from file
SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo));
@ -273,14 +247,8 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
return NULL;
}
SArray* pTable = taosArrayInit(4, sizeof(STable*));
if (pTable == NULL) {
taosArrayDestroy(pTableCheckInfo);
return NULL;
}
// todo apply the lastkey of table check to avoid to load header file
for (int32_t i = 0; i < sizeOfGroup; ++i) {
for (int32_t i = 0; i < numOfGroup; ++i) {
SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i);
size_t gsize = taosArrayGetSize(group);
@ -289,12 +257,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
for (int32_t j = 0; j < gsize; ++j) {
STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j);
STableCheckInfo info = { .lastKey = pKeyInfo->lastKey};
// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
info.tableId = pKeyInfo->uid;
STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid};
if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) {
info.lastKey = pTsdbReadHandle->window.skey;
@ -306,17 +269,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
}
taosArrayPush(pTableCheckInfo, &info);
tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId);
tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" %s", pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->idStr);
}
}
// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
size_t gsize = taosArrayGetSize(pTableCheckInfo);
// for (int32_t i = 0; i < gsize; ++i) {
// STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i);
// }
*psTable = pTable;
// TODO group table according to the tag value.
taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar);
return pTableCheckInfo;
}
@ -389,13 +347,13 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond*
}
if (updateTs) {
tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64
", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey,
pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 ", %s",
pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey,
pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
}
}
static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) {
static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) {
STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle));
if (pReadHandle == NULL) {
goto _end;
@ -408,7 +366,6 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
pReadHandle->cur.win = TSWINDOW_INITIALIZER;
pReadHandle->checkFiles = true;
pReadHandle->activeIndex = 0; // current active table index
pReadHandle->qId = qId;
pReadHandle->allocSize = 0;
pReadHandle->locateStart = false;
pReadHandle->loadType = pCond->type;
@ -417,6 +374,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
pReadHandle->loadExternalRow = pCond->loadExternalRows;
pReadHandle->currentLoadExternalRows = pCond->loadExternalRows;
char buf[128] = {0};
snprintf(buf, tListLen(buf), "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, qId);
pReadHandle->idStr = strdup(buf);
if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) {
goto _end;
}
@ -455,7 +416,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock);
if (pReadHandle->pDataCols == NULL) {
tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId);
tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _end;
}
@ -471,8 +432,8 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
return NULL;
}
tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) {
STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef);
tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) {
STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId);
if (pTsdbReadHandle == NULL) {
return NULL;
}
@ -481,20 +442,17 @@ tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroup
return (tsdbReadHandleT*) pTsdbReadHandle;
}
SArray* psTable = NULL;
// todo apply the lastkey of table check to avoid to load header file
pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable);
pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList);
if (pTsdbReadHandle->pTableCheckInfo == NULL) {
// tsdbCleanupQueryHandle(pTsdbReadHandle);
taosArrayDestroy(psTable);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable);
tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %"PRIzu" %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo),
taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr);
tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId);
return (tsdbReadHandleT) pTsdbReadHandle;
}
@ -575,7 +533,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond
// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
}
tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) {
pCond->twindow = updateLastrowForEachGroup(groupList);
// no qualified table
@ -583,7 +541,7 @@ tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup
return NULL;
}
STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef);
STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, taskId);
if (pTsdbReadHandle == NULL) {
return NULL;
}
@ -666,7 +624,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
return pNew;
}
tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) {
tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) {
STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
if (pNew->numOfTables == 0) {
@ -681,7 +639,7 @@ tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond
}
}
STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef);
STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, taskId);
pTsdbReadHandle->loadExternalRow = true;
pTsdbReadHandle->currentLoadExternalRows = true;
@ -734,8 +692,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
TSKEY key = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId);
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s",
pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->idStr);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
@ -744,7 +702,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
}
} else {
tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId);
tsdbDebug("%p uid:%"PRId64", no data in mem, %s", pHandle, pCheckInfo->tableId, pHandle->idStr);
}
if (!imemEmpty) {
@ -754,8 +712,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
TSKEY key = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId);
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s",
pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->idStr);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
@ -763,7 +721,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
assert(pCheckInfo->lastKey >= key);
}
} else {
tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId);
tsdbDebug("%p uid:%"PRId64", no data in imem, %s", pHandle, pCheckInfo->tableId, pHandle->idStr);
}
return true;
@ -952,8 +910,8 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
}
pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle,
pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId);
tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, %s", pHandle,
pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->idStr);
// all data in mem are checked already.
if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) ||
@ -1138,21 +1096,21 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
STSchema *pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0);
int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error;
}
code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error;
}
code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _error;
}
@ -1188,15 +1146,15 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
int64_t elapsedTime = (taosGetTimestampUs() - st);
pTsdbReadHandle->cost.blockLoadTime += elapsedTime;
tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64,
pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId);
tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, %s",
pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->idStr);
return TSDB_CODE_SUCCESS;
_error:
pBlock->numOfRows = 0;
tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64,
pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId);
tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, %s",
pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->idStr);
return terrno;
}
@ -1219,9 +1177,9 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update);
if (key != TSKEY_INITIAL_VAL) {
tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId);
tsdbDebug("%p key in mem:%"PRId64", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr);
} else {
tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
tsdbDebug("%p no data in mem, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
}
if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
@ -1288,11 +1246,11 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
assert(cur->blockCompleted);
if (cur->rows == binfo.rows) {
tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64,
pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId);
tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %s",
pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->idStr);
} else {
tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64,
pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId);
tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %s",
pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->idStr);
}
}
@ -1815,8 +1773,8 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa
updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
doCheckGeneratedBlockRange(pTsdbReadHandle);
tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64,
pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId);
tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %s",
pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->idStr);
}
int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) {
@ -1868,9 +1826,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo);
tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d,"
"end:%d, 0x%"PRIx64,
"end:%d, %s",
pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey,
blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId);
blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr);
// compared with the data from in-memory buffer, to generate the correct timestamp array list
int32_t numOfRows = 0;
@ -2027,8 +1985,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos);
doCheckGeneratedBlockRange(pTsdbReadHandle);
tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64,
pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId);
tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %s",
pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->idStr);
}
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) {
@ -2202,13 +2160,13 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks);
cleanBlockOrderSupporter(&sup, numOfQualTables);
tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt,
pTsdbReadHandle->qId);
tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted %s", pTsdbReadHandle, cnt,
pTsdbReadHandle->idStr);
return TSDB_CODE_SUCCESS;
}
tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt,
numOfQualTables, pTsdbReadHandle->qId);
tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pTsdbReadHandle, cnt,
numOfQualTables, pTsdbReadHandle->idStr);
assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0
sup.numOfTables = numOfQualTables;
@ -2244,7 +2202,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
* }
*/
tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId);
tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr);
cleanBlockOrderSupporter(&sup, numOfTables);
free(pTree);
@ -2302,8 +2260,8 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi
if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
(!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle,
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle,
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
pTsdbReadHandle->pFileGroup = NULL;
assert(pTsdbReadHandle->numOfBlocks == 0);
break;
@ -2326,8 +2284,8 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi
break;
}
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables,
pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId);
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pTsdbReadHandle, numOfBlocks, numOfTables,
pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->idStr);
assert(numOfBlocks >= 0);
if (numOfBlocks == 0) {
@ -2420,8 +2378,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist*
if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
(!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle,
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId);
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle,
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
pTsdbReadHandle->pFileGroup = NULL;
break;
}
@ -2444,8 +2402,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist*
break;
}
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables,
pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId);
tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pTsdbReadHandle, numOfBlocks, numOfTables,
pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->idStr);
if (numOfBlocks == 0) {
continue;
@ -2499,8 +2457,8 @@ static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exis
if ((!cur->mixBlock) || cur->blockCompleted) {
// all data blocks in current file has been checked already, try next file if exists
} else {
tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos,
pTsdbReadHandle->qId);
tsdbDebug("%p continue in current data block, index:%d, pos:%d, %s", pTsdbReadHandle, cur->slot, cur->pos,
pTsdbReadHandle->idStr);
int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo);
*exists = (pTsdbReadHandle->realNumOfRows > 0);
@ -2624,8 +2582,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
}
int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle,
elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId);
tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %s", pTsdbReadHandle,
elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr);
return numOfRows;
}
@ -2639,7 +2597,7 @@ static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) {
break;
}
STableKeyInfo info = {.pTable = NULL, .lastKey = TSKEY_INITIAL_VAL, uid = id};
STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id};
taosArrayPush(list, &info);
}
@ -2937,7 +2895,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
if (emptyQueryTimewindow(pTsdbReadHandle)) {
tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId);
tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
return false;
}
@ -3052,7 +3010,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) {
// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
// }
//
// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef);
// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->idStr, pMemRef);
// tfree(cond.colList);
//
// // current table, only one table
@ -3230,7 +3188,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) {
if (key < lastKey) {
key = lastKey;
keyInfo.pTable = pInfo->pTable;
// keyInfo.pTable = pInfo->pTable;
keyInfo.lastKey = key;
pInfo->lastKey = key;
@ -3244,29 +3202,19 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) {
}
}
// clear current group, unref unused table
for (int32_t i = 0; i < numOfTables; ++i) {
STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i);
// keyInfo.pTable may be NULL here.
if (pInfo->pTable != keyInfo.pTable) {
// tsdbUnRefTable(pInfo->pTable);
}
}
// more than one table in each group, only one table left for each group
if (keyInfo.pTable != NULL) {
totalNumOfTable++;
if (taosArrayGetSize(pGroup) == 1) {
// do nothing
} else {
taosArrayClear(pGroup);
taosArrayPush(pGroup, &keyInfo);
}
} else { // mark all the empty groups, and remove it later
taosArrayDestroy(pGroup);
taosArrayPush(emptyGroup, &j);
}
// if (keyInfo.pTable != NULL) {
// totalNumOfTable++;
// if (taosArrayGetSize(pGroup) == 1) {
// // do nothing
// } else {
// taosArrayClear(pGroup);
// taosArrayPush(pGroup, &keyInfo);
// }
// } else { // mark all the empty groups, and remove it later
// taosArrayDestroy(pGroup);
// taosArrayPush(emptyGroup, &j);
// }
}
// window does not being updated, so set the original
@ -3456,11 +3404,13 @@ void filterPrepare(void* expr, void* param) {
}
}
#endif
static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
#if 0
STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
STable* pTable1 = ((STableKeyInfo*) p1)->pTable;
STable* pTable2 = ((STableKeyInfo*) p2)->pTable;
STable* pTable1 = ((STableKeyInfo*) p1)->uid;
STable* pTable2 = ((STableKeyInfo*) p2)->uid;
for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
@ -3508,10 +3458,9 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa
return ret;
}
}
#endif
return 0;
}
#endif
static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) {
@ -3527,10 +3476,9 @@ static int tsdbCheckInfoCompar(const void* key1, const void* key2) {
void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey,
STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) {
STable* pTable = taosArrayGetP(pTableList, 0);
SArray* g = taosArrayInit(16, sizeof(STableKeyInfo));
STableKeyInfo info = {.pTable = pTable, .lastKey = skey};
STableKeyInfo info = {.lastKey = skey};
taosArrayPush(g, &info);
for (int32_t i = 1; i < numOfTables; ++i) {
@ -3541,13 +3489,13 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
assert(ret == 0 || ret == -1);
if (ret == 0) {
STableKeyInfo info1 = {.pTable = *p, .lastKey = skey};
STableKeyInfo info1 = {.lastKey = skey};
taosArrayPush(g, &info1);
} else {
taosArrayPush(pGroups, &g); // current group is ended, start a new group
g = taosArrayInit(16, sizeof(STableKeyInfo));
STableKeyInfo info1 = {.pTable = *p, .lastKey = skey};
STableKeyInfo info1 = {.lastKey = skey};
taosArrayPush(g, &info1);
}
}
@ -3580,8 +3528,8 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd
sup.pTagSchema = pTagSchema->pSchema;
sup.pCols = pCols;
// taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn);
// createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn);
taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn);
createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn);
}
return pTableGroup;
@ -3689,16 +3637,16 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd
int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId) {
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) {
STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid);
if (pTbCfg == NULL) {
tsdbError("%p failed to get stable, uid:%"PRIu64", reqId:0x%"PRIx64, tsdb, uid, reqId);
tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
goto _error;
}
if (pTbCfg->type != META_SUPER_TABLE) {
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", reId:0x%"PRIx64, tsdb, uid, reqId);
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
goto _error;
}
@ -3717,8 +3665,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch
pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res);
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu", tsdb,
pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList));
tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb,
pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId);
taosArrayDestroy(res);
return ret;
@ -3891,7 +3839,7 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) {
tfree(pTsdbReadHandle->statis);
if (!emptyQueryTimewindow(pTsdbReadHandle)) {
tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle);
} else {
assert(pTsdbReadHandle->pTableCheckInfo == NULL);
}
@ -3910,8 +3858,8 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) {
SIOCostSummary* pCost = &pTsdbReadHandle->cost;
tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64,
pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId);
tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %s",
pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->idStr);
tfree(pTsdbReadHandle);
}

2
source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c
View File

@ -114,4 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) {
if (pNode) {
free(pNode);
}
}
}

1
source/dnode/vnode/src/vnd/vnodeBufferPool.c
View File

@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) {
void vnodeCloseBufPool(SVnode *pVnode) {
if (pVnode->pBufPool) {
tfree(pVnode->pBufPool->pMAF);
vmaDestroy(pVnode->pBufPool->inuse);
while (true) {

3
source/dnode/vnode/src/vnd/vnodeMgr.c
View File

@ -98,6 +98,8 @@ int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) {
/* ------------------------ STATIC METHODS ------------------------ */
static void* loop(void* arg) {
setThreadName("vnode-commit");
SVnodeTask* pTask;
for (;;) {
pthread_mutex_lock(&(vnodeMgr.mutex));
@ -119,6 +121,7 @@ static void* loop(void* arg) {
pthread_mutex_unlock(&(vnodeMgr.mutex));
(*(pTask->execute))(pTask->arg);
free(pTask);
}
return NULL;

42
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -17,14 +17,14 @@
#include "vnd.h"
static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg);
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg);
int vnodeQueryOpen(SVnode *pVnode) {
return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode,
(putReqToQueryQFp)vnodePutReqToVQueryQ);
}
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in query queue is processing");
switch (pMsg->msgType) {
@ -38,7 +38,7 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
}
}
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in fetch queue is processing");
switch (pMsg->msgType) {
case TDMT_VND_FETCH:
@ -57,24 +57,24 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return vnodeGetTableList(pVnode, pMsg);
// return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg);
case TDMT_VND_TABLE_META:
return vnodeGetTableMeta(pVnode, pMsg, pRsp);
return vnodeGetTableMeta(pVnode, pMsg);
case TDMT_VND_CONSUME:
return tqProcessConsumeReq(pVnode->pTq, pMsg, pRsp);
return tqProcessConsumeReq(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in fetch queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
}
}
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
STableInfoReq * pReq = (STableInfoReq *)(pMsg->pCont);
STbCfg * pTbCfg = NULL;
STbCfg * pStbCfg = NULL;
tb_uid_t uid;
int32_t nCols;
int32_t nTagCols;
SSchemaWrapper *pSW;
STableMetaRsp * pTbMetaMsg = NULL;
SSchemaWrapper *pSW = NULL;
STableMetaRsp *pTbMetaMsg = NULL;
SSchema * pTagSchema;
SRpcMsg rpcMsg;
int msgLen = 0;
@ -145,6 +145,22 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
_exit:
if (pSW != NULL) {
tfree(pSW->pSchema);
tfree(pSW);
}
if (pTbCfg) {
tfree(pTbCfg->name);
if (pTbCfg->type == META_SUPER_TABLE) {
free(pTbCfg->stbCfg.pTagSchema);
} else if (pTbCfg->type == META_SUPER_TABLE) {
kvRowFree(pTbCfg->ctbCfg.pTag);
}
tfree(pTbCfg);
}
rpcMsg.handle = pMsg->handle;
rpcMsg.ahandle = pMsg->ahandle;
rpcMsg.pCont = pTbMetaMsg;
@ -156,8 +172,8 @@ _exit:
return 0;
}
static void freeItemHelper(void* pItem) {
char* p = *(char**)pItem;
static void freeItemHelper(void *pItem) {
char *p = *(char **)pItem;
free(p);
}
@ -187,14 +203,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
// TODO: temp debug, and should del when show tables command ok
vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables);
if (numOfTables > 10000) {
numOfTables = 10000;
numOfTables = 10000;
}
metaCloseTbCursor(pCur);
int32_t rowLen =
(TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4;
//int32_t numOfTables = (int32_t)taosArrayGetSize(pArray);
// int32_t numOfTables = (int32_t)taosArrayGetSize(pArray);
int32_t payloadLen = rowLen * numOfTables;
// SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
@ -208,6 +224,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
STR_TO_VARSTR(p, n);
p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
// free(n);
}
pFetchRsp->numOfRows = htonl(numOfTables);
@ -222,6 +239,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
};
rpcSendResponse(&rpcMsg);
taosArrayDestroyEx(pArray, freeItemHelper);
return 0;
}

12
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
// ser request version
void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int64_t ver = pVnode->state.processed++;
taosEncodeFixedU64(&pBuf, ver);
@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SVCreateTbReq vCreateTbReq;
SVCreateTbBatchReq vCreateTbBatchReq;
void *ptr = vnodeMalloc(pVnode, pMsg->contLen);
void * ptr = vnodeMalloc(pVnode, pMsg->contLen);
if (ptr == NULL) {
// TODO: handle error
}
@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
}
// TODO: maybe need to clear the requst struct
free(vCreateTbReq.stbCfg.pSchema);
free(vCreateTbReq.stbCfg.pTagSchema);
free(vCreateTbReq.name);
break;
case TDMT_VND_CREATE_TABLE:
tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq);
@ -112,7 +115,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
}
break;
case TDMT_VND_MQ_SET_CONN: {
if (tqProcessSetConnReq(pVnode->pTq, ptr) < 0) {
if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) {
// TODO: handle error
}
} break;
default:
@ -129,7 +133,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO: handle error
}
}
return 0;
}

2
source/libs/executor/inc/executil.h
View File

@ -38,7 +38,7 @@
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId)
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)

24
source/libs/executor/inc/executorimpl.h
View File

@ -239,7 +239,7 @@ typedef struct STaskIdInfo {
uint64_t queryId; // this is also a request id
uint64_t subplanId;
uint64_t templateId;
uint64_t taskId; // this is a subplan id
char *str;
} STaskIdInfo;
typedef struct SExecTaskInfo {
@ -250,9 +250,8 @@ typedef struct SExecTaskInfo {
STaskCostInfo cost;
int64_t owner; // if it is in execution
int32_t code;
uint64_t totalRows; // total number of rows
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
pthread_mutex_t lock; // used to synchronize the rsp/query threads
char *sql; // query sql string
jmp_buf env; //
struct SOperatorInfo *pRoot;
@ -378,8 +377,10 @@ typedef struct SExchangeInfo {
SSDataBlock *pResult;
int32_t current;
uint64_t rowsOfCurrentSource;
uint64_t bytes; // total load bytes from remote
uint64_t totalRows;
uint64_t totalSize; // total load bytes from remote
uint64_t totalRows; // total number of rows
uint64_t totalElapsed;// total elapsed time
} SExchangeInfo;
typedef struct STableScanInfo {
@ -622,8 +623,6 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq *pQueryMsg, int32_t nu
int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters);
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableReq *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start,
int32_t prevResultLen, void* merger);
@ -639,32 +638,29 @@ int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg);
bool isTaskKilled(SExecTaskInfo *pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables);
bool checkNeedToCompressQueryCol(SQInfo *pQInfo);
bool doBuildResCheck(SQInfo* pQInfo);
void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status);
bool onlyQueryTags(STaskAttr* pQueryAttr);
//void destroyUdfInfo(struct SUdfInfo* pUdfInfo);
bool isValidQInfo(void *param);
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen);
size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows);
void setQueryKilled(SQInfo *pQInfo);
void setTaskKilled(SExecTaskInfo *pTaskInfo);
void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType);
void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code);
void calculateOperatorProfResults(SQInfo* pQInfo);
void queryCostStatis(SQInfo *pQInfo);
void queryCostStatis(SExecTaskInfo *pTaskInfo);
void doDestroyTask(SQInfo *pQInfo);
void doDestroyTask(SExecTaskInfo *pTaskInfo);
void freeQueryAttr(STaskAttr *pQuery);
int32_t getMaximumIdleDurationSec();
void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId);
#endif // TDENGINE_EXECUTORIMPL_H

10
source/libs/executor/src/dataDispatcher.c
View File

@ -121,11 +121,19 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat
}
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
if (taosQueueSize(pDispatcher->pDataBlocks) >= pDispatcher->pManager->cfg.maxDataBlockNumPerQuery) {
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueSize(pDispatcher->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueSize(pDispatcher->pDataBlocks));
return false;
}
pBuf->allocSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap) + pDispatcher->schema.resultRowSize * pInput->pData->info.rows;
pBuf->pData = malloc(pBuf->allocSize);
if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
}
return NULL != pBuf->pData;
}

20
source/libs/executor/src/executor.c
View File

@ -18,20 +18,20 @@
#include "executorimpl.h"
#include "planner.h"
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) {
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != OP_StreamScan) {
if (pOperator->numOfDownstream == 0) {
qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId);
qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
if (pOperator->numOfDownstream > 1) { // not handle this in join query
qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId);
qError("join not supported for stream block scan, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
return doSetStreamBlock(pOperator->pDownstream[0], input, reqId);
return doSetStreamBlock(pOperator->pDownstream[0], input, id);
} else {
SStreamBlockScanInfo* pInfo = pOperator->info;
tqReadHandleSetMsg(pInfo->readerHandle, input, 0);
@ -52,16 +52,16 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) {
int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo));
qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo));
} else {
qDebug("set the stream block successfully, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo));
qDebug("%s set the stream block successfully", GET_TASKID(pTaskInfo));
}
return code;
}
qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) {
if (pMsg == NULL || streamReadHandle == NULL) {
qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
if (msg == NULL || streamReadHandle == NULL) {
return NULL;
}
@ -74,14 +74,14 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle
#endif
struct SSubplan* plan = NULL;
int32_t code = qStringToSubplan(pMsg->msg, &plan);
int32_t code = qStringToSubplan(msg, &plan);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(streamReadHandle, 0, plan, &pTaskInfo, NULL);
code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL);
if (code != TSDB_CODE_SUCCESS) {
// TODO: destroy SSubplan & pTaskInfo
terrno = code;

247
source/libs/executor/src/executorMain.c
View File

@ -69,11 +69,11 @@ void freeParam(STaskParam *param) {
tfree(param->prevResult);
}
int32_t qCreateExecTask(void* readHandle, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) {
int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) {
assert(readHandle != NULL && pSubplan != NULL);
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle);
int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -83,8 +83,9 @@ int32_t qCreateExecTask(void* readHandle, int32_t vgId, SSubplan* pSubplan, qTas
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = dsCreateDataSinker(pSubplan->pDataSink, handle);
if (handle) {
code = dsCreateDataSinker(pSubplan->pDataSink, handle);
}
_error:
// if failed to add ref for all tables in this query, abort current query
@ -140,16 +141,10 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int64_t threadId = taosGetSelfPthreadId();
// todo: remove it.
if (tinfo == NULL) {
return TSDB_CODE_SUCCESS;
}
*pRes = NULL;
int64_t curOwner = 0;
if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo,
qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo,
(void*)curOwner);
pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC;
return pTaskInfo->code;
@ -160,7 +155,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
}
if (isTaskKilled(pTaskInfo)) {
qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo));
qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
return TSDB_CODE_SUCCESS;
}
@ -169,12 +164,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
if (ret != TSDB_CODE_SUCCESS) {
publishQueryAbortEvent(pTaskInfo, ret);
pTaskInfo->code = ret;
qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo),
qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo),
tstrerror(pTaskInfo->code));
return pTaskInfo->code;
}
qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo));
qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));
bool newgroup = false;
publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC);
@ -183,66 +178,25 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
st = taosGetTimestampUs();
*pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup);
pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st);
uint64_t el = (taosGetTimestampUs() - st);
pTaskInfo->cost.elapsedTime += el;
publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (NULL == *pRes) {
*useconds = pTaskInfo->cost.elapsedTime;
}
qDebug("QInfo:0x%" PRIx64 " query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d",
GET_TASKID(pTaskInfo), 0, 0L, 0);
int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0;
pTaskInfo->totalRows += current;
qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0);
atomic_store_64(&pTaskInfo->owner, 0);
return pTaskInfo->code;
}
int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) {
SQInfo *pQInfo = (SQInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
qError("QInfo invalid qhandle");
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
*buildRes = false;
if (IS_QUERY_KILLED(pQInfo)) {
qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code);
return pQInfo->code;
}
int32_t code = TSDB_CODE_SUCCESS;
if (tsRetrieveBlockingModel) {
pQInfo->rspContext = pRspContext;
tsem_wait(&pQInfo->ready);
*buildRes = true;
code = pQInfo->code;
} else {
STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
pthread_mutex_lock(&pQInfo->lock);
assert(pQInfo->rspContext == NULL);
if (pQInfo->dataReady == QUERY_RESULT_READY) {
*buildRes = true;
qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize,
GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code));
} else {
*buildRes = false;
qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId);
pQInfo->rspContext = pRspContext;
assert(pQInfo->rspContext != NULL);
}
code = pQInfo->code;
pthread_mutex_unlock(&pQInfo->lock);
}
return code;
}
void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) {
SQInfo* pQInfo = (SQInfo*) qinfo;
assert(pQInfo != NULL);
@ -251,18 +205,18 @@ void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) {
}
int32_t qKillTask(qTaskInfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId);
setQueryKilled(pQInfo);
qDebug("%s execTask killed", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo);
// Wait for the query executing thread being stopped/
// Once the query is stopped, the owner of qHandle will be cleared immediately.
while (pQInfo->owner != 0) {
while (pTaskInfo->owner != 0) {
taosMsleep(100);
}
@ -270,14 +224,14 @@ int32_t qKillTask(qTaskInfo_t qinfo) {
}
int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
SQInfo *pQInfo = (SQInfo *)qinfo;
SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo;
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qDebug("QInfo:0x%"PRIx64" query async killed", pQInfo->qId);
setQueryKilled(pQInfo);
qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
setTaskKilled(pTaskInfo);
return TSDB_CODE_SUCCESS;
}
@ -292,145 +246,12 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER);
}
void qDestroyTask(qTaskInfo_t qHandle) {
SQInfo* pQInfo = (SQInfo*) qHandle;
if (!isValidQInfo(pQInfo)) {
return;
}
void qDestroyTask(qTaskInfo_t qTaskHandle) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) qTaskHandle;
qDebug("%s execTask completed, numOfRows:%"PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows);
qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId);
queryCostStatis(pQInfo); // print the query cost summary
doDestroyTask(pQInfo);
}
void* qOpenTaskMgmt(int32_t vgId) {
const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool
char cacheName[128] = {0};
sprintf(cacheName, "qhandle_%d", vgId);
STaskMgmt* pTaskMgmt = calloc(1, sizeof(STaskMgmt));
if (pTaskMgmt == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
pTaskMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName);
pTaskMgmt->closed = false;
pTaskMgmt->vgId = vgId;
pthread_mutex_init(&pTaskMgmt->lock, NULL);
qDebug("vgId:%d, open queryTaskMgmt success", vgId);
return pTaskMgmt;
}
void qTaskMgmtNotifyClosing(void* pQMgmt) {
if (pQMgmt == NULL) {
return;
}
STaskMgmt* pQueryMgmt = pQMgmt;
qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId);
pthread_mutex_lock(&pQueryMgmt->lock);
pQueryMgmt->closed = true;
pthread_mutex_unlock(&pQueryMgmt->lock);
taosCacheRefresh(pQueryMgmt->qinfoPool, taskMgmtKillTaskFn, NULL);
}
void qQueryMgmtReOpen(void *pQMgmt) {
if (pQMgmt == NULL) {
return;
}
STaskMgmt *pQueryMgmt = pQMgmt;
qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId);
pthread_mutex_lock(&pQueryMgmt->lock);
pQueryMgmt->closed = false;
pthread_mutex_unlock(&pQueryMgmt->lock);
}
void qCleanupTaskMgmt(void* pQMgmt) {
if (pQMgmt == NULL) {
return;
}
STaskMgmt* pQueryMgmt = pQMgmt;
int32_t vgId = pQueryMgmt->vgId;
assert(pQueryMgmt->closed);
SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool;
pQueryMgmt->qinfoPool = NULL;
taosCacheCleanup(pqinfoPool);
pthread_mutex_destroy(&pQueryMgmt->lock);
tfree(pQueryMgmt);
qDebug("vgId:%d, queryMgmt cleanup completed", vgId);
}
void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo) {
if (pMgmt == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
STaskMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL) {
qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo);
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
pthread_mutex_lock(&pQueryMgmt->lock);
if (pQueryMgmt->closed) {
pthread_mutex_unlock(&pQueryMgmt->lock);
qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo);
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
} else {
void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE),
(getMaximumIdleDurationSec()*1000));
pthread_mutex_unlock(&pQueryMgmt->lock);
return handle;
}
}
void** qAcquireTask(void* pMgmt, uint64_t _key) {
STaskMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->closed) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
return NULL;
}
if (pQueryMgmt->qinfoPool == NULL) {
terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
return NULL;
}
void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key));
if (handle == NULL || *handle == NULL) {
terrno = TSDB_CODE_QRY_INVALID_QHANDLE;
return NULL;
} else {
return handle;
}
}
void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle) {
STaskMgmt *pQueryMgmt = pMgmt;
if (pQueryMgmt->qinfoPool == NULL) {
return NULL;
}
taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
return 0;
queryCostStatis(pTaskInfo); // print the query cost summary
doDestroyTask(pTaskInfo);
}
#if 0
@ -444,8 +265,8 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo
if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId);
setQueryKilled(pQInfo);
qWarn("%s be killed(no memory commit).", pQInfo->qId);
setTaskKilled(pQInfo);
// wait query stop
int32_t loop = 0;
@ -461,4 +282,4 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo
return error;
}
#endif
#endif

458
source/libs/executor/src/executorimpl.c
View File

@ -2432,10 +2432,6 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) {
}
bool isTaskKilled(SExecTaskInfo *pTaskInfo) {
if (IS_QUERY_KILLED(pTaskInfo)) {
return true;
}
// query has been executed more than tsShellActivityTimer, and the retrieve has not arrived
// abort current query execution.
if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec())
@ -2444,13 +2440,13 @@ bool isTaskKilled(SExecTaskInfo *pTaskInfo) {
assert(pTaskInfo->cost.start != 0);
// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64
// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec());
return true;
// return true;
}
return false;
}
void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;}
void setTaskKilled(SExecTaskInfo *pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;}
//static bool isFixedOutputQuery(STaskAttr* pQueryAttr) {
// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) {
@ -4420,33 +4416,32 @@ void calculateOperatorProfResults(SQInfo* pQInfo) {
taosArrayDestroy(opStack);
}
void queryCostStatis(SQInfo *pQInfo) {
STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
STaskCostInfo *pSummary = &pQInfo->summary;
void queryCostStatis(SExecTaskInfo *pTaskInfo) {
STaskCostInfo *pSummary = &pTaskInfo->cost;
uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable);
hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map);
pSummary->hashSize = hashSize;
// uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable);
// hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map);
// pSummary->hashSize = hashSize;
// add the merge time
pSummary->elapsedTime += pSummary->firstStageMergeTime;
SResultRowPool* p = pQInfo->runtimeEnv.pool;
if (p != NULL) {
pSummary->winInfoSize = getResultRowPoolMemSize(p);
pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p);
} else {
pSummary->winInfoSize = 0;
pSummary->numOfTimeWindows = 0;
}
calculateOperatorProfResults(pQInfo);
//qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, "
// "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
// pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
// pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
// SResultRowPool* p = pTaskInfo->pool;
// if (p != NULL) {
// pSummary->winInfoSize = getResultRowPoolMemSize(p);
// pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p);
// } else {
// pSummary->winInfoSize = 0;
// pSummary->numOfTimeWindows = 0;
// }
//
// calculateOperatorProfResults(pQInfo);
qDebug("%s :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, "
"load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64,
GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis,
pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows);
//
//qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0,
// pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0);
@ -4995,7 +4990,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
pResultRowInfo->curPos = 0;
}
qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
}
@ -5006,7 +5001,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) {
// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
if (pResultRowInfo->size > 0) {
@ -5151,6 +5146,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources);
if (pExchangeInfo->current >= totalSources) {
qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources,
pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0);
return NULL;
}
@ -5171,7 +5168,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
epSet.port[0] = pSource->addr.epAddr[0].port;
tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0]));
qDebug("QID:0x%" PRIx64 " build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
int64_t startTs = taosGetTimestampUs();
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources);
pMsg->header.vgId = htonl(pSource->addr.nodeId);
@ -5182,7 +5180,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
// send the fetch remote task result reques
pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("QID:0x%" PRIx64 " prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _error;
}
@ -5199,7 +5197,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp;
if (pRsp->numOfRows == 0) {
qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next",
qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next",
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1,
pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows);
@ -5207,6 +5205,11 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
pExchangeInfo->current += 1;
if (pExchangeInfo->current >= totalSources) {
int64_t el = taosGetTimestampUs() - startTs;
pExchangeInfo->totalElapsed += el;
qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources,
pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0);
return NULL;
} else {
continue;
@ -5233,21 +5236,24 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) {
pRes->info.numOfCols = pOperator->numOfOutput;
pRes->info.rows = pRsp->numOfRows;
int64_t el = taosGetTimestampUs() - startTs;
pExchangeInfo->totalRows += pRsp->numOfRows;
pExchangeInfo->bytes += pRsp->compLen;
pExchangeInfo->totalSize += pRsp->compLen;
pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows;
pExchangeInfo->totalElapsed += el;
if (pRsp->completed == 1) {
qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->bytes,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->totalSize,
pExchangeInfo->current + 1, totalSources);
pExchangeInfo->rowsOfCurrentSource = 0;
pExchangeInfo->current += 1;
} else {
qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->bytes);
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->totalSize);
}
return pExchangeInfo->pResult;
@ -5294,7 +5300,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "TSC";
rpcInit.label = "EX";
rpcInit.numOfThreads = 1;
rpcInit.cfp = processRspMsg;
rpcInit.sessions = tsMaxConnections;
@ -7729,33 +7735,37 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t
return TSDB_CODE_SUCCESS;
}
static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) {
static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) {
SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo));
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pthread_mutex_init(&pTaskInfo->lock, NULL);
pTaskInfo->cost.created = taosGetTimestampMs();
pTaskInfo->id.queryId = queryId;
char* p = calloc(1, 128);
snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId);
pTaskInfo->id.str = strdup(p);
return pTaskInfo;
}
static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId);
static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId);
SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId) {
SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) {
if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) {
if (pPhyNode->info.type == OP_TableScan) {
SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode;
size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets);
tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId);
tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId);
return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo);
} else if (pPhyNode->info.type == OP_DataBlocksOptScan) {
SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode;
size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets);
tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId);
tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId);
return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo);
} else if (pPhyNode->info.type == OP_Exchange) {
@ -7773,13 +7783,13 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask
for (int32_t i = 0; i < size; ++i) {
SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i);
SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId);
SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId, taskId);
return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo);
}
}
}
static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId) {
static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) {
STsdbQueryCond cond = {.loadExternalRows = false};
cond.order = pTableScanNode->scan.order;
@ -7798,10 +7808,10 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S
cond.colList[i].colId = pSchema->colId;
}
return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, NULL);
return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId);
}
static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId) {
static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) {
int32_t code = 0;
STableGroupInfo groupInfo = {0};
@ -7811,7 +7821,7 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode,
if (tableType == TSDB_SUPER_TABLE) {
code =
tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId);
tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId, taskId);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -7821,35 +7831,35 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode,
SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo));
STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid};
STableKeyInfo info = {.lastKey = 0, .uid = uid};
taosArrayPush(pa, &info);
taosArrayPush(groupInfo.pGroupList, &pa);
}
if (groupInfo.numOfTables == 0) {
code = 0;
qDebug("no table qualified for query, reqId:0x%"PRIx64, queryId);
qDebug("no table qualified for query, TID:0x%"PRIx64", QID:0x%"PRIx64, taskId, queryId);
goto _error;
}
return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId);
return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId, taskId);
_error:
terrno = code;
return NULL;
}
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) {
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId) {
uint64_t queryId = pPlan->id.queryId;
int32_t code = TSDB_CODE_SUCCESS;
*pTaskInfo = createExecTaskInfo(queryId);
*pTaskInfo = createExecTaskInfo(queryId, taskId);
if (*pTaskInfo == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _complete;
}
(*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId);
(*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId, taskId);
if ((*pTaskInfo)->pRoot == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _complete;
@ -8673,229 +8683,6 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
return ((SQInfo *)qHandle)->qId == qId;
}
SQInfo* createQInfoImpl(SQueryTableReq* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId,
char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo) {
int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput;
SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
if (pQInfo == NULL) {
goto _cleanup_qinfo;
}
pQInfo->qId = qId;
pQInfo->startExecTs = 0;
pQInfo->runtimeEnv.pUdfInfo = pUdfInfo;
// to make sure third party won't overwrite this structure
pQInfo->signature = pQInfo;
STaskAttr* pQueryAttr = &pQInfo->query;
pQInfo->runtimeEnv.pQueryAttr = pQueryAttr;
pQueryAttr->tableGroupInfo = *pTableGroupInfo;
pQueryAttr->numOfCols = numOfCols;
pQueryAttr->numOfOutput = numOfOutput;
pQueryAttr->limit.limit = pQueryMsg->limit;
pQueryAttr->limit.offset = pQueryMsg->offset;
pQueryAttr->order.order = pQueryMsg->order;
pQueryAttr->order.col.info.colId = pQueryMsg->orderColId;
pQueryAttr->pExpr1 = pExprs;
pQueryAttr->pExpr2 = pSecExprs;
pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput;
pQueryAttr->pGroupbyExpr = pGroupbyExpr;
memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval));
pQueryAttr->fillType = pQueryMsg->fillType;
pQueryAttr->numOfTags = pQueryMsg->numOfTags;
pQueryAttr->tagColList = pTagCols;
pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit;
pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX;
// pQueryAttr->sw = pQueryMsg->sw;
pQueryAttr->vgId = vgId;
pQueryAttr->stableQuery = pQueryMsg->stableQuery;
pQueryAttr->topBotQuery = pQueryMsg->topBotQuery;
pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn;
pQueryAttr->hasTagResults = pQueryMsg->hasTagResults;
pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo;
pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist;
pQueryAttr->stabledev = pQueryMsg->stabledev;
pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery;
pQueryAttr->simpleAgg = pQueryMsg->simpleAgg;
pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery;
pQueryAttr->needReverseScan = pQueryMsg->needReverseScan;
pQueryAttr->stateWindow = pQueryMsg->stateWindow;
pQueryAttr->vgId = vgId;
// pQueryAttr->pFilters = pFilters;
pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo));
if (pQueryAttr->tableCols == NULL) {
goto _cleanup;
}
pQueryAttr->srcRowSize = 0;
pQueryAttr->maxTableColumnWidth = 0;
for (int16_t i = 0; i < numOfCols; ++i) {
pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i];
// pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters);
pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes;
if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) {
pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes;
}
}
for (int16_t col = 0; col < numOfOutput; ++col) {
assert(pExprs[col].base.resSchema.bytes > 0);
pQueryAttr->resultRowSize += pExprs[col].base.resSchema.bytes;
// keep the tag length
if (TSDB_COL_IS_TAG(pExprs[col].base.pColumns->flag)) {
pQueryAttr->tagLen += pExprs[col].base.resSchema.bytes;
}
// if (pExprs[col].base.flist.filterInfo) {
// ++pQueryAttr->havingNum;
// }
}
doUpdateExprColumnIndex(pQueryAttr);
if (pSecExprs != NULL) {
int32_t resultRowSize = 0;
// calculate the result row size
for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) {
assert(pSecExprs[col].base.resSchema.bytes > 0);
resultRowSize += pSecExprs[col].base.resSchema.bytes;
}
if (resultRowSize > pQueryAttr->resultRowSize) {
pQueryAttr->resultRowSize = resultRowSize;
}
}
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput);
if (pQueryAttr->fillVal == NULL) {
goto _cleanup;
}
// the first column is the timestamp
memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t));
}
size_t numOfGroups = 0;
if (pTableGroupInfo->pGroupList != NULL) {
numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo;
pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
pTableqinfo->numOfTables = pTableGroupInfo->numOfTables;
pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
}
pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));
if (pQInfo->pBuf == NULL) {
goto _cleanup;
}
pQInfo->dataReady = QUERY_RESULT_NOT_READY;
pQInfo->rspContext = NULL;
pQInfo->sql = sql;
pthread_mutex_init(&pQInfo->lock, NULL);
tsem_init(&pQInfo->ready, 0, 0);
pQueryAttr->window = pQueryMsg->window;
updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery);
STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
STimeWindow window = pQueryAttr->window;
int32_t index = 0;
for(int32_t i = 0; i < numOfGroups; ++i) {
SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);
size_t s = taosArrayGetSize(pa);
SArray* p1 = taosArrayInit(s, POINTER_BYTES);
if (p1 == NULL) {
goto _cleanup;
}
taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1);
for(int32_t j = 0; j < s; ++j) {
// STableKeyInfo* info = taosArrayGet(pa, j);
// window.skey = info->lastKey;
//
// void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo);
// STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf);
// if (item == NULL) {
// goto _cleanup;
// }
//
// item->groupIndex = i;
// taosArrayPush(p1, &item);
// STableId* id = TSDB_TABLEID(info->pTable);
// taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES);
// index += 1;
}
}
colIdCheck(pQueryAttr, pQInfo->qId);
// int32_t functionId = getExprFunctionId(&pExpr[0]);
// pQInfo->query.queryBlockDist = (functionId == FUNCTION_BLKINFO);
//qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo);
return pQInfo;
_cleanup_qinfo:
// tsdbDestroyTableGroup(pTableGroupInfo);
if (pGroupbyExpr != NULL) {
taosArrayDestroy(pGroupbyExpr->columnInfo);
free(pGroupbyExpr);
}
tfree(pTagCols);
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExprInfo = &pExprs[i];
if (pExprInfo->pExpr != NULL) {
tExprTreeDestroy(pExprInfo->pExpr, NULL);
pExprInfo->pExpr = NULL;
}
// if (pExprInfo->base.flist.filterInfo) {
// freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters);
// }
}
tfree(pExprs);
// filterFreeInfo(pFilters);
_cleanup:
doDestroyTask(pQInfo);
return NULL;
}
bool isValidQInfo(void *param) {
SQInfo *pQInfo = (SQInfo *)param;
if (pQInfo == NULL) {
return false;
}
/*
* pQInfo->signature may be changed by another thread, so we assign value of signature
* into local variable, then compare by using local variable
*/
uint64_t sig = (uint64_t)pQInfo->signature;
return (sig == (uint64_t)pQInfo);
}
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start,
int32_t prevResultLen, void* merger) {
int32_t code = TSDB_CODE_SUCCESS;
@ -8957,7 +8744,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo*
_error:
// table query ref will be decrease during error handling
doDestroyTask(pQInfo);
// doDestroyTask(pQInfo);
return code;
}
@ -9038,113 +8825,16 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) {
return NULL;
}
void doDestroyTask(SQInfo *pQInfo) {
if (!isValidQInfo(pQInfo)) {
return;
}
void doDestroyTask(SExecTaskInfo *pTaskInfo) {
doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo);
// taosArrayDestroy(pTaskInfo->summary.queryProfEvents);
// taosHashCleanup(pTaskInfo->summary.operatorProfResults);
//qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId);
tfree(pTaskInfo->sql);
tfree(pTaskInfo->id.str);
qDebug("%s execTask is freed", GET_TASKID(pTaskInfo));
STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables);
doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo);
teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
freeQueryAttr(pQueryAttr);
// tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo);
tfree(pQInfo->pBuf);
tfree(pQInfo->sql);
taosArrayDestroy(pQInfo->summary.queryProfEvents);
taosHashCleanup(pQInfo->summary.operatorProfResults);
taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows);
pQInfo->signature = 0;
//qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId);
tfree(pQInfo);
}
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) {
// the remained number of retrieved rows, not the interpolated result
STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
// load data from file to msg buffer
if (pQueryAttr->tsCompQuery) {
SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0);
FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor
// make sure file exist
if (f) {
off_t s = lseek(fileno(f), 0, SEEK_END);
assert(s == pRuntimeEnv->outputBuf->info.rows);
//qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s);
if (fseek(f, 0, SEEK_SET) >= 0) {
size_t sz = fread(data, 1, s, f);
if(sz < s) { // todo handle error
//qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s);
assert(0);
}
} else {
UNUSED(s);
//qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno));
assert(0);
}
// dump error info
if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) {
// qDump(data, s);
assert(0);
}
fclose(f);
*(FILE **)pColInfoData->pData = NULL;
}
// all data returned, set query over
if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) {
// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER);
}
} else {
doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen);
}
//qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId,
// pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total);
if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) {
//qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit);
// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER);
}
return TSDB_CODE_SUCCESS;
}
bool doBuildResCheck(SQInfo* pQInfo) {
bool buildRes = false;
pthread_mutex_lock(&pQInfo->lock);
pQInfo->dataReady = QUERY_RESULT_READY;
buildRes = needBuildResAfterQueryComplete(pQInfo);
// clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is
// put into task to be executed.
assert(pQInfo->owner == taosGetSelfPthreadId());
pQInfo->owner = 0;
pthread_mutex_unlock(&pQInfo->lock);
// used in retrieve blocking model.
tsem_post(&pQInfo->ready);
return buildRes;
tfree(pTaskInfo);
}
static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) {

2
source/libs/executor/test/executorTests.cpp
View File

@ -219,7 +219,7 @@ TEST(testCase, build_executor_tree_Test) {
SExecTaskInfo* pTaskInfo = nullptr;
DataSinkHandle sinkHandle = nullptr;
int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle);
int32_t code = qCreateExecTask((void*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
}
#pragma GCC diagnostic pop

2
source/libs/parser/inc/dataBlockMgt.h
View File

@ -171,6 +171,8 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs);
void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols);
void destroyBoundColumnInfo(SParsedDataColInfo* pColList);
void destroyBlockArrayList(SArray* pDataBlockList);
void destroyBlockHashmap(SHashObj* pDataBlockHash);
int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen);
int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,

2
source/libs/parser/inc/sql.y
View File

@ -65,7 +65,7 @@ program ::= cmd. {}
//////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);}
cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}

32
source/libs/parser/src/astValidate.c
View File

@ -213,10 +213,11 @@ SQueryStmtInfo *createQueryInfo() {
pQueryInfo->slimit.limit = -1;
pQueryInfo->slimit.offset = 0;
pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->window = TSWINDOW_INITIALIZER;
pQueryInfo->exprList = calloc(10, POINTER_BYTES);
for(int32_t i = 0; i < 10; ++i) {
pQueryInfo->exprList[i] = taosArrayInit(4, POINTER_BYTES);
}
@ -232,7 +233,8 @@ static void destroyQueryInfoImpl(SQueryStmtInfo* pQueryInfo) {
cleanupFieldInfo(&pQueryInfo->fieldsInfo);
dropAllExprInfo(pQueryInfo->exprList, 10);
pQueryInfo->exprList = NULL;
tfree(pQueryInfo->exprList);
columnListDestroy(pQueryInfo->colList);
pQueryInfo->colList = NULL;
@ -258,10 +260,10 @@ void destroyQueryInfo(SQueryStmtInfo* pQueryInfo) {
size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pDownstream);
for (int32_t i = 0; i < numOfUpstream; ++i) {
SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pDownstream, i);
destroyQueryInfoImpl(pUpQueryInfo);
clearAllTableMetaInfo(pUpQueryInfo, false, 0);
tfree(pUpQueryInfo);
SQueryStmtInfo* pDownstream = taosArrayGetP(pQueryInfo->pDownstream, i);
destroyQueryInfoImpl(pDownstream);
clearAllTableMetaInfo(pDownstream, false, 0);
tfree(pDownstream);
}
destroyQueryInfoImpl(pQueryInfo);
@ -1395,6 +1397,13 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf
static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo);
static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo);
static void freeItemHelper(void* pItem) {
void** p = pItem;
if (*p != NULL) {
tfree(*p);
}
}
int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) {
assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0));
@ -1590,7 +1599,10 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf*
SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]);
extractFunctionDesc(functionList, &pQueryInfo->info);
if ((code = checkForInvalidExpr(pQueryInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) {
code = checkForInvalidExpr(pQueryInfo, pMsgBuf);
taosArrayDestroyEx(functionList, freeItemHelper);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
@ -2902,6 +2914,8 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S
}
pQueryInfo->info.projectionQuery = true;
taosArrayDestroy(pColumnList);
return TSDB_CODE_SUCCESS;
}
@ -3983,5 +3997,9 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI
validateSqlNode(p, pQueryInfo, &buf);
}
taosArrayDestroy(data.pTableMeta);
taosArrayDestroy(req.pUdf);
taosArrayDestroy(req.pTableName);
return code;
}

23
source/libs/parser/src/dataBlockMgt.c
View File

@ -249,7 +249,7 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo
}
}
void destroyDataBlock(STableDataBlocks* pDataBlock) {
static void destroyDataBlock(STableDataBlocks* pDataBlock) {
if (pDataBlock == NULL) {
return;
}
@ -273,12 +273,29 @@ void destroyBlockArrayList(SArray* pDataBlockList) {
size_t size = taosArrayGetSize(pDataBlockList);
for (int32_t i = 0; i < size; i++) {
destroyDataBlock(taosArrayGetP(pDataBlockList, i));
void* p = taosArrayGetP(pDataBlockList, i);
destroyDataBlock(p);
}
taosArrayDestroy(pDataBlockList);
}
void destroyBlockHashmap(SHashObj* pDataBlockHash) {
if (pDataBlockHash == NULL) {
return;
}
void** p1 = taosHashIterate(pDataBlockHash, NULL);
while (p1) {
STableDataBlocks* pBlocks = *p1;
destroyDataBlock(pBlocks);
p1 = taosHashIterate(pDataBlockHash, p1);
}
taosHashCleanup(pDataBlockHash);
}
// data block is disordered, sort it in ascending order
void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) {
SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData;
@ -490,7 +507,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t
}
// the maximum expanded size in byte when a row-wise data is converted to SDataRow format
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta);

20
source/libs/parser/src/insertParser.c
View File

@ -525,10 +525,28 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) {
tdDestroyKVRowBuilder(&pCxt->tagsBuilder);
}
static void destroyDataBlock(STableDataBlocks* pDataBlock) {
if (pDataBlock == NULL) {
return;
}
tfree(pDataBlock->pData);
if (!pDataBlock->cloned) {
// free the refcount for metermeta
if (pDataBlock->pTableMeta != NULL) {
tfree(pDataBlock->pTableMeta);
}
destroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
}
tfree(pDataBlock);
}
static void destroyInsertParseContext(SInsertParseContext* pCxt) {
destroyInsertParseContextForTable(pCxt);
taosHashCleanup(pCxt->pVgroupsHashObj);
taosHashCleanup(pCxt->pTableBlockHashObj);
destroyBlockHashmap(pCxt->pTableBlockHashObj);
destroyBlockArrayList(pCxt->pTableDataBlocks);
destroyBlockArrayList(pCxt->pVgDataBlocks);
}

11
source/libs/parser/src/parser.c
View File

@ -248,10 +248,15 @@ void qDestroyQuery(SQueryNode* pQueryNode) {
if (NULL == pQueryNode) {
return;
}
if (nodeType(pQueryNode) == TSDB_SQL_INSERT || nodeType(pQueryNode) == TSDB_SQL_CREATE_TABLE) {
int32_t type = nodeType(pQueryNode);
if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) {
SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode;
taosArrayDestroy(pModifInfo->pDataBlocks);
}
tfree(pQueryNode);
tfree(pQueryNode);
} else if (type == TSDB_SQL_SELECT) {
SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*) pQueryNode;
destroyQueryInfo(pQueryStmtInfo);
}
}

12
source/libs/parser/src/parserUtil.c
View File

@ -732,18 +732,8 @@ void cleanupFieldInfo(SFieldInfo* pFieldInfo) {
return;
}
if (pFieldInfo->internalField != NULL) {
size_t num = taosArrayGetSize(pFieldInfo->internalField);
for (int32_t i = 0; i < num; ++i) {
// SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i);
// if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) {
// sqlExprDestroy(pfield->pExpr);
// }
}
}
taosArrayDestroy(pFieldInfo->internalField);
// tfree(pFieldInfo->final);
tfree(pFieldInfo->final);
memset(pFieldInfo, 0, sizeof(SFieldInfo));
}

7
source/libs/parser/src/queryInfoUtil.c
View File

@ -191,10 +191,12 @@ void destroyExprInfo(SExprInfo* pExprInfo) {
for(int32_t i = 0; i < pExprInfo->base.numOfParams; ++i) {
taosVariantDestroy(&pExprInfo->base.param[i]);
}
tfree(pExprInfo->base.pColumns);
tfree(pExprInfo);
}
static void dropOneLevelExprInfo(SArray* pExprInfo) {
void dropOneLevelExprInfo(SArray* pExprInfo) {
size_t size = taosArrayGetSize(pExprInfo);
for (int32_t i = 0; i < size; ++i) {
@ -239,6 +241,9 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
#endif
dst->pExpr = exprdup(src->pExpr);
dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn));
memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols);
memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param));
for (int32_t j = 0; j < src->base.numOfParams; ++j) {
taosVariantAssign(&dst->base.param[j], &src->base.param[j]);

2
source/libs/parser/src/sql.c
View File

@ -2232,7 +2232,7 @@ static void yy_reduce(
{ setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
break;
case 3: /* cmd ::= SHOW FUNCTIONS */
{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);}
break;
case 4: /* cmd ::= SHOW MNODES */
{ setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}

14
source/libs/planner/src/logicPlan.c
View File

@ -265,7 +265,6 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo*
} else {
// here we can push down the projection to tablescan operator.
pNode->numOfExpr = num;
pNode->pExpr = taosArrayInit(num, POINTER_BYTES);
taosArrayAddAll(pNode->pExpr, p);
}
}
@ -357,7 +356,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) {
SArray* exprList = taosArrayInit(4, POINTER_BYTES);
if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
// dropAllExprInfo(exprList);
exit(-1);
}
@ -373,7 +371,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) {
// 4. add the projection query node
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, &info, exprList, tableColumnList);
columnListDestroy(tableColumnList);
// dropAllExprInfo(exprList);
taosArrayPush(pDownstream, &pNode);
}
@ -398,7 +395,8 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) {
}
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
if (pQueryNode->info.type == QNODE_MODIFY) {
int32_t type = nodeType(pQueryNode);
if (type == QNODE_MODIFY) {
SDataPayloadInfo* pInfo = pQueryNode->pExtInfo;
size_t size = taosArrayGetSize(pInfo->payload);
@ -410,10 +408,16 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
taosArrayDestroy(pInfo->payload);
}
if (type == QNODE_STREAMSCAN || type == QNODE_TABLESCAN) {
SQueryTableInfo* pQueryTableInfo = pQueryNode->pExtInfo;
tfree(pQueryTableInfo->tableName);
}
taosArrayDestroy(pQueryNode->pExpr);
tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name);
// dropAllExprInfo(pQueryNode->pExpr);
if (pQueryNode->pChildren != NULL) {
int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren);

36
source/libs/planner/src/physicalPlan.c
View File

@ -155,6 +155,16 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si
return node;
}
static void cleanupPhyNode(SPhyNode* pPhyNode) {
if (pPhyNode == NULL) {
return;
}
dropOneLevelExprInfo(pPhyNode->pTargets);
tfree(pPhyNode->targetSchema.pSchema);
tfree(pPhyNode);
}
static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) {
SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size);
@ -445,3 +455,29 @@ void setExchangSourceNode(uint64_t templateId, SDownstreamSource *pSource, SPhyN
void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) {
setExchangSourceNode(templateId, pSource, subplan->pNode);
}
static void destroyDataSinkNode(SDataSink* pSinkNode) {
if (pSinkNode == NULL) {
return;
}
if (nodeType(pSinkNode) == DSINK_Dispatch) {
SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode;
tfree(pDdSink->sink.schema.pSchema);
}
tfree(pSinkNode);
}
void qDestroySubplan(SSubplan* pSubplan) {
if (pSubplan == NULL) {
return;
}
taosArrayDestroy(pSubplan->pChildren);
taosArrayDestroy(pSubplan->pParents);
destroyDataSinkNode(pSubplan->pDataSink);
cleanupPhyNode(pSubplan->pNode);
tfree(pSubplan);
}

10
source/libs/planner/src/physicalPlanJson.c
View File

@ -87,6 +87,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f
static const char* jkPnodeType = "Type";
static int32_t getPnodeTypeSize(cJSON* json) {
switch (getNumber(json, jkPnodeType)) {
case OP_StreamScan:
case OP_TableScan:
case OP_DataBlocksOptScan:
case OP_TableSeqScan:
@ -869,6 +870,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
SPhyNode* phyNode = (SPhyNode*)obj;
switch (phyNode->info.type) {
case OP_TableScan:
case OP_StreamScan:
case OP_DataBlocksOptScan:
case OP_TableSeqScan:
return tableScanNodeFromJson(json, obj);
@ -1121,6 +1123,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
}
*str = cJSON_Print(json);
cJSON_Delete(json);
// printf("====Physical plan:====\n");
// printf("%s\n", *str);
*len = strlen(*str) + 1;
@ -1187,14 +1191,14 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) {
if(pDag == NULL) {
return NULL;
}
pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans"));
pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "queryId"));
pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number"));
pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId"));
pDag->pSubplans = taosArrayInit(0, sizeof(SArray));
if (pDag->pSubplans == NULL) {
free(pDag);
return NULL;
}
cJSON* pLevels = cJSON_GetObjectItem(pRoot, "pSubplans");
cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans");
int level = cJSON_GetArraySize(pLevels);
for(int i = 0; i < level; i++) {
SArray* plansOneLevel = taosArrayInit(0, sizeof(void*));

20
source/libs/planner/src/planner.c
View File

@ -18,25 +18,6 @@
static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols);
static void destroyDataSinkNode(SDataSink* pSinkNode) {
if (pSinkNode == NULL) {
return;
}
tfree(pSinkNode);
}
void qDestroySubplan(SSubplan* pSubplan) {
if (pSubplan == NULL) {
return;
}
taosArrayDestroy(pSubplan->pChildren);
taosArrayDestroy(pSubplan->pParents);
destroyDataSinkNode(pSubplan->pDataSink);
// todo destroy pNode
tfree(pSubplan);
}
void qDestroyQueryDag(struct SQueryDag* pDag) {
if (pDag == NULL) {
return;
@ -51,6 +32,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) {
SSubplan* pSubplan = taosArrayGetP(pa, j);
qDestroySubplan(pSubplan);
}
taosArrayDestroy(pa);
}

16
source/libs/qworker/inc/qworkerInt.h
View File

@ -173,17 +173,17 @@ typedef struct SQWorkerMgmt {
#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__)
#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0)

2
source/libs/qworker/inc/qworkerMsg.h
View File

@ -32,7 +32,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t
int32_t qwBuildAndSendDropRsp(void *connection, int32_t code);
int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code);
int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection);
int32_t qwBuildAndSendSchSinkMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection);
int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code);

40
source/libs/qworker/src/qworker.c
View File

@ -456,7 +456,7 @@ _return:
QW_RET(code);
}
int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
int32_t code = 0;
bool qcontinue = true;
SSDataBlock* pRes = NULL;
@ -467,11 +467,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
DataSinkHandle sinkHandle = ctx->sinkHandle;
while (true) {
QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++);
QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
code = qExecTask(*taskHandle, &pRes, &useconds);
if (code) {
QW_TASK_ELOG("qExecTask failed, code:%x", code);
QW_TASK_ELOG("qExecTask failed, code:%s", tstrerror(code));
QW_ERR_JRET(code);
}
@ -485,6 +485,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
if (TASK_TYPE_TEMP == ctx->taskType) {
qwFreeTaskHandle(QW_FPARAMS(), taskHandle);
}
if (queryEnd) {
*queryEnd = true;
}
break;
}
@ -492,7 +496,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL};
code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue);
if (code) {
QW_TASK_ELOG("dsPutDataBlock failed, code:%x", code);
QW_TASK_ELOG("dsPutDataBlock failed, code:%s", tstrerror(code));
QW_ERR_JRET(code);
}
@ -587,12 +591,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void
QW_ERR_RET(code);
}
queryEnd = pOutput->queryEnd;
pOutput->queryEnd = false;
if (DS_BUF_EMPTY == pOutput->bufStatus && queryEnd) {
pOutput->queryEnd = true;
if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) {
QW_SCH_TASK_DLOG("task all fetched, status:%d", JOB_TASK_STATUS_SUCCEED);
QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED));
}
@ -974,7 +973,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
QW_ERR_JRET(code);
}
code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle);
code = qCreateExecTask(qwMsg->node, 0, tId, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle);
if (code) {
QW_TASK_ELOG("qCreateExecTask failed, code:%s", tstrerror(code));
QW_ERR_JRET(code);
@ -996,7 +995,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
atomic_store_ptr(&ctx->sinkHandle, sinkHandle);
if (pTaskInfo && sinkHandle) {
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx));
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL));
}
_return:
@ -1083,6 +1082,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
SQWPhaseOutput output = {0};
void *rsp = NULL;
int32_t dataLen = 0;
bool queryEnd = false;
do {
QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output));
@ -1102,7 +1102,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
DataSinkHandle sinkHandle = ctx->sinkHandle;
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx));
QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd));
if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
SOutputData sOutput = {0};
@ -1114,13 +1114,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
// RC WARNING
atomic_store_8(&ctx->queryContinue, 1);
}
if (sOutput.queryEnd) {
needStop = true;
}
if (rsp) {
qwBuildFetchRsp(rsp, &sOutput, dataLen);
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
@ -1131,6 +1128,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
}
}
if (queryEnd) {
needStop = true;
}
_return:
if (NULL == ctx) {
@ -1196,7 +1197,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
if (NULL == rsp) {
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH);
} else {
qwBuildFetchRsp(rsp, &sOutput, dataLen);
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
}
if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) {

8
source/libs/qworker/src/qworkerMsg.c
View File

@ -26,11 +26,11 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) {
return TSDB_CODE_SUCCESS;
}
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len) {
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
rsp->useconds = htobe64(input->useconds);
rsp->completed = input->queryEnd;
rsp->completed = qComplete;
rsp->precision = input->precision;
rsp->compressed = input->compressed;
rsp->compLen = htonl(len);
@ -258,12 +258,12 @@ int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId,
int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg);
if (TSDB_CODE_SUCCESS != code) {
QW_SCH_TASK_ELOG("put query continue msg to queue failed, code:%x", code);
QW_SCH_TASK_ELOG("put query continue msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code));
rpcFreeCont(req);
QW_ERR_RET(code);
}
QW_SCH_TASK_DLOG("put query continue msg to query queue, vgId:%d", mgmt->nodeId);
QW_SCH_TASK_DLOG("put task continue exec msg to query queue, vgId:%d", mgmt->nodeId);
return TSDB_CODE_SUCCESS;
}

11
source/libs/scheduler/src/scheduler.c
View File

@ -1482,22 +1482,23 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) {
}
int32_t msgSize = sizeof(SSubQueryMsg) + msgLen;
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;
SSubQueryMsg* pMsg = calloc(1, msgSize);
/*SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;*/
memcpy(pMsg->msg, msg, msgLen);
pMsg->header.vgId = htonl(tInfo.addr.nodeId);
pMsg->header.vgId = tInfo.addr.nodeId;
pMsg->sId = schMgmt.sId;
pMsg->queryId = plan->id.queryId;
pMsg->taskId = schGenUUID();
pMsg->taskType = TASK_TYPE_PERSISTENT;
pMsg->contentLen = msgLen;
memcpy(pMsg->msg, msg, msgLen);
/*memcpy(pMsg->msg, ((SSubQueryMsg*)msg)->msg, msgLen);*/
tInfo.msg = pMsg;
@ -1512,9 +1513,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) {
info = NULL;
_return:
schedulerFreeTaskList(info);
SCH_RET(code);
}

2
source/libs/tdb/CMakeLists.txt
View File

@ -1,5 +1,5 @@
set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr")
set(TDB_SUBDIRS "db")
foreach(TDB_SUBDIR ${TDB_SUBDIRS})
aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC)
endforeach()

51
source/libs/tdb/inc/tdb.h
View File

@ -22,30 +22,41 @@
extern "C" {
#endif
#define TDB_EXTERN
#define TDB_PUBLIC
#define TDB_STATIC static
// #define TDB_EXTERN
// #define TDB_PUBLIC
// #define TDB_STATIC static
typedef enum {
TDB_BTREE_T = 0,
TDB_HASH_T,
TDB_HEAP_T,
} tdb_db_t;
// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t;
// Forward declaration
typedef struct TDB TDB;
typedef struct TDB_CURSOR TDB_CURSOR;
// // Forward declarations
// typedef struct TDB TDB;
// // typedef struct TDB_MPOOL TDB_MPOOL;
// // typedef struct TDB_MPFILE TDB_MPFILE;
// // typedef struct TDB_CURSOR TDB_CURSOR;
// SKey
typedef struct {
void* bdata;
uint32_t size;
} TDB_KEY, TDB_VALUE;
// typedef struct {
// void* bdata;
// uint32_t size;
// } TDB_KEY, TDB_VALUE;
// TDB Operations
TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type);
TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags);
// // TDB Operations
// int tdbCreateDB(TDB** dbpp, tdb_db_t type);
// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags);
// int tdbCloseDB(TDB* dbp, uint32_t flags);
// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags);
// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags);
// // TDB_MPOOL
// int tdbOpenMPool(TDB_MPOOL** mp);
// int tdbCloseMPool(TDB_MPOOL* mp);
// // TDB_MPFILE
// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp);
// int tdbCloseMPFile(TDB_MPFILE** mpf);
// // TDB_CURSOR
// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp);
// int tdbCloseCurosr(TDB_CURSOR* tdbcp);
#ifdef __cplusplus
}

86
source/libs/tdb/src/db/tdbDB.c
View File

@ -1,86 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbDB.h"
#include "tdb.h"
TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) {
TDB* dbp;
int ret;
dbp = calloc(1, sizeof(*dbp));
if (dbp == NULL) {
return -1;
}
dbp->pageSize = TDB_DEFAULT_PGSIZE;
dbp->type = type;
switch (type) {
case TDB_BTREE_T:
// ret = tdbInitBtreeDB(dbp);
// if (ret < 0) goto _err;
break;
case TDB_HASH_T:
// ret = tdbInitHashDB(dbp);
// if (ret < 0) goto _err;
break;
case TDB_HEAP_T:
// ret = tdbInitHeapDB(dbp);
// if (ret < 0) goto _err;
break;
default:
break;
}
*dbpp = dbp;
return 0;
_err:
if (dbp) {
free(dbp);
}
*dbpp = NULL;
return 0;
}
TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) {
int ret = 0;
if ((dbp->fname = strdup(fname)) == NULL) {
ret = -1;
return ret;
}
// Create the backup file if the file not exists
// Open the file as a sub-db or a master-db
if (dbname) {
if ((dbp->dbname = strdup(dbname)) == NULL) {
ret = -1;
return ret;
}
// TODO: Open the DB as a SUB-DB in this file
} else {
// TODO: Open the DB as a MASTER-DB in this file
}
return ret;
}
TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) {
// TODO
return 0;
}

23
source/libs/tdb/src/inc/tdbHeap.h → source/libs/tdb/src/db/tdb_db.c
View File

@ -13,23 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_HEAP_H_
#define _TD_TDB_HEAP_H_
#include "tdb_db.h"
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) {
// TODO
} TDB_HEAP;
TDB_PUBLIC int tdbInitHeapDB(TDB *dbp);
#ifdef __cplusplus
return 0;
}
#endif
#endif /*_TD_TDB_HEAP_H_*/
int tdbClose(TDB *dbp, uint32_t flags) {
// TODO
return 0;
}

379
source/libs/tdb/src/db/tdb_mpool.c Normal file
View File

@ -0,0 +1,379 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdb_mpool.h"
static int tdbGnrtFileID(const char *fname, uint8_t *fileid);
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf);
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid);
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p);
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p);
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp);
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) {
TDB_MPOOL *mp = NULL;
size_t tsize;
pg_t * pagep;
// check parameters
if (!TDB_IS_PGSIZE_VLD(pgsize)) {
tdbError("invalid page size");
return -1;
}
// allocate handle
mp = (TDB_MPOOL *)calloc(1, sizeof(*mp));
if (mp == NULL) {
tdbError("failed to malloc memory pool handle");
goto _err;
}
// initialize the handle
mp->cachesize = cachesize;
mp->pgsize = pgsize;
mp->npages = cachesize / pgsize;
mp->clockHand = 0;
TD_DLIST_INIT(&mp->freeList);
mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t));
if (mp->pages == NULL) {
tdbError("failed to malloc memory pool pages");
goto _err;
}
for (frame_id_t i = 0; i < mp->npages; i++) {
mp->pages[i].p = malloc(pgsize);
if (mp->pages[i].p == NULL) {
goto _err;
}
taosInitRWLatch(&mp->pages[i].rwLatch);
mp->pages[i].frameid = i;
mp->pages[i].pgid = TDB_IVLD_PGID;
// add new page to the free list
TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free);
}
#define PGTAB_FACTOR 1.0
mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR;
mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t));
if (mp->pgtab.hashtab == NULL) {
tdbError("failed to malloc memory pool hash table");
goto _err;
}
// return
*mpp = mp;
return 0;
_err:
tdbMPoolClose(mp);
*mpp = NULL;
return -1;
}
int tdbMPoolClose(TDB_MPOOL *mp) {
if (mp) {
tfree(mp->pgtab.hashtab);
if (mp->pages) {
for (int i = 0; i < mp->npages; i++) {
tfree(mp->pages[i].p);
}
free(mp->pages);
}
free(mp);
}
return 0;
}
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) {
TDB_MPFILE *mpf;
if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) {
return -1;
}
mpf->fd = -1;
if ((mpf->fname = strdup(fname)) == NULL) {
goto _err;
}
if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) {
goto _err;
}
if (tdbGnrtFileID(fname, mpf->fileid) < 0) {
goto _err;
}
// Register current MPF to MP
tdbMPoolRegFile(mp, mpf);
*mpfp = mpf;
return 0;
_err:
tdbMPoolFileClose(mpf);
*mpfp = NULL;
return -1;
}
int tdbMPoolFileClose(TDB_MPFILE *mpf) {
if (mpf) {
if (mpf->fd > 0) {
close(mpf->fd);
}
tfree(mpf->fname);
free(mpf);
}
return 0;
}
#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \
})
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) {
// TODO
return 0;
}
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
pg_t * pagep;
TDB_MPOOL *mp;
pg_list_t *pglist;
mp = mpf->mp;
// check if the page already in pool
pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket);
pagep = TD_DLIST_HEAD(pglist);
while (pagep) {
if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) {
break;
}
pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash);
}
if (pagep) {
// page is found
// todo: pin the page and return
*(void **)addr = pagep->p;
return 0;
}
// page not found
pagep = TD_DLIST_HEAD(&mp->freeList);
if (pagep) {
// has free page
TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free);
} else {
// no free page available
tdbMPoolClockEvictPage(mp, &pagep);
if (pagep) {
if (pagep->dirty) {
// TODO: Handle dirty page eviction
}
}
}
if (pagep == NULL) {
// no available container page
return -1;
}
// load page from the disk if a container page is available
// TODO: load the page from the disk
if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) {
return -1;
}
memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN);
pagep->pgid.pgno = pgno;
pagep->dirty = 0;
pagep->pinRef = 1;
// add current page to page table
TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash);
return 0;
}
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) {
// TODO
return 0;
}
static int tdbGnrtFileID(const char *fname, uint8_t *fileid) {
struct stat statbuf;
if (stat(fname, &statbuf) < 0) {
return -1;
}
memset(fileid, 0, TDB_FILE_ID_LEN);
((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino;
((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev;
((uint64_t *)fileid)[2] = rand();
return 0;
}
#define MPF_GET_BUCKETID(fileid) \
({ \
uint64_t *tmp = (uint64_t *)fileid; \
(tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \
})
static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node);
taosWUnLockLatch(&(bktp->latch));
mpf->mp = mp;
}
static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) {
TDB_MPFILE * mpf = NULL;
mpf_bucket_t *bktp;
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid);
taosRLockLatch(&(bktp->latch));
mpf = TD_DLIST_HEAD(bktp);
while (mpf) {
if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) {
break;
}
mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node);
}
taosRUnLockLatch(&(bktp->latch));
return mpf;
}
static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) {
mpf_bucket_t *bktp;
TDB_MPFILE * tmpf;
if (mpf->mp == NULL) return;
ASSERT(mpf->mp == mp);
bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid);
taosWLockLatch(&(bktp->latch));
tmpf = TD_DLIST_HEAD(bktp);
while (tmpf) {
if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) {
TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node);
break;
}
tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node);
}
taosWUnLockLatch(&(bktp->latch));
ASSERT(tmpf == mpf);
}
static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) {
pgsize_t pgsize;
TDB_MPOOL *mp;
off_t offset;
size_t rsize;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
// TODO: use loop to read all data
rsize = pread(mpf->fd, p, pgsize, offset);
// TODO: error handle
return 0;
}
static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) {
pgsize_t pgsize;
TDB_MPOOL *mp;
off_t offset;
mp = mpf->mp;
pgsize = mp->pgsize;
offset = pgno * pgsize;
lseek(mpf->fd, offset, SEEK_SET);
// TODO: handle error
write(mpf->fd, p, pgsize);
// TODO: handle error
return 0;
}
static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) {
pg_t * pagep;
frame_id_t och;
*pagepp = NULL;
och = mp->clockHand;
do {
pagep = mp->pages + mp->clockHand;
mp->clockHand = (mp->clockHand + 1) % mp->npages;
if (pagep->pinRef == 0) {
if (pagep->rbit == 1) {
pagep->rbit = 0;
} else {
break;
}
}
if (mp->clockHand == och) {
return;
}
} while (1);
*pagepp = pagep;
}

74
source/libs/tdb/src/dmgr/tdbDiskMgr.c
View File

@ -1,74 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbDiskMgr.h"
struct STkvDiskMgr {
char * fname;
uint16_t pgsize;
FileFd fd;
pgid_t npgid;
};
#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE))
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) {
STkvDiskMgr *pDiskMgr;
pDiskMgr = malloc(sizeof(*pDiskMgr));
if (pDiskMgr == NULL) {
return -1;
}
pDiskMgr->fname = strdup(fname);
if (pDiskMgr->fname == NULL) {
free(pDiskMgr);
return -1;
}
pDiskMgr->pgsize = pgsize;
pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pDiskMgr->fd < 0) {
free(pDiskMgr->fname);
free(pDiskMgr);
return -1;
}
*ppDiskMgr = pDiskMgr;
return 0;
}
int tdmClose(STkvDiskMgr *pDiskMgr) {
close(pDiskMgr->fd);
free(pDiskMgr->fname);
free(pDiskMgr);
return 0;
}
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) {
taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET);
taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize);
return 0;
}
int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); }
int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; }

35
source/libs/tdb/src/inc/tdbBtree.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_BTREE_H_
#define _TD_TDB_BTREE_H_
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
pgid_t root; // root page number
} TDB_BTREE;
TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_BTREE_H_*/

39
source/libs/tdb/src/inc/tdbBufPool.h
View File

@ -1,39 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_BUF_POOL_H_
#define _TD_TDB_BUF_POOL_H_
#include "tdbPage.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct STdbBufPool STdbBufPool;
int tbpOpen(STdbBufPool **ppTkvBufPool);
int tbpClose(STdbBufPool *pTkvBufPool);
STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool);
int tbpDelPage(STdbBufPool *pTkvBufPool);
STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid);
void tbpFlushPages(STdbBufPool *pTkvBufPool);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_BUF_POOL_H_*/

40
source/libs/tdb/src/inc/tdbDiskMgr.h
View File

@ -1,40 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDISK_MGR_H_
#define _TD_TDISK_MGR_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tdbDef.h"
typedef struct STkvDiskMgr STkvDiskMgr;
int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize);
int tdmClose(STkvDiskMgr *pDiskMgr);
int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData);
int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData);
int tdmFlush(STkvDiskMgr *pDiskMgr);
pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDISK_MGR_H_*/

35
source/libs/tdb/src/inc/tdbHash.h
View File

@ -1,35 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_HASH_H_
#define _TD_TDB_HASH_H_
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
// TODO
} TDB_HASH;
TDB_PUBLIC int tdbInitHashDB(TDB *dbp);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_HASH_H_*/

43
source/libs/tdb/src/inc/tdbPage.h
View File

@ -1,43 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TKV_PAGE_H_
#define _TD_TKV_PAGE_H_
#include "os.h"
#include "tdbDef.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
pgid_t pgid;
int32_t pinCount;
bool idDirty;
char* pData;
} STdbPage;
typedef struct {
uint16_t dbver;
uint16_t pgsize;
uint32_t cksm;
} STdbPgHdr;
#ifdef __cplusplus
}
#endif
#endif /*_TD_TKV_PAGE_H_*/

36
source/libs/tdb/src/inc/tdbDB.h → source/libs/tdb/src/inc/tdb_db.h
View File

@ -16,38 +16,28 @@
#ifndef _TD_TDB_DB_H_
#define _TD_TDB_DB_H_
#include "tdb.h"
#include "tdbBtree.h"
#include "tdbHash.h"
#include "tdbHeap.h"
#include "tdb_mpool.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
// TODO
} TDB_MPOOL;
typedef struct {
int fd;
} TDB_FH;
typedef struct TDB TDB;
struct TDB {
pgsize_t pageSize;
tdb_db_t type;
char * fname;
char * dbname;
union {
TDB_BTREE *btree;
TDB_HASH * hash;
TDB_HEAP * heap;
} dbam; // db access method
TDB_FH * fhp; // The backup file handle
TDB_MPOOL *mph; // The memory pool handle
char * fname;
char * dbname;
TDB_MPFILE *mpf;
// union {
// TDB_BTREE *btree;
// TDB_HASH * hash;
// TDB_HEAP * heap;
// } dbam; // db access method
};
int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags);
int tdbClose(TDB *dbp, uint32_t flags);
#ifdef __cplusplus
}
#endif

27
source/libs/tdb/src/inc/tdbDef.h → source/libs/tdb/src/inc/tdb_inc.h
View File

@ -13,18 +13,30 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_DEF_H_
#define _TD_TDB_DEF_H_
#ifndef _TD_TDB_INC_H_
#define _TD_TDB_INC_H_
#include "os.h"
#include "tlist.h"
#include "tlockfree.h"
#ifdef __cplusplus
extern "C" {
#endif
// pgno_t
typedef int32_t pgno_t;
#define TDB_IVLD_PGNO ((pgno_t)-1)
// fileid
#define TDB_FILE_ID_LEN 24
// pgid_t
typedef int32_t pgid_t;
#define TDB_IVLD_PGID ((pgid_t)-1)
typedef struct {
uint8_t fileid[TDB_FILE_ID_LEN];
pgno_t pgno;
} pgid_t;
#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO};
// framd_id_t
typedef int32_t frame_id_t;
@ -34,10 +46,13 @@ typedef int32_t pgsize_t;
#define TDB_MIN_PGSIZE 512
#define TDB_MAX_PGSIZE 16384
#define TDB_DEFAULT_PGSIZE 4096
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE))
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
// tdb_log
#define tdbError(var)
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_DEF_H_*/
#endif /*_TD_TDB_INC_H_*/

94
source/libs/tdb/src/inc/tdb_mpool.h Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TDB_MPOOL_H_
#define _TD_TDB_MPOOL_H_
#include "tdb_inc.h"
#ifdef __cplusplus
extern "C" {
#endif
// Exposed handle
typedef struct TDB_MPOOL TDB_MPOOL;
typedef struct TDB_MPFILE TDB_MPFILE;
typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t;
typedef struct pg_t {
SRWLatch rwLatch;
frame_id_t frameid;
pgid_t pgid;
uint8_t dirty;
uint8_t rbit;
int32_t pinRef;
pg_free_dlist_node_t free;
pg_hash_dlist_node_t hash;
void * p;
} pg_t;
typedef TD_DLIST(pg_t) pg_list_t;
typedef struct {
SRWLatch latch;
TD_DLIST(TDB_MPFILE);
} mpf_bucket_t;
struct TDB_MPOOL {
int64_t cachesize;
pgsize_t pgsize;
int32_t npages;
pg_t * pages;
pg_list_t freeList;
frame_id_t clockHand;
struct {
int32_t nbucket;
pg_list_t *hashtab;
} pgtab; // page table, hash<pgid_t, pg_t>
struct {
#define MPF_HASH_BUCKETS 16
mpf_bucket_t buckets[MPF_HASH_BUCKETS];
} mpfht; // MPF hash table. MPFs using this MP will be put in this hash table
};
#define MP_PAGE_AT(mp, idx) (mp)->pages[idx]
typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t;
struct TDB_MPFILE {
char * fname; // file name
int fd; // fd
uint8_t fileid[TDB_FILE_ID_LEN]; // file ID
TDB_MPOOL * mp; // underlying memory pool
td_mpf_dlist_node_t node;
};
/*=================================================== Exposed apis ==================================================*/
// TDB_MPOOL
int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize);
int tdbMPoolClose(TDB_MPOOL *mp);
int tdbMPoolSync(TDB_MPOOL *mp);
// TDB_MPFILE
int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp);
int tdbMPoolFileClose(TDB_MPFILE *mpf);
int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr);
int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr);
int tdbMPoolFileSync(TDB_MPFILE *mpf);
#ifdef __cplusplus
}
#endif
#endif /*_TD_TDB_MPOOL_H_*/

54
source/libs/tdb/src/mpool/tdbBufPool.c
View File

@ -1,54 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "thash.h"
#include "tlist.h"
#include "tdbBufPool.h"
#include "tdbDiskMgr.h"
#include "tdbPage.h"
struct SFrameIdWrapper {
TD_SLIST_NODE(SFrameIdWrapper);
frame_id_t id;
};
struct STdbBufPool {
STdbPage* pages;
STkvDiskMgr* pDiskMgr;
SHashObj* pgTb; // page_id_t --> frame_id_t
TD_SLIST(SFrameIdWrapper) freeList;
pthread_mutex_t mutex;
};
typedef struct STkvLRUReplacer {
} STkvLRUReplacer;
typedef struct STkvLFUReplacer {
} STkvLFUReplacer;
typedef struct STkvCLKReplacer {
} STkvCLKReplacer;
typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t;
typedef struct STkvReplacer {
tkv_replacer_t type;
union {
STkvLRUReplacer lruRep;
STkvLFUReplacer lfuRep;
STkvCLKReplacer clkRep;
};
} STkvReplacer;

4
source/libs/tdb/test/CMakeLists.txt
View File

@ -1,3 +1,7 @@
# tdbMPoolTest
add_executable(tdbMPoolTest "tdbMPoolTest.cpp")
target_link_libraries(tdbMPoolTest tdb gtest gtest_main)
# tdbTest
add_executable(tdbTest "tdbTest.cpp")
target_link_libraries(tdbTest tdb gtest gtest_main)

31
source/libs/tdb/test/tdbMPoolTest.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "gtest/gtest.h"
#include <iostream>
#include "tdb_mpool.h"
TEST(tdb_mpool_test, test1) {
TDB_MPOOL * mp;
TDB_MPFILE *mpf;
pgno_t pgno;
void * pgdata;
// open mp
tdbMPoolOpen(&mp, 16384, 4096);
// open mpf
tdbMPoolFileOpen(&mpf, "test.db", mp);
#define TEST1_TOTAL_PAGES 100
for (int i = 0; i < TEST1_TOTAL_PAGES; i++) {
tdbMPoolFileNewPage(mpf, &pgno, pgdata);
*(pgno_t *)pgdata = i;
}
// close mpf
tdbMPoolFileClose(mpf);
// close mp
tdbMPoolClose(mp);
}

4
source/libs/tdb/test/tdbTest.cpp
View File

@ -3,8 +3,8 @@
#include "tdb.h"
TEST(tdb_api_test, tdb_create_open_close_db_test) {
int ret;
TDB *dbp;
// int ret;
// TDB *dbp;
// tdbCreateDB(&dbp, TDB_BTREE_T);

9
source/libs/transport/inc/transComm.h
View File

@ -123,9 +123,9 @@ typedef struct {
} SRpcReqContext;
typedef struct {
SRpcInfo* pRpc; // associated SRpcInfo
SEpSet epSet; // ip list provided by app
void* ahandle; // handle provided by app
SRpcInfo* pTransInst; // associated SRpcInfo
SEpSet epSet; // ip list provided by app
void* ahandle; // handle provided by app
// struct SRpcConn* pConn; // pConn allocated
tmsg_t msgType; // message type
uint8_t* pCont; // content provided by app
@ -182,7 +182,7 @@ typedef struct {
#define TRANS_RESERVE_SIZE (sizeof(STranConnCtx))
#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead) + sizeof(STransDigestMsg))
#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead))
#define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead)))
#define transContFromHead(msg) (msg + sizeof(STransMsgHead))
#define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead))
@ -201,6 +201,7 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen);
void transConnCtxDestroy(STransConnCtx* ctx);
void transFreeMsg(void* msg);
typedef struct SConnBuffer {
char* buf;
int len;

39
source/libs/transport/src/rpcMain.c
View File

@ -42,6 +42,8 @@ int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
SHashObj *tsFqdnHash;
#ifndef USE_UV
typedef struct {
@ -215,6 +217,8 @@ static void rpcInitImp(void) {
tsRpcOverhead = sizeof(SRpcReqContext);
tsRpcRefId = taosOpenRef(200, rpcFree);
tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
}
int32_t rpcInit(void) {
@ -224,6 +228,9 @@ int32_t rpcInit(void) {
void rpcCleanup(void) {
taosCloseRef(tsRpcRefId);
taosHashClear(tsFqdnHash);
taosHashCleanup(tsFqdnHash);
tsFqdnHash = NULL;
tsRpcRefId = -1;
}
@ -235,11 +242,14 @@ void *rpcOpen(const SRpcInit *pInit) {
pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) return NULL;
if (pInit->label) tstrncpy(pRpc->label, pInit->label, strlen(pInit->label));
if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label));
pRpc->connType = pInit->connType;
if (pRpc->connType == TAOS_CONN_CLIENT) {
pRpc->numOfThreads = pInit->numOfThreads;
if (pRpc->numOfThreads >= 10) {
pRpc->numOfThreads = 10;
}
} else {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
}
@ -571,7 +581,17 @@ static void rpcFreeMsg(void *msg) {
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) {
SRpcConn *pConn;
uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn);
uint32_t peerIp = 0;
uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1);
if (pPeerIp != NULL) {
peerIp = *pPeerIp;
} else {
peerIp = taosGetIpv4FromFqdn(peerFqdn);
if (peerIp != 0xFFFFFFFF) {
taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp));
}
}
if (peerIp == 0xFFFFFFFF) {
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
terrno = TSDB_CODE_RPC_FQDN_ERROR;
@ -752,8 +772,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) {
}
taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, sid,
hashstr, pConn->spi);
tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid,
sid, hashstr, pConn->spi);
}
return pConn;
@ -1340,9 +1360,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType),
pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
} else {
if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured
tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType),
pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
if (pHead->code == 0) {
pConn->secured = 1; // for success response, set link as secured
}
char ipport[40] = {0};
taosIpPort2String(pConn->peerIp, pConn->peerPort, ipport);
tDebug("%s, %s is sent to %s, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType),
ipport, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId);
}
// tTrace("connection type is: %d", pConn->connType);

114
source/libs/transport/src/transCli.c
View File

@ -30,6 +30,7 @@ typedef struct SCliConn {
char spi;
char secured;
uint64_t expireTime;
int8_t notifyCount; // timers already notify to client
} SCliConn;
typedef struct SCliMsg {
@ -72,8 +73,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co
// register timer in each thread to clear expire conn
static void clientTimeoutCb(uv_timer_t* handle);
// process data read from server, auth/decompress etc later
static void clientHandleResp(SCliConn* conn);
// check whether already read complete packet from server
static bool clientReadComplete(SConnBuffer* pBuf);
// alloc buf for read
@ -88,10 +87,15 @@ static void clientAsyncCb(uv_async_t* handle);
static void clientDestroy(uv_handle_t* handle);
static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void clientMsgDestroy(SCliMsg* pMsg);
// process data read from server, auth/decompress etc later
static void clientHandleResp(SCliConn* conn);
// handle except about conn
static void clientHandleExcept(SCliConn* conn);
// handle req from app
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void clientMsgDestroy(SCliMsg* pMsg);
static void destroyTransConnCtx(STransConnCtx* ctx);
// thread obj
static SCliThrdObj* createThrdObj();
static void destroyThrdObj(SCliThrdObj* pThrd);
@ -100,22 +104,50 @@ static void* clientThread(void* arg);
static void clientHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx;
SRpcInfo* pRpc = pCtx->pRpc;
SRpcMsg rpcMsg;
SRpcInfo* pRpc = pCtx->pTransInst;
rpcMsg.pCont = conn->readBuf.buf;
rpcMsg.contLen = conn->readBuf.len;
STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf);
pHead->code = htonl(pHead->code);
pHead->msgLen = htonl(pHead->msgLen);
SRpcMsg rpcMsg;
rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = transContFromHead(pHead);
rpcMsg.code = pHead->code;
rpcMsg.msgType = pHead->msgType;
rpcMsg.ahandle = pCtx->ahandle;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
conn->notifyCount += 1;
SCliThrdObj* pThrd = conn->hostThrd;
tfree(conn->data);
addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
// start thread's timer of conn pool if not active
if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) {
uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
free(pCtx->ip);
free(pCtx);
// impl
destroyTransConnCtx(pCtx);
}
static void clientHandleExcept(SCliConn* pConn) {
SCliMsg* pMsg = pConn->data;
STransConnCtx* pCtx = pMsg->ctx;
SRpcInfo* pRpc = pCtx->pTransInst;
transFreeMsg((pMsg->msg.pCont));
pMsg->msg.pCont = NULL;
SRpcMsg rpcMsg = {0};
rpcMsg.ahandle = pCtx->ahandle;
rpcMsg.code = -1;
// SRpcInfo* pRpc = pMsg->ctx->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
tfree(pConn->data);
pConn->notifyCount += 1;
destroyTransConnCtx(pCtx);
clientConnDestroy(pConn, true);
}
static void clientTimeoutCb(uv_timer_t* handle) {
@ -191,6 +223,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
conn->notifyCount = 0;
// list already create before
assert(plist != NULL);
QUEUE_PUSH(&plist->conn, &conn->conn);
@ -246,19 +279,21 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
if (nread > 0) {
pBuf->len += nread;
if (clientReadComplete(pBuf)) {
tDebug("alread read complete");
tDebug("conn %p read complete", conn);
clientHandleResp(conn);
} else {
tDebug("read half packet, continue to read");
tDebug("conn %p read partial packet, continue to read", conn);
}
return;
}
assert(nread <= 0);
if (nread == 0) {
tError("conn %p closed", conn);
return;
}
if (nread != UV_EOF) {
tDebug("read error %s", uv_err_name(nread));
if (nread < 0) {
tError("conn %p read error: %s", conn, uv_err_name(nread));
clientHandleExcept(conn);
}
// tDebug("Read error %s\n", uv_err_name(nread));
// uv_close((uv_handle_t*)handle, clientDestroy);
@ -282,19 +317,24 @@ static void clientDestroy(uv_handle_t* handle) {
static void clientWriteCb(uv_write_t* req, int status) {
SCliConn* pConn = req->data;
SCliMsg* pMsg = pConn->data;
transFreeMsg((pMsg->msg.pCont));
pMsg->msg.pCont = NULL;
if (status == 0) {
tDebug("data already was written on stream");
tDebug("conn %p data already was written out", pConn);
} else {
tError("failed to write: %s", uv_err_name(status));
clientConnDestroy(pConn, true);
tError("conn %p failed to write: %s", pConn, uv_err_name(status));
clientHandleExcept(pConn);
return;
}
SCliThrdObj* pThrd = pConn->hostThrd;
if (pConn->stream == NULL) {
pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream);
pConn->stream->data = pConn;
}
// if (pConn->stream == NULL) {
// pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
// uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream);
// pConn->stream->data = pConn;
//}
uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb);
// impl later
}
@ -310,30 +350,19 @@ static void clientWrite(SCliConn* pConn) {
pHead->msgLen = (int32_t)htonl((uint32_t)msgLen);
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen);
tDebug("conn %p data write out, msgType : %d, len: %d", pConn, pHead->msgType, msgLen);
uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb);
}
static void clientConnCb(uv_connect_t* req, int status) {
// impl later
SCliConn* pConn = req->data;
SCliMsg* pMsg = pConn->data;
STransConnCtx* pCtx = pMsg->ctx;
SRpcInfo* pRpc = pCtx->pRpc;
if (status != 0) {
// tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status));
tError("failed to connect server, errmsg: %s", uv_strerror(status));
// call user fp later
SRpcMsg rpcMsg;
rpcMsg.ahandle = pCtx->ahandle;
// SRpcInfo* pRpc = pMsg->ctx->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, NULL);
clientConnDestroy(pConn, true);
// uv_close((uv_handle_t*)req->handle, clientDestroy);
tError("conn %p failed to connect server: %s", pConn, uv_strerror(status));
clientHandleExcept(pConn);
return;
}
tDebug("conn %p create", pConn);
assert(pConn->stream == req->handle);
clientWrite(pConn);
@ -349,6 +378,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) {
// impl later
tDebug("conn %p get from conn pool", conn);
conn->data = pMsg;
conn->writeReq->data = conn;
@ -462,6 +492,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) {
free(pThrd->loop);
free(pThrd);
}
static void destroyTransConnCtx(STransConnCtx* ctx) {
if (ctx != NULL) {
free(ctx->ip);
}
free(ctx);
}
//
void taosCloseClient(void* arg) {
// impl later
@ -472,7 +509,6 @@ void taosCloseClient(void* arg) {
free(cli->pThreadObj);
free(cli);
}
void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) {
// impl later
char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]);
@ -487,7 +523,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t*
STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx));
pCtx->pRpc = (SRpcInfo*)shandle;
pCtx->pTransInst = (SRpcInfo*)shandle;
pCtx->ahandle = pMsg->ahandle;
pCtx->msgType = pMsg->msgType;
pCtx->ip = strdup(ip);

7
source/libs/transport/src/transComm.c
View File

@ -191,4 +191,11 @@ void transConnCtxDestroy(STransConnCtx* ctx) {
free(ctx->ip);
free(ctx);
}
void transFreeMsg(void* msg) {
if (msg == NULL) {
return;
}
free((char*)msg - sizeof(STransMsgHead));
}
#endif

47
source/libs/transport/src/transSrv.c
View File

@ -16,6 +16,7 @@
#ifdef USE_UV
#include "transComm.h"
typedef struct SConn {
uv_tcp_t* pTcp;
uv_write_t* pWriter;
@ -26,7 +27,6 @@ typedef struct SConn {
int ref;
int persist; // persist connection or not
SConnBuffer connBuf; // read buf,
int count;
int inType;
void* pTransInst; // rpc init
void* ahandle; //
@ -226,7 +226,7 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) {
tDebug("%p timeout since no activity", conn);
}
static void uvProcessData(SConn* pConn) {
static void uvHandleReq(SConn* pConn) {
SRecvInfo info;
SRecvInfo* p = &info;
SConnBuffer* pBuf = &pConn->connBuf;
@ -271,6 +271,7 @@ static void uvProcessData(SConn* pConn) {
rpcMsg.ahandle = NULL;
rpcMsg.handle = pConn;
pConn->ref++;
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
// uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth
@ -283,20 +284,23 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
SConnBuffer* pBuf = &conn->connBuf;
if (nread > 0) {
pBuf->len += nread;
tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread);
tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread);
if (readComplete(pBuf)) {
tDebug("alread read complete packet");
uvProcessData(conn);
tDebug("conn %p alread read complete packet", conn);
uvHandleReq(conn);
} else {
tDebug("read half packet, continue to read");
tDebug("conn %p read partial packet, continue to read", conn);
}
return;
}
if (nread == 0) {
tDebug("conn %p except read", conn);
// destroyConn(conn, true);
return;
}
if (nread != UV_EOF) {
tDebug("read error %s", uv_err_name(nread));
tDebug("conn %p read error: %s", conn, uv_err_name(nread));
destroyConn(conn, true);
}
}
void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
@ -306,7 +310,8 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b
void uvOnTimeoutCb(uv_timer_t* handle) {
// opt
tDebug("time out");
SConn* pConn = handle->data;
tDebug("conn %p time out", pConn);
}
void uvOnWriteCb(uv_write_t* req, int status) {
@ -316,10 +321,14 @@ void uvOnWriteCb(uv_write_t* req, int status) {
buf->len = 0;
memset(buf->buf, 0, buf->cap);
buf->left = -1;
SRpcMsg* pMsg = &conn->sendMsg;
transFreeMsg(pMsg->pCont);
if (status == 0) {
tDebug("data already was written on stream");
tDebug("conn %p data already was written on stream", conn);
} else {
tDebug("failed to write data, %s", uv_err_name(status));
tDebug("conn %p failed to write data, %s", conn, uv_err_name(status));
destroyConn(conn, true);
}
// opt
@ -334,7 +343,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) {
// impl later;
tDebug("prepare to send back");
tDebug("conn %p prepare to send resp", conn);
SRpcMsg* pMsg = &conn->sendMsg;
if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0);
@ -427,6 +436,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
assert(pending == UV_TCP);
SConn* pConn = createConn();
pConn->pTransInst = pThrd->pTransInst;
/* init conn timer*/
pConn->pTimer = malloc(sizeof(uv_timer_t));
@ -448,7 +458,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) {
uv_os_fd_t fd;
uv_fileno((const uv_handle_t*)pConn->pTcp, &fd);
tDebug("new connection created: %d", fd);
tDebug("conn %p created, fd: %d", pConn, fd);
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
} else {
tDebug("failed to create new connection");
@ -515,19 +525,19 @@ void* workerThread(void* arg) {
static SConn* createConn() {
SConn* pConn = (SConn*)calloc(1, sizeof(SConn));
++pConn->ref;
return pConn;
}
static void connCloseCb(uv_handle_t* handle) {
// impl later
//
}
static void destroyConn(SConn* conn, bool clear) {
if (conn == NULL) {
return;
}
if (--conn->ref == 0) {
return;
}
if (clear) {
uv_handle_t handle = *((uv_handle_t*)conn->pTcp);
uv_close(&handle, NULL);
uv_close((uv_handle_t*)conn->pTcp, NULL);
}
uv_timer_stop(conn->pTimer);
free(conn->pTimer);
@ -646,6 +656,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) {
pthread_mutex_lock(&pThrd->connMtx);
QUEUE_PUSH(&pThrd->conn, &pConn->queue);
pthread_mutex_unlock(&pThrd->connMtx);
tDebug("conn %p start to send resp", pConn);
uv_async_send(pConn->pWorkerAsync);
}

2
source/libs/transport/test/rclient.c
View File

@ -63,7 +63,7 @@ static void *sendRequest(void *param) {
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
// tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem);
tDebug("recv response");
tDebug("recv response succefully");
// usleep(100000000);
}

1
source/util/src/terror.c
View File

@ -248,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg")
// mnode-trans
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists")

141
source/util/src/tqueue.c
View File

@ -13,35 +13,36 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taoserror.h"
#define _DEFAULT_SOURCE
#include "tqueue.h"
#include "taoserror.h"
#include "ulog.h"
typedef struct STaosQnode STaosQnode;
typedef struct STaosQnode {
STaosQnode *next;
STaosQueue *queue;
char item[];
} STaosQnode;
typedef struct STaosQueue {
int32_t itemSize;
int32_t numOfItems;
STaosQnode *head;
STaosQnode *tail;
STaosQueue *next; // for queue set
STaosQset *qset; // for queue set
void *ahandle; // for queue set
FProcessItem itemFp;
FProcessItems itemsFp;
int32_t threadId;
STaosQnode * head;
STaosQnode * tail;
STaosQueue * next; // for queue set
STaosQset * qset; // for queue set
void * ahandle; // for queue set
FItem itemFp;
FItems itemsFp;
pthread_mutex_t mutex;
} STaosQueue;
typedef struct STaosQset {
STaosQueue *head;
STaosQueue *current;
STaosQueue * head;
STaosQueue * current;
pthread_mutex_t mutex;
int32_t numOfQueues;
int32_t numOfItems;
@ -56,19 +57,23 @@ typedef struct STaosQall {
} STaosQall;
STaosQueue *taosOpenQueue() {
STaosQueue *queue = calloc(sizeof(STaosQueue), 1);
STaosQueue *queue = calloc(1, sizeof(STaosQueue));
if (queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pthread_mutex_init(&queue->mutex, NULL);
if (pthread_mutex_init(&queue->mutex, NULL) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
uTrace("queue:%p is opened", queue);
queue->threadId = -1;
uDebug("queue:%p is opened", queue);
return queue;
}
void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp) {
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) {
if (queue == NULL) return;
queue->itemFp = itemFp;
queue->itemsFp = itemsFp;
@ -77,7 +82,7 @@ void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsF
void taosCloseQueue(STaosQueue *queue) {
if (queue == NULL) return;
STaosQnode *pTemp;
STaosQset *qset;
STaosQset * qset;
pthread_mutex_lock(&queue->mutex);
STaosQnode *pNode = queue->head;
@ -85,7 +90,9 @@ void taosCloseQueue(STaosQueue *queue) {
qset = queue->qset;
pthread_mutex_unlock(&queue->mutex);
if (queue->qset) taosRemoveFromQset(qset, queue);
if (queue->qset) {
taosRemoveFromQset(qset, queue);
}
while (pNode) {
pTemp = pNode;
@ -96,7 +103,7 @@ void taosCloseQueue(STaosQueue *queue) {
pthread_mutex_destroy(&queue->mutex);
free(queue);
uTrace("queue:%p is closed", queue);
uDebug("queue:%p is closed", queue);
}
bool taosQueueEmpty(STaosQueue *queue) {
@ -120,19 +127,23 @@ int32_t taosQueueSize(STaosQueue *queue) {
}
void *taosAllocateQitem(int32_t size) {
STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1);
STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size);
if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
if (pNode == NULL) return NULL;
uTrace("item:%p, node:%p is allocated", pNode->item, pNode);
return (void *)pNode->item;
}
void taosFreeQitem(void *param) {
if (param == NULL) return;
void taosFreeQitem(void *pItem) {
if (pItem == NULL) return;
char *temp = (char *)param;
char *temp = pItem;
temp -= sizeof(STaosQnode);
uTrace("item:%p, node:%p is freed", param, temp);
uTrace("item:%p, node:%p is freed", pItem, temp);
free(temp);
}
@ -175,7 +186,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) {
queue->numOfItems--;
if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1);
code = 1;
uDebug("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems);
uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems);
}
pthread_mutex_unlock(&queue->mutex);
@ -183,7 +194,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) {
return code;
}
STaosQall *taosAllocateQall() { return calloc(sizeof(STaosQall), 1); }
STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); }
void taosFreeQall(STaosQall *qall) { free(qall); }
@ -238,7 +249,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) {
void taosResetQitems(STaosQall *qall) { qall->current = qall->start; }
STaosQset *taosOpenQset() {
STaosQset *qset = (STaosQset *)calloc(sizeof(STaosQset), 1);
STaosQset *qset = calloc(sizeof(STaosQset), 1);
if (qset == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@ -247,7 +258,7 @@ STaosQset *taosOpenQset() {
pthread_mutex_init(&qset->mutex, NULL);
tsem_init(&qset->sem, 0, 0);
uTrace("qset:%p is opened", qset);
uDebug("qset:%p is opened", qset);
return qset;
}
@ -268,7 +279,7 @@ void taosCloseQset(STaosQset *qset) {
pthread_mutex_destroy(&qset->mutex);
tsem_destroy(&qset->sem);
free(qset);
uTrace("qset:%p is closed", qset);
uDebug("qset:%p is closed", qset);
}
// tsem_post 'qset->sem', so that reader threads waiting for it
@ -338,12 +349,12 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) {
pthread_mutex_unlock(&qset->mutex);
uTrace("queue:%p is removed from qset:%p", queue, qset);
uDebug("queue:%p is removed from qset:%p", queue, qset);
}
int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; }
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp) {
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp) {
STaosQnode *pNode = NULL;
int32_t code = 0;
@ -365,6 +376,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP
*ppItem = pNode->item;
if (ahandle) *ahandle = queue->ahandle;
if (itemFp) *itemFp = queue->itemFp;
queue->head = pNode->next;
if (queue->head == NULL) queue->tail = NULL;
queue->numOfItems--;
@ -382,7 +394,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP
return code;
}
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp) {
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp) {
STaosQueue *queue;
int32_t code = 0;
@ -411,7 +423,9 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand
queue->tail = NULL;
queue->numOfItems = 0;
atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems);
for (int32_t j = 1; j < qall->numOfItems; ++j) tsem_wait(&qset->sem);
for (int32_t j = 1; j < qall->numOfItems; ++j) {
tsem_wait(&qset->sem);
}
}
pthread_mutex_unlock(&queue->mutex);
@ -423,6 +437,65 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand
return code;
}
int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId) {
STaosQnode *pNode = NULL;
int32_t code = -1;
tsem_wait(&qset->sem);
pthread_mutex_lock(&qset->mutex);
for (int32_t i = 0; i < qset->numOfQueues; ++i) {
if (qset->current == NULL) qset->current = qset->head;
STaosQueue *queue = qset->current;
if (queue) qset->current = queue->next;
if (queue == NULL) break;
if (queue->head == NULL) continue;
if (queue->threadId != -1 && queue->threadId != threadId) {
code = 0;
continue;
}
pthread_mutex_lock(&queue->mutex);
if (queue->head) {
pNode = queue->head;
pNode->queue = queue;
queue->threadId = threadId;
*ppItem = pNode->item;
if (ahandle) *ahandle = queue->ahandle;
if (itemFp) *itemFp = queue->itemFp;
queue->head = pNode->next;
if (queue->head == NULL) queue->tail = NULL;
queue->numOfItems--;
atomic_sub_fetch_32(&qset->numOfItems, 1);
code = 1;
uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems);
}
pthread_mutex_unlock(&queue->mutex);
if (pNode) break;
}
pthread_mutex_unlock(&qset->mutex);
return code;
}
void taosResetQsetThread(STaosQset *qset, void *pItem) {
if (pItem == NULL) return;
STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode));
pthread_mutex_lock(&qset->mutex);
pNode->queue->threadId = -1;
for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) {
tsem_post(&qset->sem);
}
pthread_mutex_unlock(&qset->mutex);
}
int32_t taosGetQueueItemsNumber(STaosQueue *queue) {
if (!queue) return 0;

164
source/util/src/tworker.c
View File

@ -14,38 +14,46 @@
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tqueue.h"
#include "tworker.h"
#include "taoserror.h"
#include "ulog.h"
typedef void* (*ThreadFp)(void *param);
typedef void *(*ThreadFp)(void *param);
int32_t tWorkerInit(SWorkerPool *pool) {
int32_t tQWorkerInit(SQWorkerPool *pool) {
pool->qset = taosOpenQset();
pool->workers = calloc(sizeof(SWorker), pool->max);
pthread_mutex_init(&pool->mutex, NULL);
for (int i = 0; i < pool->max; ++i) {
SWorker *worker = pool->workers + i;
pool->workers = calloc(sizeof(SQWorker), pool->max);
if (pool->workers == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (pthread_mutex_init(&pool->mutex, NULL)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < pool->max; ++i) {
SQWorker *worker = pool->workers + i;
worker->id = i;
worker->pool = pool;
}
uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
return 0;
}
void tWorkerCleanup(SWorkerPool *pool) {
for (int i = 0; i < pool->max; ++i) {
SWorker *worker = pool->workers + i;
void tQWorkerCleanup(SQWorkerPool *pool) {
for (int32_t i = 0; i < pool->max; ++i) {
SQWorker *worker = pool->workers + i;
if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) {
taosQsetThreadResume(pool->qset);
}
}
for (int i = 0; i < pool->max; ++i) {
SWorker *worker = pool->workers + i;
for (int32_t i = 0; i < pool->max; ++i) {
SQWorker *worker = pool->workers + i;
if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) {
pthread_join(worker->thread, NULL);
@ -56,15 +64,15 @@ void tWorkerCleanup(SWorkerPool *pool) {
taosCloseQset(pool->qset);
pthread_mutex_destroy(&pool->mutex);
uInfo("worker:%s is closed", pool->name);
uDebug("worker:%s is closed", pool->name);
}
static void *tWorkerThreadFp(SWorker *worker) {
SWorkerPool *pool = worker->pool;
FProcessItem fp = NULL;
static void *tQWorkerThreadFp(SQWorker *worker) {
SQWorkerPool *pool = worker->pool;
FItem fp = NULL;
void *msg = NULL;
void *ahandle = NULL;
void * msg = NULL;
void * ahandle = NULL;
int32_t code = 0;
taosBlockSIGPIPE();
@ -77,7 +85,7 @@ static void *tWorkerThreadFp(SWorker *worker) {
break;
}
if (fp) {
if (fp != NULL) {
(*fp)(ahandle, msg);
}
}
@ -85,11 +93,12 @@ static void *tWorkerThreadFp(SWorker *worker) {
return NULL;
}
STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) {
STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) {
pthread_mutex_lock(&pool->mutex);
STaosQueue *queue = taosOpenQueue();
if (queue == NULL) {
pthread_mutex_unlock(&pool->mutex);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -99,14 +108,18 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp)
// spawn a thread to process queue
if (pool->num < pool->max) {
do {
SWorker *worker = pool->workers + pool->num;
SQWorker *worker = pool->workers + pool->num;
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWorkerThreadFp, worker) != 0) {
if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) {
uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno));
taosCloseQueue(queue);
terrno = TSDB_CODE_OUT_OF_MEMORY;
queue = NULL;
break;
}
pthread_attr_destroy(&thAttr);
@ -121,19 +134,73 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp)
return queue;
}
void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue) {
STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp);
}
void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) {
taosCloseQueue(queue);
uDebug("worker:%s, queue:%p is freed", pool->name, queue);
}
int32_t tMWorkerInit(SMWorkerPool *pool) {
pool->nextId = 0;
pool->workers = calloc(sizeof(SMWorker), pool->max);
if (pool->workers == NULL) return -1;
int32_t tFWorkerInit(SFWorkerPool *pool) { return tQWorkerInit((SQWorkerPool *)pool); }
void tFWorkerCleanup(SFWorkerPool *pool) { tQWorkerCleanup(pool); }
static void *tFWorkerThreadFp(SQWorker *worker) {
SQWorkerPool *pool = worker->pool;
FItem fp = NULL;
void * msg = NULL;
void * ahandle = NULL;
int32_t code = 0;
taosBlockSIGPIPE();
setThreadName(pool->name);
uDebug("worker:%s:%d is running", pool->name, worker->id);
while (1) {
code = taosReadQitemFromQsetByThread(pool->qset, (void **)&msg, &ahandle, &fp, worker->id);
if (code < 0) {
uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset);
break;
} else if (code == 0) {
// uTrace("worker:%s:%d qset:%p, got no message and continue", pool->name, worker->id, pool->qset);
continue;
}
if (fp != NULL) {
(*fp)(ahandle, msg);
}
taosResetQsetThread(pool->qset, msg);
}
return NULL;
}
STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tFWorkerThreadFp);
}
void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); }
int32_t tWWorkerInit(SWWorkerPool *pool) {
pool->nextId = 0;
pool->workers = calloc(sizeof(SWWorker), pool->max);
if (pool->workers == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (pthread_mutex_init(&pool->mutex, NULL) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pthread_mutex_init(&pool->mutex, NULL);
for (int32_t i = 0; i < pool->max; ++i) {
SMWorker *worker = pool->workers + i;
SWWorker *worker = pool->workers + i;
worker->id = i;
worker->qall = NULL;
worker->qset = NULL;
@ -144,16 +211,18 @@ int32_t tMWorkerInit(SMWorkerPool *pool) {
return 0;
}
void tMWorkerCleanup(SMWorkerPool *pool) {
void tWWorkerCleanup(SWWorkerPool *pool) {
for (int32_t i = 0; i < pool->max; ++i) {
SMWorker *worker = pool->workers + i;
SWWorker *worker = pool->workers + i;
if (taosCheckPthreadValid(worker->thread)) {
if (worker->qset) taosQsetThreadResume(worker->qset);
if (worker->qset) {
taosQsetThreadResume(worker->qset);
}
}
}
for (int32_t i = 0; i < pool->max; ++i) {
SMWorker *worker = pool->workers + i;
SWWorker *worker = pool->workers + i;
if (taosCheckPthreadValid(worker->thread)) {
pthread_join(worker->thread, NULL);
taosFreeQall(worker->qall);
@ -167,12 +236,12 @@ void tMWorkerCleanup(SMWorkerPool *pool) {
uInfo("worker:%s is closed", pool->name);
}
static void *tWriteWorkerThreadFp(SMWorker *worker) {
SMWorkerPool *pool = worker->pool;
FProcessItems fp = NULL;
static void *tWWorkerThreadFp(SWWorker *worker) {
SWWorkerPool *pool = worker->pool;
FItems fp = NULL;
void *msg = NULL;
void *ahandle = NULL;
void * msg = NULL;
void * ahandle = NULL;
int32_t numOfMsgs = 0;
int32_t qtype = 0;
@ -187,7 +256,7 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) {
break;
}
if (fp) {
if (fp != NULL) {
(*fp)(ahandle, worker->qall, numOfMsgs);
}
}
@ -195,13 +264,14 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) {
return NULL;
}
STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) {
STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) {
pthread_mutex_lock(&pool->mutex);
SMWorker *worker = pool->workers + pool->nextId;
SWWorker *worker = pool->workers + pool->nextId;
STaosQueue *queue = taosOpenQueue();
if (queue == NULL) {
pthread_mutex_unlock(&pool->mutex);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
@ -221,17 +291,19 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems
taosCloseQset(worker->qset);
taosCloseQueue(queue);
pthread_mutex_unlock(&pool->mutex);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWriteWorkerThreadFp, worker) != 0) {
if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) {
uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno));
taosFreeQall(worker->qall);
taosCloseQset(worker->qset);
taosCloseQueue(queue);
terrno = TSDB_CODE_OUT_OF_MEMORY;
queue = NULL;
} else {
uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max);
@ -250,7 +322,7 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems
return queue;
}
void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue) {
void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) {
taosCloseQueue(queue);
uDebug("worker:%s, queue:%p is freed", pool->name, queue);
}

4
source/util/test/CMakeLists.txt
View File

@ -13,12 +13,12 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread gcov)
TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread)
LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/cacheTest.cpp)
LIST(APPEND SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hashTest.cpp)
ADD_EXECUTABLE(hashTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(hashTest util common os gtest pthread gcov)
TARGET_LINK_LIBRARIES(hashTest util common os gtest pthread)
LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c)
ADD_EXECUTABLE(trefTest ${BIN_SRC})

38
tests/script/sh/dockerbuilder/addDnodeToCluster.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
#
# deploy test cluster
set -e
#set -x
dnodeNumber=1
serverPort=6030
while getopts "hn:p:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
p)
serverPort=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
for ((i=2; i<=${dnodeNumber}; i++)); do
taos -s "create dnode node${i} port ${serverPort};" ||:
echo "create dnode node${i} port ${serverPort};"
done

46
tests/script/sh/dockerbuilder/addFqdnToHost.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
#
# deploy test cluster
set -e
#set -x
dnodeNumber=1
subnet="172.33.0.0/16"
while getopts "hn:s:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
s)
subnet=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
echo " -s [ subnet] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
addFqdnToHosts() {
index=$1
ipPrefix=$2
let ipIndex=index+1
echo "${ipPrefix}.${ipIndex} node${i}" >> /etc/hosts
}
ipPrefix=${subnet%.*}
for ((i=1; i<=${dnodeNumber}; i++)); do
addFqdnToHosts ${i} ${ipPrefix}
done

Some files were not shown because too many files have changed in this diff Show More