xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-05-18 20:32:20 +08:00
parent 97a20ee86b 9bbd91e999
commit b2fc04b8cc
57 changed files with 1296 additions and 891 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/tmsgcb.h
View File

@ -60,9 +60,9 @@ typedef struct {
ReportStartup reportStartupFp; ReportStartup reportStartupFp;
} SMsgCb; } SMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); void tmsgSetDefault(const SMsgCb* msgcb);
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg);
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
void tmsgSendRsp(SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg);
void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);

2
include/common/ttypes.h
View File

@ -179,6 +179,8 @@ typedef struct {
} \ } \
} while (0) } while (0)
//TODO: use varchar(0) to represent NULL type
#define IS_NULL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_NULL)
#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT)
#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)

4
include/libs/executor/executor.h
View File

@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
* @param isAdd * @param isAdd
* @return * @return
*/ */
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
/** /**
* Create the exec task object according to task json * Create the exec task object according to task json
@ -169,7 +169,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
include/libs/nodes/nodes.h
View File

@ -208,6 +208,7 @@ typedef enum ENodeType {
QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_SORT,
QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_FILL,
QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,

15
source/client/src/tmq.c
View File

@ -830,10 +830,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
} }
// init hb timer // init hb timer
if (tmq->hbTimer == NULL) {
tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer);
}
// init auto commit timer // init auto commit timer
if (tmq->autoCommit) { if (tmq->autoCommit && tmq->commitTimer == NULL) {
tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer);
} }
@ -1456,9 +1458,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
if (tmq->status == TMQ_CONSUMER_STATUS__READY) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
if (rsp == TMQ_RESP_ERR__SUCCESS) {
// TODO: free resources
return TMQ_RESP_ERR__SUCCESS;
} else {
return TMQ_RESP_ERR__FAIL;
}
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst); tmq_list_destroy(lst);
if (rsp == TMQ_RESP_ERR__SUCCESS) { if (rsp == TMQ_RESP_ERR__SUCCESS) {
// TODO: free resources // TODO: free resources
return TMQ_RESP_ERR__SUCCESS; return TMQ_RESP_ERR__SUCCESS;

28
source/common/src/tmsgcb.c
View File

@ -17,46 +17,46 @@
#include "tmsgcb.h" #include "tmsgcb.h"
#include "taoserror.h" #include "taoserror.h"
static SMsgCb tsDefaultMsgCb; static SMsgCb defaultMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) { int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
PutToQueueFp fp = pMsgCb->queueFps[qtype]; PutToQueueFp fp = msgcb->queueFps[qtype];
return (*fp)(pMsgCb->mgmt, pMsg); return (*fp)(msgcb->mgmt, pMsg);
} }
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) {
GetQueueSizeFp fp = pMsgCb->qsizeFp; GetQueueSizeFp fp = msgcb->qsizeFp;
return (*fp)(pMsgCb->mgmt, vgId, qtype); return (*fp)(msgcb->mgmt, vgId, qtype);
} }
int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) {
SendReqFp fp = tsDefaultMsgCb.sendReqFp; SendReqFp fp = defaultMsgCb.sendReqFp;
return (*fp)(epSet, pMsg); return (*fp)(epSet, pMsg);
} }
void tmsgSendRsp(SRpcMsg* pMsg) { void tmsgSendRsp(SRpcMsg* pMsg) {
SendRspFp fp = tsDefaultMsgCb.sendRspFp; SendRspFp fp = defaultMsgCb.sendRspFp;
return (*fp)(pMsg); return (*fp)(pMsg);
} }
void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) {
SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; SendRedirectRspFp fp = defaultMsgCb.sendRedirectRspFp;
(*fp)(pMsg, pNewEpSet); (*fp)(pMsg, pNewEpSet);
} }
void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) {
RegisterBrokenLinkArgFp fp = tsDefaultMsgCb.registerBrokenLinkArgFp; RegisterBrokenLinkArgFp fp = defaultMsgCb.registerBrokenLinkArgFp;
(*fp)(pMsg); (*fp)(pMsg);
} }
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) {
ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; ReleaseHandleFp fp = defaultMsgCb.releaseHandleFp;
(*fp)(pHandle, type); (*fp)(pHandle, type);
} }
void tmsgReportStartup(const char* name, const char* desc) { void tmsgReportStartup(const char* name, const char* desc) {
ReportStartup fp = tsDefaultMsgCb.reportStartupFp; ReportStartup fp = defaultMsgCb.reportStartupFp;
(*fp)(name, desc); (*fp)(name, desc);
} }

2
source/dnode/mgmt/mgmt_bnode/src/bmWorker.c
View File

@ -63,7 +63,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
bmSendRsp(pMsg, code); bmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }

2
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -86,7 +86,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527};
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
dTrace("send status msg to mnode, app:%p", rpcMsg.info.ahandle); dTrace("send status msg to mnode");
SEpSet epSet = {0}; SEpSet epSet = {0};
dmGetMnodeEpSet(pMgmt->pData, &epSet); dmGetMnodeEpSet(pMgmt->pData, &epSet);

2
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -160,7 +160,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }

37
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -29,7 +29,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) {
static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1; int32_t code = -1;
dTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); dTrace("msg:%p, get from mnode queue", pMsg);
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_DND_ALTER_MNODE: case TDMT_DND_ALTER_MNODE:
@ -51,7 +51,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
mmSendRsp(pMsg, code); mmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -73,7 +73,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -84,13 +84,20 @@ static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
return 0; return 0;
} }
int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); } int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg);
}
int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); } int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg);
}
int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); } int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg);
}
int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
} }
int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -101,25 +108,25 @@ static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc)
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) return -1; if (pMsg == NULL) return -1;
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pMsg);
} }
int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pMsg);
} }
int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc); return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg);
} }
int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); } int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
SSingleWorkerCfg qCfg = { SSingleWorkerCfg qCfg = {

8
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -44,7 +44,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -60,7 +60,7 @@ static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -76,7 +76,7 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
qmSendRsp(pMsg, code); qmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -105,7 +105,7 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR
return -1; return -1;
} }
dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;

10
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -44,7 +44,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
smSendRsp(pMsg, code); smSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -166,7 +166,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -174,7 +174,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->monitorWorker; SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -187,7 +187,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -195,7 +195,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->sharedWorker; SSingleWorker *pWorker = &pMgmt->sharedWorker;
dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -51,7 +51,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
break; break;
default: default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dError("msg:%p, not processed in vnode-mgmt/monitor queue", pMsg); dError("msg:%p, not processed in vnode queue", pMsg);
} }
if (msgType & 1u) { if (msgType & 1u) {
@ -59,7 +59,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
} }
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -67,13 +67,13 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-query queue", pMsg); dTrace("msg:%p, get from vnode-query queue", pMsg);
int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -82,13 +82,13 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); dTrace("msg:%p, get from vnode-fetch queue", pMsg);
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -96,7 +96,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
SRpcMsg rsp;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
if (pArray == NULL) { if (pArray == NULL) {
@ -108,7 +107,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
SRpcMsg *pMsg = NULL; SRpcMsg *pMsg = NULL;
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
dTrace("msg:%p, will be processed in vnode-write queue", pMsg); dTrace("msg:%p, get from vnode-write queue", pMsg);
if (taosArrayPush(pArray, &pMsg) == NULL) { if (taosArrayPush(pArray, &pMsg) == NULL) {
dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); dTrace("msg:%p, failed to process since %s", pMsg, terrstr());
vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
@ -116,21 +115,12 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
} }
for (int i = 0; i < taosArrayGetSize(pArray); i++) { for (int i = 0; i < taosArrayGetSize(pArray); i++) {
SRpcMsg *pMsg; SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
SRpcMsg *pRpc; SRpcMsg rsp = {.info = pMsg->info, .pCont = NULL, .contLen = 0};
pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false);
pRpc = pMsg;
rsp.info = pRpc->info;
rsp.pCont = NULL;
rsp.contLen = 0;
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
// rsp.code = TSDB_CODE_SYN_NOT_LEADER; dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId);
// tmsgSendRsp(&rsp);
dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl)));
rsp.code = TSDB_CODE_RPC_REDIRECT; rsp.code = TSDB_CODE_RPC_REDIRECT;
SEpSet newEpSet; SEpSet newEpSet;
syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet);
@ -181,7 +171,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
// apply data into tsdb // apply data into tsdb
if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
rsp.code = terrno; rsp.code = terrno;
dTrace("vnodeProcessWriteReq error, code:%d", terrno); dTrace("msg:%p, process write error since %s", pMsg, terrstr());
} }
syncApplyMsgDestroy(pSyncApplyMsg); syncApplyMsgDestroy(pSyncApplyMsg);
@ -215,7 +205,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
SRpcMsg rsp = {0}; SRpcMsg rsp = {0};
rsp.code = terrno; rsp.code = terrno;
rsp.info = pMsg->info; rsp.info = pMsg->info;
dTrace("vmProcessSyncQueue error, code:%d", terrno); dTrace("msg:%p, process sync queue error since code:%s", pMsg, terrstr());
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
} }
@ -232,13 +222,13 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
dTrace("msg:%p, will be processed in vnode-merge queue", pMsg); dTrace("msg:%p, get from vnode-merge queue", pMsg);
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) { if (code != 0) {
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
vmSendRsp(pMsg, code); vmSendRsp(pMsg, code);
dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -255,29 +245,29 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); dError("vgId:%d, failed to put msg:%p into vnode-queue since %s", pHead->vgId, pMsg, terrstr());
return terrno != 0 ? terrno : -1; return terrno != 0 ? terrno : -1;
} }
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-query queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-fetch queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-write queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-sync queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pSyncQ, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, type:%s will be written into vnode-merge queue", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
default: default:
@ -312,7 +302,7 @@ int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->mgmtWorker; SSingleWorker *pWorker = &pMgmt->mgmtWorker;
dTrace("msg:%p, will be put into vnode-mgmt queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into vnode-mgmt worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -320,7 +310,7 @@ int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->monitorWorker; SSingleWorker *pWorker = &pMgmt->monitorWorker;
dTrace("msg:%p, will be put into vnode-monitor queue, worker:%s", pMsg, pWorker->name); dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
return 0; return 0;
} }
@ -335,32 +325,30 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q
int32_t code = 0; int32_t code = 0;
if (pMsg != NULL) { if (pMsg != NULL) {
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
// if (pMsg->handle != NULL) assert(pMsg->refId != 0);
switch (qtype) { switch (qtype) {
case WRITE_QUEUE: case WRITE_QUEUE:
dTrace("msg:%p, will be put into vnode-write queue", pMsg); dTrace("msg:%p, create and put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pWriteQ, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg);
break; break;
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg); dTrace("msg:%p, create and put into vnode-query queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pQueryQ, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); dTrace("msg:%p, create and put into vnode-fetch queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pFetchQ, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case APPLY_QUEUE: case APPLY_QUEUE:
dTrace("msg:%p, will be put into vnode-apply queue", pMsg); dTrace("msg:%p, create and put into vnode-apply queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pApplyQ, pMsg); taosWriteQitem(pVnode->pApplyQ, pMsg);
break; break;
case MERGE_QUEUE: case MERGE_QUEUE:
dTrace("msg:%p, will be put into vnode-merge queue", pMsg); dTrace("msg:%p, create and put into vnode-merge queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dTrace("msg:%p, will be put into vnode-sync queue", pMsg); dTrace("msg:%p, create and put into vnode-sync queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
taosWriteQitem(pVnode->pSyncQ, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg);
break; break;
default: default:

17
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -274,25 +274,24 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus)
} }
} }
void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) {
dDebug("net test req is received"); dDebug("msg:%p, net test req will be processed", pMsg);
SRpcMsg rsp = {.code = 0, .info = pReq->info}; SRpcMsg rsp = {.code = 0, .info = pMsg->info};
rsp.pCont = rpcMallocCont(pReq->contLen); rsp.pCont = rpcMallocCont(pMsg->contLen);
if (rsp.pCont == NULL) { if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY; rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else { } else {
rsp.contLen = pReq->contLen; rsp.contLen = pMsg->contLen;
} }
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
} }
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) { void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) {
dDebug("server startup status req is received"); dDebug("msg:%p, server startup status req will be processed", pMsg);
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
dmGetServerStartupStatus(pDnode, &statusRsp); dmGetServerStartupStatus(pDnode, &statusRsp);
SRpcMsg rspMsg = {.info = pReq->info}; SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) { if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;

13
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -37,6 +37,7 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) {
dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
return -1; return -1;
} }
dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize); dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize);
return 0; return 0;
} }
@ -60,7 +61,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) {
taosIgnSignal(SIGCHLD); taosIgnSignal(SIGCHLD);
pWrapper->proc.pid = pid; pWrapper->proc.pid = pid;
dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); dInfo("node:%s, continue running in new pid:%d", pWrapper->name, pid);
return 0; return 0;
} }
@ -77,7 +78,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) { if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) {
tmsgSetDefaultMsgCb(&input.msgCb); tmsgSetDefault(&input.msgCb);
} }
if (OnlyInSingleProc(pWrapper)) { if (OnlyInSingleProc(pWrapper)) {
@ -177,11 +178,11 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
if (OnlyInParentProc(pWrapper)) { if (OnlyInParentProc(pWrapper)) {
int32_t pid = pWrapper->proc.pid; int32_t pid = pWrapper->proc.pid;
if (pid > 0 && taosProcExist(pid)) { if (pid > 0 && taosProcExist(pid)) {
dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pid); dInfo("node:%s, send kill signal to the child pid:%d", pWrapper->name, pid);
taosKillProc(pid); taosKillProc(pid);
dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pid); dInfo("node:%s, wait for child pid:%d to stop", pWrapper->name, pid);
taosWaitProc(pid); taosWaitProc(pid);
dInfo("node:%s, child process:%d is stopped", pWrapper->name, pid); dInfo("node:%s, child pid:%d is stopped", pWrapper->name, pid);
} }
} }
@ -255,7 +256,7 @@ static void dmWatchNodes(SDnode *pDnode) {
if (!OnlyInParentProc(pWrapper)) continue; if (!OnlyInParentProc(pWrapper)) continue;
if (proc->pid <= 0 || !taosProcExist(proc->pid)) { if (proc->pid <= 0 || !taosProcExist(proc->pid)) {
dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); dError("node:%s, pid:%d is killed and needs to restart", pWrapper->name, proc->pid);
dmCloseProcRpcHandles(&pWrapper->proc); dmCloseProcRpcHandles(&pWrapper->proc);
dmNewProc(pWrapper, ntype); dmNewProc(pWrapper, ntype);
} }

4
source/dnode/mgmt/node_mgmt/src/dmProc.c
View File

@ -162,7 +162,7 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg
return 0; return 0;
} }
static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { static inline int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) {
tsem_wait(&queue->sem); tsem_wait(&queue->sem);
taosThreadMutexLock(&queue->mutex); taosThreadMutexLock(&queue->mutex);
@ -412,7 +412,7 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) {
SProc *proc = &pWrapper->proc; SProc *proc = &pWrapper->proc;
if (proc->name == NULL) return; if (proc->name == NULL) return;
dDebug("node:%s, start to clean up proc", pWrapper->name); dDebug("node:%s, start to cleanup proc", pWrapper->name);
dmStopProc(proc); dmStopProc(proc);
dmCleanupProcQueue(proc->cqueue); dmCleanupProcQueue(proc->cqueue);
dmCleanupProcQueue(proc->pqueue); dmCleanupProcQueue(proc->pqueue);

67
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -43,8 +43,8 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
pMsg->info.wrapper = pWrapper; pMsg->info.wrapper = pWrapper;
dTrace("msg:%p, will be processed by %s, handle:%p", pMsg, pWrapper->name, pMsg->info.handle);
return (*msgFp)(pWrapper->pMgmt, pMsg); return (*msgFp)(pWrapper->pMgmt, pMsg);
} }
@ -56,8 +56,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
SMgmtWrapper *pWrapper = NULL; SMgmtWrapper *pWrapper = NULL;
dTrace("msg:%s is received, handle:%p cont:%p len:%d code:0x%04x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), dTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType),
pRpc->info.handle, pRpc->pCont, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
pRpc->info.noResp = 0; pRpc->info.noResp = 0;
pRpc->info.persistHandle = 0; pRpc->info.persistHandle = 0;
pRpc->info.wrapper = NULL; pRpc->info.wrapper = NULL;
@ -344,66 +344,6 @@ void dmCleanupClient(SDnode *pDnode) {
} }
} }
static inline int32_t dmGetHideUserAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
int32_t code = 0;
char pass[TSDB_PASSWORD_LEN + 1] = {0};
if (strcmp(user, INTERNAL_USER) == 0) {
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
} else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
} else {
code = -1;
}
if (code == 0) {
memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 1;
*encrypt = 0;
*ckey = 0;
}
return code;
}
static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret,
char *ckey) {
if (dmGetHideUserAuth(user, spi, encrypt, secret, ckey) == 0) {
dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
return 0;
}
SAuthReq authReq = {0};
tstrncpy(authReq.user, user, TSDB_USER_LEN);
int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
void *pReq = rpcMallocCont(contLen);
tSerializeSAuthReq(pReq, contLen, &authReq);
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .info.ahandle = (void *)9528};
SRpcMsg rpcRsp = {0};
SEpSet epSet = {0};
dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt);
dmGetMnodeEpSet(&pDnode->data, &epSet);
dmSendRecv(&epSet, &rpcMsg, &rpcRsp);
if (rpcRsp.code != 0) {
terrno = rpcRsp.code;
dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr());
} else {
SAuthRsp authRsp = {0};
tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp);
memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN);
memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN);
*spi = authRsp.spi;
*encrypt = authRsp.encrypt;
dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi,
authRsp.encrypt);
}
rpcFreeCont(rpcRsp.pCont);
return rpcRsp.code;
}
int32_t dmInitServer(SDnode *pDnode) { int32_t dmInitServer(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
@ -416,7 +356,6 @@ int32_t dmInitServer(SDnode *pDnode) {
rpcInit.sessions = tsMaxShellConns; rpcInit.sessions = tsMaxShellConns;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = (RpcAfp)dmRetrieveUserAuthInfo;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);

5
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -627,6 +627,10 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) {
pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
} else {
SArray *tmp = pOldConsumer->rebNewTopics; SArray *tmp = pOldConsumer->rebNewTopics;
pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics;
pNewConsumer->rebNewTopics = tmp; pNewConsumer->rebNewTopics = tmp;
@ -642,6 +646,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->subscribeTime = pNewConsumer->upTime; pOldConsumer->subscribeTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
}
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);

3
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -197,8 +197,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
goto CONN_OVER; goto CONN_OVER;
} }
if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->conn.user, connReq.passwd, mError("user:%s, failed to auth while acquire user, input:%s", pReq->conn.user, connReq.passwd);
pUser->pass);
code = TSDB_CODE_RPC_AUTH_FAILURE; code = TSDB_CODE_RPC_AUTH_FAILURE;
goto CONN_OVER; goto CONN_OVER;
} }

8
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -842,7 +842,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
} }
taosMemoryFree(pTrans->rpcRsp); taosMemoryFree(pTrans->rpcRsp);
mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage,
pTrans->rpcInfo.ahandle); pTrans->rpcInfo.ahandle);
SRpcMsg rspMsg = { SRpcMsg rspMsg = {
.info = pTrans->rpcInfo, .info = pTrans->rpcInfo,
@ -899,7 +899,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) {
} }
} }
mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->code, mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%04x", transId, action, pRsp->code,
pAction->acceptableCode); pAction->acceptableCode);
mndTransExecute(pMnode, pTrans); mndTransExecute(pMnode, pTrans);
@ -1031,7 +1031,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions);
return 0; return 0;
} else { } else {
mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF);
mndTransResetActions(pMnode, pTrans, pArray); mndTransResetActions(pMnode, pTrans, pArray);
terrno = errCode; terrno = errCode;
return errCode; return errCode;
@ -1222,7 +1222,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr());
} }
mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
return continueExec; return continueExec;
} }

7
source/dnode/mnode/impl/src/mnode.c
View File

@ -343,20 +343,21 @@ void mndStop(SMnode *pMnode) { return mndCleanupTimer(pMnode); }
int32_t mndProcessMsg(SRpcMsg *pMsg) { int32_t mndProcessMsg(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
void *ahandle = pMsg->info.ahandle; void *ahandle = pMsg->info.ahandle;
mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle); mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle);
if (IsReq(pMsg) && !mndIsMaster(pMnode)) { if (IsReq(pMsg)) {
if (!mndIsMaster(pMnode)) {
terrno = TSDB_CODE_APP_NOT_READY; terrno = TSDB_CODE_APP_NOT_READY;
mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
return -1; return -1;
} }
if (IsReq(pMsg) && (pMsg->contLen == 0 || pMsg->pCont == NULL)) { if (pMsg->contLen == 0 || pMsg->pCont == NULL) {
terrno = TSDB_CODE_INVALID_MSG_LEN; terrno = TSDB_CODE_INVALID_MSG_LEN;
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
return -1; return -1;
} }
}
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)];
if (fp == NULL) { if (fp == NULL) {

2
source/dnode/mnode/impl/test/trans/trans2.cpp
View File

@ -56,7 +56,7 @@ class MndTestTrans2 : public ::testing::Test {
msgCb.sendReqFp = sendReq; msgCb.sendReqFp = sendReq;
msgCb.sendRspFp = sendRsp; msgCb.sendRspFp = sendRsp;
msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack
tmsgSetDefaultMsgCb(&msgCb); tmsgSetDefault(&msgCb);
SMnodeOpt opt = {0}; SMnodeOpt opt = {0};
opt.deploy = 1; opt.deploy = 1;

2
source/dnode/vnode/inc/vnode.h
View File

@ -210,7 +210,7 @@ struct SMetaReader {
}; };
struct SMTbCursor { struct SMTbCursor {
TDBC *pDbc; TBC *pDbc;
void *pKey; void *pKey;
void *pVal; void *pVal;
int kLen; int kLen;

18
source/dnode/vnode/src/inc/meta.h
View File

@ -63,16 +63,16 @@ struct SMeta {
char* path; char* path;
SVnode* pVnode; SVnode* pVnode;
TENV* pEnv; TDB* pEnv;
TXN txn; TXN txn;
TDB* pTbDb; TTB* pTbDb;
TDB* pSkmDb; TTB* pSkmDb;
TDB* pUidIdx; TTB* pUidIdx;
TDB* pNameIdx; TTB* pNameIdx;
TDB* pCtbIdx; TTB* pCtbIdx;
TDB* pTagIdx; TTB* pTagIdx;
TDB* pTtlIdx; TTB* pTtlIdx;
TDB* pSmaIdx; TTB* pSmaIdx;
SMetaIdx* pIdx; SMetaIdx* pIdx;
}; };

10
source/dnode/vnode/src/inc/sma.h
View File

@ -46,7 +46,7 @@ struct SSmaEnv {
TXN txn; TXN txn;
void *pPool; // SPoolMem void *pPool; // SPoolMem
SDiskID did; SDiskID did;
TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? TDB *dbEnv; // TODO: If it's better to put it in smaIndex level?
char *path; // relative path char *path; // relative path
SSmaStat *pStat; SSmaStat *pStat;
}; };
@ -93,16 +93,16 @@ typedef struct SDBFile SDBFile;
struct SDBFile { struct SDBFile {
int32_t fid; int32_t fid;
TDB *pDB; TTB *pDB;
char *path; char *path;
}; };
int32_t tdSmaBeginCommit(SSmaEnv *pEnv); int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
int32_t tdSmaEndCommit(SSmaEnv *pEnv); int32_t tdSmaEndCommit(SSmaEnv *pEnv);
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); int32_t smaOpenDBEnv(TDB **ppEnv, const char *path);
int32_t smaCloseDBEnv(TENV *pEnv); int32_t smaCloseDBEnv(TDB *pEnv);
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF);
int32_t smaCloseDBF(SDBFile *pDBF); int32_t smaCloseDBF(SDBFile *pDBF);
int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -117,6 +117,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*); void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);

54
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -50,63 +50,63 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
taosMkDir(pMeta->path); taosMkDir(pMeta->path);
// open env // open env
ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTbDb // open pTbDb
ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pSkmDb // open pSkmDb
ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pUidIdx // open pUidIdx
ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pNameIdx // open pNameIdx
ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pCtbIdx // open pCtbIdx
ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTagIdx // open pTagIdx
ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pTtlIdx // open pTtlIdx
ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
} }
// open pSmaIdx // open pSmaIdx
ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx);
if (ret < 0) { if (ret < 0) {
metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err; goto _err;
@ -125,15 +125,15 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
_err: _err:
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbClose(pMeta->pEnv);
metaDestroyLock(pMeta); metaDestroyLock(pMeta);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
return -1; return -1;
@ -142,15 +142,15 @@ _err:
int metaClose(SMeta *pMeta) { int metaClose(SMeta *pMeta) {
if (pMeta) { if (pMeta) {
if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pIdx) metaCloseIdx(pMeta);
if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx);
if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx);
if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb);
if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb);
if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); if (pMeta->pEnv) tdbClose(pMeta->pEnv);
metaDestroyLock(pMeta); metaDestroyLock(pMeta);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
} }

40
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u
STbDbKey tbDbKey = {.version = version, .uid = uid}; STbDbKey tbDbKey = {.version = version, .uid = uid};
// query table.db // query table.db
if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
goto _err; goto _err;
} }
@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
int64_t version; int64_t version;
// query uid.idx // query uid.idx
if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
tb_uid_t uid; tb_uid_t uid;
// query name.idx // query name.idx
if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -100,9 +100,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
metaReaderInit(&pTbCur->mr, pMeta, 0); metaReaderInit(&pTbCur->mr, pMeta, 0);
tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL);
tdbDbcMoveToFirst(pTbCur->pDbc); tdbTbcMoveToFirst(pTbCur->pDbc);
return pTbCur; return pTbCur;
} }
@ -113,7 +113,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
tdbFree(pTbCur->pVal); tdbFree(pTbCur->pVal);
metaReaderClear(&pTbCur->mr); metaReaderClear(&pTbCur->mr);
if (pTbCur->pDbc) { if (pTbCur->pDbc) {
tdbDbcClose(pTbCur->pDbc); tdbTbcClose(pTbCur->pDbc);
} }
taosMemoryFree(pTbCur); taosMemoryFree(pTbCur);
} }
@ -125,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
STbCfg tbCfg; STbCfg tbCfg;
for (;;) { for (;;) {
ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
pKey = &skmDbKey; pKey = &skmDbKey;
kLen = sizeof(skmDbKey); kLen = sizeof(skmDbKey);
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); ret = tdbTbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen);
metaULock(pMeta); metaULock(pMeta);
if (ret < 0) { if (ret < 0) {
return NULL; return NULL;
@ -184,7 +184,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
struct SMCtbCursor { struct SMCtbCursor {
SMeta *pMeta; SMeta *pMeta;
TDBC *pCur; TBC *pCur;
tb_uid_t suid; tb_uid_t suid;
void *pKey; void *pKey;
void *pVal; void *pVal;
@ -207,7 +207,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
pCtbCur->suid = uid; pCtbCur->suid = uid;
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaULock(pMeta); metaULock(pMeta);
taosMemoryFree(pCtbCur); taosMemoryFree(pCtbCur);
@ -217,9 +217,9 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
// move to the suid // move to the suid
ctbIdxKey.suid = uid; ctbIdxKey.suid = uid;
ctbIdxKey.uid = INT64_MIN; ctbIdxKey.uid = INT64_MIN;
tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); tdbTbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c);
if (c > 0) { if (c > 0) {
tdbDbcMoveToNext(pCtbCur->pCur); tdbTbcMoveToNext(pCtbCur->pCur);
} }
return pCtbCur; return pCtbCur;
@ -229,7 +229,7 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) {
if (pCtbCur) { if (pCtbCur) {
if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta);
if (pCtbCur->pCur) { if (pCtbCur->pCur) {
tdbDbcClose(pCtbCur->pCur); tdbTbcClose(pCtbCur->pCur);
tdbFree(pCtbCur->pKey); tdbFree(pCtbCur->pKey);
tdbFree(pCtbCur->pVal); tdbFree(pCtbCur->pVal);
@ -243,7 +243,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
int ret; int ret;
SCtbIdxKey *pCtbIdxKey; SCtbIdxKey *pCtbIdxKey;
ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); ret = tdbTbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen);
if (ret < 0) { if (ret < 0) {
return 0; return 0;
} }
@ -299,7 +299,7 @@ int metaGetTbNum(SMeta *pMeta) {
typedef struct { typedef struct {
SMeta *pMeta; SMeta *pMeta;
TDBC *pCur; TBC *pCur;
tb_uid_t uid; tb_uid_t uid;
void *pKey; void *pKey;
void *pVal; void *pVal;
@ -323,7 +323,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
pSmaCur->uid = uid; pSmaCur->uid = uid;
metaRLock(pMeta); metaRLock(pMeta);
ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); ret = tdbTbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaULock(pMeta); metaULock(pMeta);
taosMemoryFree(pSmaCur); taosMemoryFree(pSmaCur);
@ -333,9 +333,9 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) {
// move to the suid // move to the suid
smaIdxKey.uid = uid; smaIdxKey.uid = uid;
smaIdxKey.smaUid = INT64_MIN; smaIdxKey.smaUid = INT64_MIN;
tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); tdbTbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c);
if (c > 0) { if (c > 0) {
tdbDbcMoveToNext(pSmaCur->pCur); tdbTbcMoveToNext(pSmaCur->pCur);
} }
return pSmaCur; return pSmaCur;
@ -345,7 +345,7 @@ void metaCloseSmaCursor(SMSmaCursor *pSmaCur) {
if (pSmaCur) { if (pSmaCur) {
if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta);
if (pSmaCur->pCur) { if (pSmaCur->pCur) {
tdbDbcClose(pSmaCur->pCur); tdbTbcClose(pSmaCur->pCur);
tdbFree(pSmaCur->pKey); tdbFree(pSmaCur->pKey);
tdbFree(pSmaCur->pVal); tdbFree(pSmaCur->pVal);
@ -359,7 +359,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) {
int ret; int ret;
SSmaIdxKey *pSmaIdxKey; SSmaIdxKey *pSmaIdxKey;
ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); ret = tdbTbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen);
if (ret < 0) { if (ret < 0) {
return 0; return 0;
} }

8
source/dnode/vnode/src/meta/metaSma.c
View File

@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderClear(&coder); tEncoderClear(&coder);
// write to table.db // write to table.db
if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err; goto _err;
} }
@ -130,17 +130,17 @@ _err:
} }
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
} }
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
} }
static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
} }
static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {

191
source/dnode/vnode/src/meta/metaTable.c
View File

@ -71,9 +71,9 @@ _err:
} }
int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
TDBC *pNameIdxc = NULL; TBC *pNameIdxc = NULL;
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
SCtbIdxKey *pCtbIdxKey; SCtbIdxKey *pCtbIdxKey;
const void *pKey = NULL; const void *pKey = NULL;
int nKey; int nKey;
@ -82,43 +82,43 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
int c, ret; int c, ret;
// prepare uid idx cursor // prepare uid idx cursor
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
terrno = TSDB_CODE_VND_TB_NOT_EXIST; terrno = TSDB_CODE_VND_TB_NOT_EXIST;
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
goto _err; goto _err;
} }
// prepare name idx cursor // prepare name idx cursor
tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
} }
tdbDbcDelete(pUidIdxc); tdbTbcDelete(pUidIdxc);
tdbDbcDelete(pNameIdxc); tdbTbcDelete(pNameIdxc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
// loop to drop each child table // loop to drop each child table
tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { if (ret < 0 || (c < 0 && tdbTbcMoveToNext(pCtbIdxc) < 0)) {
tdbDbcClose(pCtbIdxc); tdbTbcClose(pCtbIdxc);
goto _exit; goto _exit;
} }
for (;;) { for (;;) {
tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); tdbTbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL);
pCtbIdxKey = (SCtbIdxKey *)pKey; pCtbIdxKey = (SCtbIdxKey *)pKey;
if (pCtbIdxKey->suid > pReq->suid) break; if (pCtbIdxKey->suid > pReq->suid) break;
// drop the child table (TODO) // drop the child table (TODO)
if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; if (tdbTbcMoveToNext(pCtbIdxc) < 0) break;
} }
_exit: _exit:
@ -134,8 +134,8 @@ _err:
int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry oStbEntry = {0}; SMetaEntry oStbEntry = {0};
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
const void *pData; const void *pData;
int nData; int nData;
int64_t oversion; int64_t oversion;
@ -143,14 +143,14 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int32_t ret; int32_t ret;
int32_t c; int32_t c;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
if (ret < 0 || c) { if (ret < 0 || c) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -158,11 +158,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(ret == 0 && c == 0); ASSERT(ret == 0 && c == 0);
ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
ASSERT(ret == 0); ASSERT(ret == 0);
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
@ -191,12 +191,12 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
metaSaveToTbDb(pMeta, &nStbEntry); metaSaveToTbDb(pMeta, &nStbEntry);
// update uid index // update uid index
tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
metaULock(pMeta); metaULock(pMeta);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
} }
@ -256,9 +256,9 @@ _err:
} }
int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
TDBC *pNameIdxc = NULL; TBC *pNameIdxc = NULL;
const void *pData; const void *pData;
int nData; int nData;
tb_uid_t uid; tb_uid_t uid;
@ -271,15 +271,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
int c = 0, ret; int c = 0, ret;
// search & delete the name idx // search & delete the name idx
tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { if (ret < 0 || !tdbTbcIsValid(pNameIdxc) || c) {
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
} }
ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pNameIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -287,36 +287,36 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
uid = *(tb_uid_t *)pData; uid = *(tb_uid_t *)pData;
tdbDbcDelete(pNameIdxc); tdbTbcDelete(pNameIdxc);
tdbDbcClose(pNameIdxc); tdbTbcClose(pNameIdxc);
// search & delete uid idx // search & delete uid idx
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
tver = *(int64_t *)pData; tver = *(int64_t *)pData;
tdbDbcDelete(pUidIdxc); tdbTbcDelete(pUidIdxc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
// search and get meta entry // search and get meta entry
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); ret = tdbTbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
if (ret < 0) { if (ret < 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -345,21 +345,21 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
taosMemoryFree(pDataCopy); taosMemoryFree(pDataCopy);
tDecoderClear(&coder); tDecoderClear(&coder);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
if (type == TSDB_CHILD_TABLE) { if (type == TSDB_CHILD_TABLE) {
// remove the pCtbIdx // remove the pCtbIdx
TDBC *pCtbIdxc = NULL; TBC *pCtbIdxc = NULL;
tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c);
if (ret < 0 || c != 0) { if (ret < 0 || c != 0) {
ASSERT(0); ASSERT(0);
return -1; return -1;
} }
tdbDbcDelete(pCtbIdxc); tdbTbcDelete(pCtbIdxc);
tdbDbcClose(pCtbIdxc); tdbTbcClose(pCtbIdxc);
// remove tags from pTagIdx (todo) // remove tags from pTagIdx (todo)
} else if (type == TSDB_NORMAL_TABLE) { } else if (type == TSDB_NORMAL_TABLE) {
@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
int c; int c;
// search name index // search name index
ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) { if (ret < 0) {
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
@ -400,22 +400,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
pVal = NULL; pVal = NULL;
// search uid index // search uid index
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
// search table.db // search table.db
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
// get table entry // get table entry
SDecoder dc = {0}; SDecoder dc = {0};
@ -443,6 +443,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
entry.version = version; entry.version = version;
int tlen; int tlen;
SSchema *pNewSchema = NULL;
switch (pAlterTbReq->action) { switch (pAlterTbReq->action) {
case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_ADD_COLUMN:
if (pColumn) { if (pColumn) {
@ -451,8 +452,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
} }
pSchema->sver++; pSchema->sver++;
pSchema->nCols++; pSchema->nCols++;
pSchema->pSchema = pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols);
taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols); memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1));
pSchema->pSchema = pNewSchema;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type;
pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags;
@ -505,21 +507,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
// save to table db // save to table db
metaSaveToTbDb(pMeta, &entry); metaSaveToTbDb(pMeta, &entry);
tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0);
metaSaveToSkmDb(pMeta, &entry); metaSaveToSkmDb(pMeta, &entry);
metaULock(pMeta); metaULock(pMeta);
if (pNewSchema) taosMemoryFree(pNewSchema);
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
_err: _err:
tDecoderClear(&dc); tDecoderClear(&dc);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return -1; return -1;
} }
@ -536,7 +539,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
int nData = 0; int nData = 0;
// search name index // search name index
ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) { if (ret < 0) {
terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
return -1; return -1;
@ -547,24 +550,24 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
pVal = NULL; pVal = NULL;
// search uid index // search uid index
TDBC *pUidIdxc = NULL; TBC *pUidIdxc = NULL;
tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
oversion = *(int64_t *)pData; oversion = *(int64_t *)pData;
// search table.db // search table.db
TDBC *pTbDbc = NULL; TBC *pTbDbc = NULL;
SDecoder dc = {0}; SDecoder dc = {0};
/* get ctbEntry */ /* get ctbEntry */
tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
ASSERT(c == 0); ASSERT(c == 0);
tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
ctbEntry.pBuf = taosMemoryMalloc(nData); ctbEntry.pBuf = taosMemoryMalloc(nData);
memcpy(ctbEntry.pBuf, pData, nData); memcpy(ctbEntry.pBuf, pData, nData);
@ -573,8 +576,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
tDecoderClear(&dc); tDecoderClear(&dc);
/* get stbEntry*/ /* get stbEntry*/
tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal);
tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey),
(void **)&stbEntry.pBuf, &nVal); (void **)&stbEntry.pBuf, &nVal);
tdbFree(pVal); tdbFree(pVal);
tDecoderInit(&dc, stbEntry.pBuf, nVal); tDecoderInit(&dc, stbEntry.pBuf, nVal);
@ -632,19 +635,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaSaveToTbDb(pMeta, &ctbEntry); metaSaveToTbDb(pMeta, &ctbEntry);
// save to uid.idx // save to uid.idx
tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn);
if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return 0; return 0;
_err: _err:
if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf);
if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf);
tdbDbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbDbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
return -1; return -1;
} }
@ -708,7 +711,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderClear(&coder); tEncoderClear(&coder);
// write to table.db // write to table.db
if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
goto _err; goto _err;
} }
@ -721,11 +724,11 @@ _err:
} }
static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
} }
static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
} }
static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
@ -748,12 +751,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
ttlKey.uid = pME->uid; ttlKey.uid = pME->uid;
return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
} }
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
} }
static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid,
@ -801,10 +804,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
SDecoder dc = {0}; SDecoder dc = {0};
// get super table // get super table
tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData);
tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.uid = pCtbEntry->ctbEntry.suid;
tbDbKey.version = *(int64_t *)pData; tbDbKey.version = *(int64_t *)pData;
tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData);
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
metaDecodeEntry(&dc, &stbEntry); metaDecodeEntry(&dc, &stbEntry);
@ -817,7 +820,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
&pTagIdxKey, &nTagIdxKey) < 0) { &pTagIdxKey, &nTagIdxKey) < 0) {
return -1; return -1;
} }
tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
metaDestroyTagIdxKey(pTagIdxKey); metaDestroyTagIdxKey(pTagIdxKey);
tDecoderClear(&dc); tDecoderClear(&dc);
@ -859,7 +862,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
tEncoderInit(&coder, pVal, vLen); tEncoderInit(&coder, pVal, vLen);
tEncodeSSchemaWrapper(&coder, pSW); tEncodeSSchemaWrapper(&coder, pSW);
if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
rcode = -1; rcode = -1;
goto _exit; goto _exit;
} }

18
source/dnode/vnode/src/sma/smaTDBImpl.c
View File

@ -17,12 +17,12 @@
#include "sma.h" #include "sma.h"
int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) {
int ret = 0; int ret = 0;
if (path == NULL) return -1; if (path == NULL) return -1;
ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param ret = tdbOpen(path, 4096, 256, ppEnv); // use as param
if (ret != 0) { if (ret != 0) {
smaError("failed to create tsdb db env, ret = %d", ret); smaError("failed to create tsdb db env, ret = %d", ret);
@ -32,7 +32,7 @@ int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) {
return 0; return 0;
} }
int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
const SSmaKey *pKey1 = (const SSmaKey *)arg1; const SSmaKey *pKey1 = (const SSmaKey *)arg1;
@ -54,21 +54,21 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int
return 0; return 0;
} }
static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
// Create a database // Create a database
compFunc = tdSmaKeyCmpr; compFunc = tdSmaKeyCmpr;
if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) {
return -1; return -1;
} }
return 0; return 0;
} }
static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) {
// TEnv is shared by a group of SDBFile // TEnv is shared by a group of SDBFile
if (!pEnv || !pDBF) { if (!pEnv || !pDBF) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in
int32_t ret; int32_t ret;
printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn);
ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
if (ret < 0) { if (ret < 0) {
smaError("failed to upsert tsma data into db, ret = %d", ret); smaError("failed to upsert tsma data into db, ret = %d", ret);
return -1; return -1;
@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_
void *pVal = NULL; void *pVal = NULL;
int ret; int ret;
ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
if (ret < 0) { if (ret < 0) {
smaError("failed to get tsma data from db, ret = %d", ret); smaError("failed to get tsma data from db, ret = %d", ret);

34
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -55,7 +55,6 @@ typedef enum {
SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
} ESmaStorageLevel; } ESmaStorageLevel;
// static func // static func
static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
@ -72,15 +71,12 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
TXN *txn); TXN *txn);
// expired window // expired window
static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version);
static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
int64_t version);
static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
// read data // read data
// implementation // implementation
/** /**
@ -157,7 +153,6 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
return false; return false;
} }
/** /**
* @brief Approximate value for week/month/year. * @brief Approximate value for week/month/year.
* *
@ -239,7 +234,6 @@ static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit,
return interval; return interval;
} }
static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
int8_t intervalUnit) { int8_t intervalUnit) {
pSmaH->pSma = pSma; pSmaH->pSma = pSma;
@ -493,7 +487,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
smaCloseDBF(&tSmaH.dFile); smaCloseDBF(&tSmaH.dFile);
} }
tdSetTSmaDataFile(&tSmaH, indexUid, fid); tdSetTSmaDataFile(&tSmaH, indexUid, fid);
smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32
" queryKey:%" PRIi64,
SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
@ -525,7 +520,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
SMA_VID(pSma), skey, tlen, indexUid); SMA_VID(pSma), skey, tlen, indexUid);
} }
} }
} }
tdSmaEndCommit(pEnv); // TODO: not commit for every insert tdSmaEndCommit(pEnv); // TODO: not commit for every insert
@ -557,7 +551,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL
TXN *txn) { TXN *txn) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert tsma data blocks into B+Tree(TDB) // TODO: insert tsma data blocks into B+Tree(TTB)
if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
@ -600,8 +594,8 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi
if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
// error handling // error handling
tdUnRefSmaStat(pSma, pStat); tdUnRefSmaStat(pSma, pStat);
smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey,
skey, indexUid); indexUid);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma),
@ -666,8 +660,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
taosSsleep(1); taosSsleep(1);
if (++nSleep > SMA_DROP_EXPIRED_TIME) { if (++nSleep > SMA_DROP_EXPIRED_TIME) {
smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal);
refVal);
break; break;
}; };
} }
@ -736,8 +729,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) {
// TODO: mark this window as expired. // TODO: mark this window as expired.
smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
querySKey, indexUid); indexUid);
} else { } else {
smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
indexUid); indexUid);
@ -766,8 +759,8 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
int64_t queryGroupId = 0; int64_t queryGroupId = 0;
tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path,
tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
void *result = NULL; void *result = NULL;
int32_t valueSize = 0; int32_t valueSize = 0;
@ -777,7 +770,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query
smaCloseDBF(&tReadH.dFile); smaCloseDBF(&tReadH.dFile);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
#endif #endif
#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ #ifdef _TEST_SMA_PRINT_DEBUG_LOG_
for (uint32_t v = 0; v < valueSize; v += 8) { for (uint32_t v = 0; v < valueSize; v += 8) {
@ -932,8 +925,6 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/** /**
* @brief Update expired window according to msg from stream computing module. * @brief Update expired window according to msg from stream computing module.
* *
@ -1044,4 +1035,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

15
source/dnode/vnode/src/tq/tq.c
View File

@ -101,6 +101,21 @@ static void tdSRowDemo() {
taosMemoryFree(pTSChema); taosMemoryFree(pTSChema);
} }
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) {
void* pIter = NULL;
STqExec* pExec = NULL;
while (1) {
pIter = taosHashIterate(pTq->execs, pIter);
if (pIter == NULL) break;
pExec = (STqExec*)pIter;
for (int32_t i = 0; i < 5; i++) {
int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true);
ASSERT(code == 0);
}
}
return 0;
}
int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
if (msgType != TDMT_VND_SUBMIT) return 0; if (msgType != TDMT_VND_SUBMIT) return 0;
void* pIter = NULL; void* pIter = NULL;

4
source/dnode/vnode/src/tsdb/tsdbSma.c
View File

@ -53,7 +53,7 @@ struct SSmaEnv {
TXN txn; TXN txn;
SPoolMem *pPool; SPoolMem *pPool;
SDiskID did; SDiskID did;
TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? TDB *dbEnv; // TODO: If it's better to put it in smaIndex level?
char *path; // relative path char *path; // relative path
SSmaStat *pStat; SSmaStat *pStat;
}; };
@ -876,7 +876,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke
TXN *txn) { TXN *txn) {
SDBFile *pDBFile = &pSmaH->dFile; SDBFile *pDBFile = &pSmaH->dFile;
// TODO: insert tsma data blocks into B+Tree(TDB) // TODO: insert tsma data blocks into B+Tree(TTB)
if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);

18
source/dnode/vnode/src/tsdb/tsdbTDBImpl.c
View File

@ -17,12 +17,12 @@
#include "tsdb.h" #include "tsdb.h"
int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) {
int ret = 0; int ret = 0;
if (path == NULL) return -1; if (path == NULL) return -1;
ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param ret = tdbOpen(path, 4096, 256, ppEnv); // use as param
if (ret != 0) { if (ret != 0) {
tsdbError("Failed to create tsdb db env, ret = %d", ret); tsdbError("Failed to create tsdb db env, ret = %d", ret);
@ -32,7 +32,7 @@ int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) {
return 0; return 0;
} }
int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
const SSmaKey *pKey1 = (const SSmaKey *)arg1; const SSmaKey *pKey1 = (const SSmaKey *)arg1;
@ -54,20 +54,20 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i
return 0; return 0;
} }
static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
int ret; int ret;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
// Create a database // Create a database
compFunc = tsdbSmaKeyCmpr; compFunc = tsdbSmaKeyCmpr;
ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB);
return 0; return 0;
} }
static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) {
// TEnv is shared by a group of SDBFile // TEnv is shared by a group of SDBFile
if (!pEnv || !pDBF) { if (!pEnv || !pDBF) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) {
int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
int32_t ret; int32_t ret;
ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
if (ret < 0) { if (ret < 0) {
tsdbError("Failed to create insert sma data into db, ret = %d", ret); tsdbError("Failed to create insert sma data into db, ret = %d", ret);
return -1; return -1;
@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32
void *pVal = NULL; void *pVal = NULL;
int ret; int ret;
ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
if (ret < 0) { if (ret < 0) {
tsdbError("Failed to get sma data from db, ret = %d", ret); tsdbError("Failed to get sma data from db, ret = %d", ret);

7
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -333,6 +333,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
SVCreateTbRsp cRsp = {0}; SVCreateTbRsp cRsp = {0};
char tbName[TSDB_TABLE_FNAME_LEN]; char tbName[TSDB_TABLE_FNAME_LEN];
STbUidStore *pStore = NULL; STbUidStore *pStore = NULL;
SArray *tbUids = NULL;
pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP;
pRsp->code = TSDB_CODE_SUCCESS; pRsp->code = TSDB_CODE_SUCCESS;
@ -348,7 +349,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
} }
rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp));
if (rsp.pArray == NULL) { tbUids = taosArrayInit(req.nReqs, sizeof(int64_t));
if (rsp.pArray == NULL || tbUids == NULL) {
rcode = -1; rcode = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
@ -376,6 +378,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
} else { } else {
cRsp.code = TSDB_CODE_SUCCESS; cRsp.code = TSDB_CODE_SUCCESS;
tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
taosArrayPush(tbUids, &pCreateReq->uid);
} }
taosArrayPush(rsp.pArray, &cRsp); taosArrayPush(rsp.pArray, &cRsp);
@ -383,6 +386,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
tDecoderClear(&decoder); tDecoderClear(&decoder);
tqUpdateTbUidList(pVnode->pTq, tbUids);
tdUpdateTbUidList(pVnode->pSma, pStore); tdUpdateTbUidList(pVnode->pSma, pStore);
tdUidStoreFree(pStore); tdUidStoreFree(pStore);
@ -402,6 +406,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
_exit: _exit:
taosArrayDestroy(rsp.pArray); taosArrayDestroy(rsp.pArray);
taosArrayDestroy(tbUids);
tDecoderClear(&decoder); tDecoderClear(&decoder);
tEncoderClear(&encoder); tEncoderClear(&encoder);
return rcode; return rcode;

27
source/libs/executor/inc/executorimpl.h
View File

@ -378,6 +378,13 @@ typedef enum EStreamScanMode {
STREAM_SCAN_FROM_DATAREADER, STREAM_SCAN_FROM_DATAREADER,
} EStreamScanMode; } EStreamScanMode;
typedef struct SCatchSupporter {
SHashObj* pWindowHashTable; // quick locate the window object for each window
SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file
int32_t keySize;
int64_t* pKeyBuf;
} SCatchSupporter;
typedef struct SStreamBlockScanInfo { typedef struct SStreamBlockScanInfo {
SArray* pBlockLists; // multiple SSDatablock. SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock SSDataBlock* pRes; // result SSDataBlock
@ -400,6 +407,8 @@ typedef struct SStreamBlockScanInfo {
EStreamScanMode scanMode; EStreamScanMode scanMode;
SOperatorInfo* pOperatorDumy; SOperatorInfo* pOperatorDumy;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SCatchSupporter childAggSup;
SArray* childIds;
} SStreamBlockScanInfo; } SStreamBlockScanInfo;
typedef struct SSysTableScanInfo { typedef struct SSysTableScanInfo {
@ -460,6 +469,16 @@ typedef struct SIntervalAggOperatorInfo {
bool invertible; bool invertible;
} SIntervalAggOperatorInfo; } SIntervalAggOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo {
SOptrBasicInfo binfo; // basic info
SGroupResInfo groupResInfo; // multiple results build supporter
SInterval interval; // interval info
int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator.
SAggSupporter aggSup; // aggregate supporter
int32_t order; // current SSDataBlock scan order
STimeWindowAggSupp twAggSup;
} SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
SOptrBasicInfo binfo; SOptrBasicInfo binfo;
SAggSupporter aggSup; SAggSupporter aggSup;
@ -696,6 +715,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
@ -771,9 +793,8 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary
TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item,
int32_t order); int32_t order);
int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize,
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, const char* pKey, const char* pDir);
uint64_t groupId, int32_t numOfOutput);
#ifdef __cplusplus #ifdef __cplusplus
} }

8
source/libs/executor/src/executor.c
View File

@ -126,7 +126,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) {
return pTaskInfo; return pTaskInfo;
} }
int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
// traverse to the stream scanner node to add this table id // traverse to the stream scanner node to add this table id
@ -141,12 +141,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); metaReaderInit(&mr, pScanInfo->readHandle.meta, 0);
for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) {
int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); int64_t* id = (int64_t*)taosArrayGet(tableIdList, i);
int32_t code = metaGetTableEntryByUid(&mr, *id); int32_t code = metaGetTableEntryByUid(&mr, *id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
continue; continue;
} }
@ -160,7 +160,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
metaReaderClear(&mr); metaReaderClear(&mr);
qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;

1
source/libs/executor/src/executorMain.c
View File

@ -133,6 +133,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
publishQueryAbortEvent(pTaskInfo, ret); publishQueryAbortEvent(pTaskInfo, ret);
pTaskInfo->code = ret; pTaskInfo->code = ret;
cleanUpUdfs();
qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo),
tstrerror(pTaskInfo->code)); tstrerror(pTaskInfo->code));
return pTaskInfo->code; return pTaskInfo->code;

35
source/libs/executor/src/executorimpl.c
View File

@ -342,28 +342,6 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
return pResultRow; return pResultRow;
} }
void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes,
uint64_t groupId, int32_t numOfOutput) {
SAggSupporter* pSup = &pInfo->aggSup;
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
GET_RES_WINDOW_KEY_LEN(bytes));
SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1);
SqlFunctionCtx* pCtx = pInfo->binfo.pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset);
struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
continue;
}
pResInfo->initialized = false;
if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
}
}
}
/** /**
* the struct of key in hash table * the struct of key in hash table
* +----------+---------------+ * +----------+---------------+
@ -5321,3 +5299,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize,
const char* pKey, const char* pDir) {
pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize);
int32_t pageSize = rowSize * 32;
int32_t bufSize = pageSize * 4096;
createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);;
return TSDB_CODE_SUCCESS;
}

122
source/libs/executor/src/scanoperator.c
View File

@ -36,6 +36,11 @@
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
typedef struct SWindowPosition {
int32_t pageId;
int32_t rowId;
} SWindowPosition;
static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity);
static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size,
const char* dbName); const char* dbName);
@ -668,12 +673,103 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti
} }
pDataBlock->info.rows = size; pDataBlock->info.rows = size;
pDataBlock->info.type = STREAM_REPROCESS; pDataBlock->info.type = STREAM_REPROCESS;
blockDataUpdateTsWindow(pDataBlock);
taosArrayClear(pInfo->tsArray); taosArrayClear(pInfo->tsArray);
return pDataBlock; return pDataBlock;
} }
return NULL; return NULL;
} }
void static setSupKeyBuf(SCatchSupporter* pSup, int64_t groupId, int64_t childId, TSKEY ts) {
int64_t* pKey = (int64_t*)pSup->pKeyBuf;
pKey[0] = groupId;
pKey[1] = childId;
pKey[2] = ts;
}
static int32_t catchWidonwInfo(SSDataBlock* pDataBlock, SCatchSupporter* pSup,
int32_t pageId, int32_t tsIndex, int64_t childId) {
SColumnInfoData* pColDataInfo = taosArrayGet(pDataBlock->pDataBlock, tsIndex);
TSKEY* tsCols = (int64_t*)pColDataInfo->pData;
for (int32_t i = 0; i < pDataBlock->info.rows; i++) {
setSupKeyBuf(pSup, pDataBlock->info.groupId, childId, tsCols[i]);
SWindowPosition* p1 = (SWindowPosition*)taosHashGet(pSup->pWindowHashTable,
pSup->pKeyBuf, pSup->keySize);
if (p1 == NULL) {
SWindowPosition pos = {.pageId = pageId, .rowId = i};
int32_t code = taosHashPut(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize, &pos,
sizeof(SWindowPosition));
if (code != TSDB_CODE_SUCCESS ) {
return code;
}
} else {
p1->pageId = pageId;
p1->rowId = i;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t catchDatablock(SSDataBlock* pDataBlock, SCatchSupporter* pSup,
int32_t tsIndex, int64_t childId) {
int32_t start = 0;
int32_t stop = 0;
int32_t pageSize = getBufPageSize(pSup->pDataBuf);
while(start < pDataBlock->info.rows) {
blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pageSize);
SSDataBlock* pDB = blockDataExtractBlock(pDataBlock, start, stop - start + 1);
if (pDB == NULL) {
return terrno;
}
int32_t pageId = -1;
void* pPage = getNewBufPage(pSup->pDataBuf, pDataBlock->info.groupId, &pageId);
if (pPage == NULL) {
blockDataDestroy(pDB);
return terrno;
}
int32_t size = blockDataGetSize(pDB) + sizeof(int32_t) + pDB->info.numOfCols * sizeof(int32_t);
assert(size <= pageSize);
blockDataToBuf(pPage, pDB);
setBufPageDirty(pPage, true);
releaseBufPage(pSup->pDataBuf, pPage);
blockDataDestroy(pDB);
start = stop + 1;
int32_t code = catchWidonwInfo(pDB, pSup, pageId, tsIndex, childId);
if (code != TSDB_CODE_SUCCESS ) {
return code;
}
}
return TSDB_CODE_SUCCESS;
}
static SSDataBlock* getDataFromCatch(SStreamBlockScanInfo* pInfo) {
SSDataBlock* pBlock = pInfo->pUpdateRes;
if (pInfo->updateResIndex < pBlock->info.rows) {
blockDataCleanup(pInfo->pRes);
SCatchSupporter* pCSup = &pInfo->childAggSup;
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, 0);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
int32_t size = taosArrayGetSize(pInfo->childIds);
for (int32_t i = 0; i < size; i++) {
int64_t id = *(int64_t *)taosArrayGet(pInfo->childIds, i);
setSupKeyBuf(pCSup, pBlock->info.groupId, id,
tsCols[pInfo->updateResIndex]);
SWindowPosition* pos = (SWindowPosition*)taosHashGet(pCSup->pWindowHashTable,
pCSup->pKeyBuf, pCSup->keySize);
void* buf = getBufPage(pCSup->pDataBuf, pos->pageId);
SSDataBlock* pDB = createOneDataBlock(pInfo->pRes, false);
blockDataFromBuf(pDB, buf);
SSDataBlock* pSub = blockDataExtractBlock(pDB, pos->rowId, 1);
blockDataMerge(pInfo->pRes, pSub, NULL);
blockDataDestroy(pDB);
blockDataDestroy(pSub);
}
pInfo->updateResIndex++;
return pInfo->pRes;
}
return NULL;
}
static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
// NOTE: this operator does never check if current status is done or not // NOTE: this operator does never check if current status is done or not
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -687,6 +783,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
size_t total = taosArrayGetSize(pInfo->pBlockLists); size_t total = taosArrayGetSize(pInfo->pBlockLists);
if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
SSDataBlock* pDB = getDataFromCatch(pInfo);
if (pDB != NULL) {
return pDB;
} else {
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
}
}
if (pInfo->validBlockIndex >= total) { if (pInfo->validBlockIndex >= total) {
doClearBufferedBlocks(pInfo); doClearBufferedBlocks(pInfo);
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
@ -694,7 +799,17 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
int32_t current = pInfo->validBlockIndex++; int32_t current = pInfo->validBlockIndex++;
return taosArrayGetP(pInfo->pBlockLists, current); SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
if (pBlock->info.type == STREAM_REPROCESS) {
pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
} else {
int32_t code = catchDatablock(pBlock, &pInfo->childAggSup, pInfo->primaryTsIndex, 0);
if (code != TDB_CODE_SUCCESS) {
pTaskInfo->code = code;
longjmp(pTaskInfo->env, code);
}
}
return pBlock;
} else { } else {
if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
blockDataDestroy(pInfo->pUpdateRes); blockDataDestroy(pInfo->pUpdateRes);
@ -768,6 +883,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
rows = pBlockInfo->rows; rows = pBlockInfo->rows;
doFilter(pInfo->pCondition, pInfo->pRes, NULL); doFilter(pInfo->pCondition, pInfo->pRes, NULL);
blockDataUpdateTsWindow(pInfo->pRes);
break; break;
} }
@ -855,6 +971,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR
pInfo->pOperatorDumy = pOperatorDumy; pInfo->pOperatorDumy = pOperatorDumy;
pInfo->interval = pSTInfo->interval; pInfo->interval = pSTInfo->interval;
size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY);
initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize,
"StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan
pOperator->name = "StreamBlockScanOperator"; pOperator->name = "StreamBlockScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
pOperator->blocking = false; pOperator->blocking = false;

196
source/libs/executor/src/timewindowoperator.c
View File

@ -8,6 +8,8 @@ typedef enum SResultTsInterpType {
RESULT_ROW_END_INTERP = 2, RESULT_ROW_END_INTERP = 2,
} SResultTsInterpType; } SResultTsInterpType;
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator);
/* /*
* There are two cases to handle: * There are two cases to handle:
* *
@ -473,8 +475,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun
} }
static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { TSKEY* primaryKeys, int32_t prevPosition, int32_t order) {
int32_t order = pInfo->order;
bool ascQuery = (order == TSDB_ORDER_ASC); bool ascQuery = (order == TSDB_ORDER_ASC);
int32_t precision = pInterval->precision; int32_t precision = pInterval->precision;
@ -723,7 +724,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
STimeWindow nextWin = win; STimeWindow nextWin = win;
while (1) { while (1) {
int32_t prevEndPos = (forwardStep - 1) * step + startPos; int32_t prevEndPos = (forwardStep - 1) * step + startPos;
startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
if (startPos < 0) { if (startPos < 0) {
break; break;
} }
@ -1031,18 +1032,41 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type
} }
} }
} }
static void doClearWindows(SIntervalAggOperatorInfo* pInfo, int32_t numOfOutput, SSDataBlock* pBlock) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData,
int16_t bytes, uint64_t groupId, int32_t numOfOutput) {
SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
SResultRowPosition* p1 =
(SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf,
GET_RES_WINDOW_KEY_LEN(bytes));
SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1);
SqlFunctionCtx* pCtx = pBinfo->pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset);
struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo;
if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) {
continue;
}
pResInfo->initialized = false;
if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo);
}
}
}
static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo,
SInterval* pIntrerval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex);
TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; TSKEY *tsCols = (TSKEY*)pColDataInfo->pData;
int32_t step = 0; int32_t step = 0;
for (int32_t i = 0; i < pBlock->info.rows; i += step) { for (int32_t i = 0; i < pBlock->info.rows; i += step) {
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], &pInfo->interval, STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pIntrerval,
pInfo->interval.precision, NULL); pIntrerval->precision, NULL);
step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i,
win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
doClearWindow(pInfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput);
} }
} }
@ -1084,7 +1108,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type);
} }
if (pBlock->info.type == STREAM_REPROCESS) { if (pBlock->info.type == STREAM_REPROCESS) {
doClearWindows(pInfo, pOperator->numOfExprs, pBlock); doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval,
pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock);
continue; continue;
} }
pInfo->order = TSDB_ORDER_ASC; pInfo->order = TSDB_ORDER_ASC;
@ -1097,8 +1122,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
// TODO: remove for stream
/*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
@ -1116,6 +1139,12 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) {
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
} }
void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param;
doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
cleanupAggSup(&pInfo->aggSup);
}
bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
for (int32_t i = 0; i < numOfCols; i++) { for (int32_t i = 0; i < numOfCols; i++) {
if (!fmIsInvertible(pFCtx[i].functionId)) { if (!fmIsInvertible(pFCtx[i].functionId)) {
@ -1185,6 +1214,63 @@ _error:
return NULL; return NULL;
} }
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo,
SExecTaskInfo* pTaskInfo) {
SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->twAggSup = *pTwAggSupp;
pInfo->primaryTsIndex = primaryTsSlotId;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(pOperator, 4096);
int32_t code =
initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock,
keyBufSize, pTaskInfo->id.str);
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1);
pOperator->name = "StreamFinalIntervalOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
pOperator->blocking = true;
pOperator->status = OP_NOT_OPENED;
pOperator->pExpr = pExprInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->numOfExprs = numOfCols;
pOperator->info = pInfo;
pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL,
destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow,
NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
return pOperator;
_error:
destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols);
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
pTaskInfo->code = code;
return NULL;
}
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId,
STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo,
@ -1548,3 +1634,91 @@ _error:
pTaskInfo->code = code; pTaskInfo->code = code;
return NULL; return NULL;
} }
static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock,
int32_t tableGroupId) {
SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
int32_t numOfOutput = pOperatorInfo->numOfExprs;
SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
int32_t step = 1;
bool ascScan = true;
TSKEY* tsCols = NULL;
SResultRow* pResult = NULL;
int32_t forwardStep = 0;
if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
}
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan);
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts,
&pInfo->interval, pInfo->interval.precision, NULL);
while (1) {
int32_t code =
setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx,
numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*)pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
forwardStep =
getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
// window start(end) key interpolation
doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep,
pInfo->order, false);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
int32_t prevEndPos = (forwardStep - 1) * step + startPos;
startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order);
if (startPos < 0) {
break;
}
}
return pUpdated;
}
static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
SOperatorInfo* downstream = pOperator->pDownstream[0];
SArray* pUpdated = NULL;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
} else if (pOperator->status == OP_RES_TO_RETURN) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
}
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}
while (1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
}
setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
if (pBlock->info.type == STREAM_REPROCESS) {
doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval,
pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock);
continue;
}
pUpdated = doHashInterval(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
}
finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
pOperator->status = OP_RES_TO_RETURN;
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
}

1
source/libs/function/inc/builtinsimpl.h
View File

@ -74,6 +74,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx);
bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunction(SqlFunctionCtx *pCtx);
int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx);
int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
int32_t topFunction(SqlFunctionCtx *pCtx); int32_t topFunction(SqlFunctionCtx *pCtx);

13
source/libs/function/src/builtins.c
View File

@ -47,8 +47,10 @@ static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t le
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} else if (IS_NULL_TYPE(paraType)) {
paraType = TSDB_DATA_TYPE_BIGINT;
} }
pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType};
@ -62,7 +64,7 @@ static int32_t translateInNumOutDou(SFunctionNode* pFunc, char* pErrBuf, int32_t
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} }
@ -115,18 +117,19 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
} }
uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
if (!IS_NUMERIC_TYPE(paraType)) { if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} }
uint8_t resType = 0; uint8_t resType = 0;
if (IS_SIGNED_NUMERIC_TYPE(paraType) || paraType == TSDB_DATA_TYPE_BOOL) { if (IS_SIGNED_NUMERIC_TYPE(paraType) || TSDB_DATA_TYPE_BOOL == paraType || IS_NULL_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_BIGINT; resType = TSDB_DATA_TYPE_BIGINT;
} else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_UBIGINT; resType = TSDB_DATA_TYPE_UBIGINT;
} else if (IS_FLOAT_TYPE(paraType)) { } else if (IS_FLOAT_TYPE(paraType)) {
resType = TSDB_DATA_TYPE_DOUBLE; resType = TSDB_DATA_TYPE_DOUBLE;
} }
pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -872,7 +875,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.getEnvFunc = getFirstLastFuncEnv, .getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup, .initFunc = functionSetup,
.processFunc = lastFunction, .processFunc = lastFunction,
.finalizeFunc = functionFinalize .finalizeFunc = lastFinalize
}, },
{ {
.name = "diff", .name = "diff",

57
source/libs/function/src/builtinsimpl.c
View File

@ -255,7 +255,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
char* in = GET_ROWCELL_INTERBUF(pResInfo); char* in = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
@ -331,8 +331,18 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) {
int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t countFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElem = getNumofElem(pCtx); int32_t numOfElem = getNumofElem(pCtx);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
char* buf = GET_ROWCELL_INTERBUF(pResInfo); char* buf = GET_ROWCELL_INTERBUF(pResInfo);
if (IS_NULL_TYPE(type)) {
//select count(NULL) returns 0
numOfElem = 1;
*((int64_t*)buf) = 0;
} else {
*((int64_t*)buf) += numOfElem; *((int64_t*)buf) += numOfElem;
}
SET_VAL(pResInfo, numOfElem, 1); SET_VAL(pResInfo, numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -383,6 +393,12 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _sum_over;
}
if (pInput->colDataAggIsSet) { if (pInput->colDataAggIsSet) {
numOfElem = pInput->numOfRows - pAgg->numOfNull; numOfElem = pInput->numOfRows - pAgg->numOfNull;
ASSERT(numOfElem >= 0); ASSERT(numOfElem >= 0);
@ -427,6 +443,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
} }
} }
_sum_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -526,6 +543,12 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _avg_over;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData; int8_t* plist = (int8_t*)pCol->pData;
@ -617,6 +640,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
break; break;
} }
_avg_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -783,6 +807,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo); SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo);
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElems = 1;
goto _min_max_over;
}
// data in current data block are qualified to the query // data in current data block are qualified to the query
if (pInput->colDataAggIsSet) { if (pInput->colDataAggIsSet) {
numOfElems = pInput->numOfRows - pAgg->numOfNull; numOfElems = pInput->numOfRows - pAgg->numOfNull;
@ -1183,6 +1213,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
} }
} }
_min_max_over:
return numOfElems; return numOfElems;
} }
@ -1215,9 +1246,9 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = *(double*) &pRes->v; float v = *(double*) &pRes->v;
colDataAppend(pCol, currentRow, (const char*)&v, false); colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes);
} else { } else {
colDataAppend(pCol, currentRow, (const char*)&pRes->v, false); colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes);
} }
setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow);
@ -1287,6 +1318,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
if (IS_NULL_TYPE(type)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 1;
goto _stddev_over;
}
switch (type) { switch (type) {
case TSDB_DATA_TYPE_TINYINT: { case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData; int8_t* plist = (int8_t*)pCol->pData;
@ -1384,6 +1421,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
break; break;
} }
_stddev_over:
// data in the check operation are all null, not output // data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1943,6 +1981,19 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
char* in = GET_ROWCELL_INTERBUF(pResInfo);
colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes);
return pResInfo->numOfRes;
}
bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SDiffInfo); pEnv->calcMemSize = sizeof(SDiffInfo);
return true; return true;

60
source/libs/function/src/tudf.c
View File

@ -1364,9 +1364,12 @@ void releaseUdfFuncHandle(char* udfName) {
SUdfcFuncStub key = {0}; SUdfcFuncStub key = {0};
strcpy(key.udfName, udfName); strcpy(key.udfName, udfName);
SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ);
ASSERT(foundStub); if (!foundStub) {
return;
}
if (foundStub->refCount > 0) {
--foundStub->refCount; --foundStub->refCount;
ASSERT(foundStub->refCount>=0); }
uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); uv_mutex_unlock(&gUdfdProxy.udfStubsMutex);
} }
@ -1377,7 +1380,7 @@ int32_t cleanUpUdfs() {
while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) {
SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i);
if (stub->refCount == 0) { if (stub->refCount == 0) {
fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle); fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
doTeardownUdf(stub->handle); doTeardownUdf(stub->handle);
} else { } else {
fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p",
@ -1530,12 +1533,15 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
} }
SUdfcUvSession *session = handle; SUdfcUvSession *session = handle;
code = doCallUdfScalarFunc(handle, input, numOfCols, output); code = doCallUdfScalarFunc(handle, input, numOfCols, output);
if (session->outputType != output->columnData->info.type if (output->columnData == NULL) {
|| session->outputLen != output->columnData->info.bytes) { fnError("udfc scalar function calculate error. no column data");
fnError("udfc scalar function calculate error, session type: %d(%d), output type: %d(%d)",
session->outputType, session->outputLen,
output->columnData->info.type, output->columnData->info.bytes);
code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE;
} else {
if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) {
fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType,
session->outputLen, output->columnData->info.type, output->columnData->info.bytes);
code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE;
}
} }
releaseUdfFuncHandle(udfName); releaseUdfFuncHandle(udfName);
return code; return code;
@ -1565,7 +1571,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
taosMemoryFree(task->session); taosMemoryFree(session);
taosMemoryFree(task); taosMemoryFree(task);
return err; return err;
@ -1573,7 +1579,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| //memory layout |---SUdfAggRes----|-----final result-----|---inter result----|
typedef struct SUdfAggRes { typedef struct SUdfAggRes {
SUdfcUvSession *session;
int8_t finalResNum; int8_t finalResNum;
int8_t interResNum; int8_t interResNum;
char* finalResBuf; char* finalResBuf;
@ -1606,7 +1611,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
udfRes->session = (SUdfcUvSession *)handle;
SUdfInterBuf buf = {0}; SUdfInterBuf buf = {0};
if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
@ -1621,22 +1625,26 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult
releaseUdfFuncHandle(pCtx->udfName); releaseUdfFuncHandle(pCtx->udfName);
return false; return false;
} }
releaseUdfFuncHandle(pCtx->udfName);
freeUdfInterBuf(&buf); freeUdfInterBuf(&buf);
return true; return true;
} }
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
SInputColumnInfoData* pInput = &pCtx->input; int32_t udfCode = 0;
int32_t numOfCols = pInput->numOfInputCols; UdfcFuncHandle handle = 0;
if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode);
SUdfcUvSession *session = udfRes->session; return udfCode;
if (session == NULL) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE;
} }
SUdfcUvSession *session = handle;
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;
SInputColumnInfoData* pInput = &pCtx->input;
int32_t numOfCols = pInput->numOfInputCols;
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
@ -1664,7 +1672,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
.numOfResult = udfRes->interResNum}; .numOfResult = udfRes->interResNum};
SUdfInterBuf newState = {0}; SUdfInterBuf newState = {0};
int32_t udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState);
if (udfCode != 0) { if (udfCode != 0) {
fnError("udfAggProcess error. code: %d", udfCode); fnError("udfAggProcess error. code: %d", udfCode);
newState.numOfResult = 0; newState.numOfResult = 0;
@ -1684,19 +1692,21 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
blockDataDestroy(inputBlock); blockDataDestroy(inputBlock);
taosArrayDestroy(tempBlock.pDataBlock); taosArrayDestroy(tempBlock.pDataBlock);
if (udfCode != 0) {
releaseUdfFuncHandle(pCtx->udfName); releaseUdfFuncHandle(pCtx->udfName);
}
freeUdfInterBuf(&newState); freeUdfInterBuf(&newState);
return udfCode; return udfCode;
} }
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t udfCode = 0;
SUdfcUvSession *session = udfRes->session; UdfcFuncHandle handle = 0;
if (session == NULL) { if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
return TSDB_CODE_UDF_NO_FUNC_HANDLE; fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode);
return udfCode;
} }
SUdfcUvSession *session = handle;
SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes);
udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen;

11
source/libs/index/src/indexComm.c
View File

@ -316,46 +316,57 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int64_t*)src, *dst, -1); indexInt2str(*(int64_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint8_t*)src, *dst, 1); indexInt2str(*(uint8_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int8_t*)src, *dst, 1); indexInt2str(*(int8_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int16_t*)src, *dst, -1); indexInt2str(*(int16_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint16_t*)src, *dst, -1); indexInt2str(*(uint16_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(int32_t*)src, *dst, -1); indexInt2str(*(int32_t*)src, *dst, -1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint32_t*)src, *dst, 1); indexInt2str(*(uint32_t*)src, *dst, 1);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%" PRIu64, *(uint64_t*)src); sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
indexInt2str(*(uint64_t*)src, *dst, 1); indexInt2str(*(uint64_t*)src, *dst, 1);
tlen = strlen(*dst);
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(float*)src); sprintf(*dst, "%.9lf", *(float*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
*dst = taosMemoryCalloc(1, bufSize + 1); *dst = taosMemoryCalloc(1, bufSize + 1);
sprintf(*dst, "%.9lf", *(double*)src); sprintf(*dst, "%.9lf", *(double*)src);
tlen = strlen(*dst);
break; break;
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));

37
source/libs/index/test/jsonUT.cc
View File

@ -66,6 +66,17 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp
indexMultiTermDestroy(terms); indexMultiTermDestroy(terms);
} }
static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId,
int8_t operType = DEL_VALUE) {
SIndexTerm* term =
indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen);
SIndexMultiTerm* terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
tIndexJsonPut(index, terms, (int64_t)tableId);
indexMultiTermDestroy(terms);
}
static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType,
SArray** result) { SArray** result) {
std::string colName(colNam); std::string colName(colNam);
@ -581,37 +592,45 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) {
{ {
double val = 10.0; double val = 10.0;
std::string colName("test1");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i);
} }
} }
{ {
double val = 2.0; double val = 2.0;
std::string colName("test1");
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000);
} }
} }
{ {
SArray* res = NULL; SArray* res = NULL;
std::string colName("test1"); std::string colName("test1");
double val = 1.9; double val = 1.9;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(2000, taosArrayGetSize(res)); EXPECT_EQ(2000, taosArrayGetSize(res));
} }
{ {
SArray* res = NULL; SArray* res = NULL;
std::string colName("test1");
double val = 2.1; double val = 2.1;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(1000, taosArrayGetSize(res)); EXPECT_EQ(1000, taosArrayGetSize(res));
} }
{ {
std::string colName("test1");
SArray* res = NULL; SArray* res = NULL;
double val = 2.1; double val = 2.1;
Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res);
EXPECT_EQ(1000, taosArrayGetSize(res));
}
{
SArray* res = NULL;
double val = 10.0;
Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_EQUAL, &res);
EXPECT_EQ(2000, taosArrayGetSize(res));
}
{
SArray* res = NULL;
double val = 10.0;
Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_THAN, &res);
EXPECT_EQ(1000, taosArrayGetSize(res)); EXPECT_EQ(1000, taosArrayGetSize(res));
} }
} }

2
source/libs/tdb/CMakeLists.txt
View File

@ -7,7 +7,7 @@ target_sources(tdb
"src/db/tdbUtil.c" "src/db/tdbUtil.c"
"src/db/tdbBtree.c" "src/db/tdbBtree.c"
"src/db/tdbDb.c" "src/db/tdbDb.c"
"src/db/tdbEnv.c" "src/db/tdbTable.c"
"src/db/tdbTxn.c" "src/db/tdbTxn.c"
"src/db/tdbPage.c" "src/db/tdbPage.c"
"src/db/tdbOs.c" "src/db/tdbOs.c"

56
source/libs/tdb/inc/tdb.h
View File

@ -25,40 +25,40 @@ extern "C" {
typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
// exposed types // exposed types
typedef struct STEnv TENV;
typedef struct STDB TDB; typedef struct STDB TDB;
typedef struct STDBC TDBC; typedef struct STTB TTB;
typedef struct STBC TBC;
typedef struct STxn TXN; typedef struct STxn TXN;
// TENV
int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv);
int tdbEnvClose(TENV *pEnv);
int tdbBegin(TENV *pEnv, TXN *pTxn);
int tdbCommit(TENV *pEnv, TXN *pTxn);
// TDB // TDB
int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb);
int tdbClose(TDB *pDb); int tdbClose(TDB *pDb);
int tdbDrop(TDB *pDb); int tdbBegin(TDB *pDb, TXN *pTxn);
int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); int tdbCommit(TDB *pDb, TXN *pTxn);
int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn);
int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
// TDBC // TTB
int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb);
int tdbDbcClose(TDBC *pDbc); int tdbTbClose(TTB *pTb);
int tdbDbcIsValid(TDBC *pDbc); int tdbTbDrop(TTB *pTb);
int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
int tdbDbcMoveToFirst(TDBC *pDbc); int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn);
int tdbDbcMoveToLast(TDBC *pDbc); int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int tdbDbcMoveToNext(TDBC *pDbc); int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbDbcMoveToPrev(TDBC *pDbc); int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
int tdbDbcDelete(TDBC *pDbc); // TBC
int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn);
int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); int tdbTbcClose(TBC *pTbc);
int tdbTbcIsValid(TBC *pTbc);
int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c);
int tdbTbcMoveToFirst(TBC *pTbc);
int tdbTbcMoveToLast(TBC *pTbc);
int tdbTbcMoveToNext(TBC *pTbc);
int tdbTbcMoveToPrev(TBC *pTbc);
int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
int tdbTbcDelete(TBC *pTbc);
int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert);
// TXN // TXN
#define TDB_TXN_WRITE 0x1 #define TDB_TXN_WRITE 0x1

220
source/libs/tdb/src/db/tdbDb.c
View File

@ -15,134 +15,164 @@
#include "tdbInt.h" #include "tdbInt.h"
struct STDB { int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) {
TENV *pEnv;
SBTree *pBt;
};
struct STDBC {
SBTC btc;
};
int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) {
TDB *pDb; TDB *pDb;
SPager *pPager; int dsize;
int zsize;
int tsize;
u8 *pPtr;
int ret; int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppDb = NULL; *ppDb = NULL;
pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb)); dsize = strlen(rootDir);
if (pDb == NULL) { zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1; return -1;
} }
// pDb->pEnv pDb = (TDB *)pPtr;
pDb->pEnv = pEnv; pPtr += sizeof(*pDb);
// pDb->rootDir
pDb->rootDir = pPtr;
memcpy(pDb->rootDir, rootDir, dsize);
pDb->rootDir[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pDb->jfname
pDb->jfname = pPtr;
memcpy(pDb->jfname, rootDir, dsize);
pDb->jfname[dsize] = '/';
memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
pPager = tdbEnvGetPager(pEnv, fname); pDb->jfd = -1;
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
tdbEnvAddPager(pEnv, pPager); pDb->nPgrHash = 8;
} tsize = sizeof(SPager *) * pDb->nPgrHash;
pDb->pgrHash = tdbOsMalloc(tsize);
ASSERT(pPager != NULL); if (pDb->pgrHash == NULL) {
// pDb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt));
if (ret < 0) {
return -1; return -1;
} }
memset(pDb->pgrHash, 0, tsize);
mkdir(rootDir, 0755);
*ppDb = pDb; *ppDb = pDb;
return 0; return 0;
} }
int tdbClose(TDB *pDb) { int tdbClose(TDB *pDb) {
SPager *pPager;
if (pDb) { if (pDb) {
tdbBtreeClose(pDb->pBt); for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {
pDb->pgrList = pPager->pNext;
tdbPagerClose(pPager);
}
tdbPCacheClose(pDb->pCache);
tdbOsFree(pDb->pgrHash);
tdbOsFree(pDb); tdbOsFree(pDb);
} }
return 0; return 0;
} }
int tdbDrop(TDB *pDb) { int tdbBegin(TDB *pDb, TXN *pTxn) {
// TODO SPager *pPager;
return 0;
}
int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
}
int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); }
int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn);
}
int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen);
}
int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen);
}
int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) {
int ret; int ret;
TDBC *pDbc = NULL;
*ppDbc = NULL; for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc)); ret = tdbPagerBegin(pPager, pTxn);
if (pDbc == NULL) { if (ret < 0) {
ASSERT(0);
return -1; return -1;
} }
tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn);
*ppDbc = pDbc;
return 0;
}
int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); }
int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); }
int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); }
int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); }
int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); }
int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) {
return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen);
}
int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); }
int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert);
}
int tdbDbcClose(TDBC *pDbc) {
if (pDbc) {
tdbBtcClose(&pDbc->btc);
tdbOsFree(pDbc);
} }
return 0; return 0;
} }
int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } int tdbCommit(TDB *pDb, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
SPager *tdbEnvGetPager(TDB *pDb, const char *fname) {
u32 hash;
SPager **ppPager;
hash = tdbCstringHash(fname);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
}
return *ppPager;
}
void tdbEnvAddPager(TDB *pDb, SPager *pPager) {
u32 hash;
SPager **ppPager;
// rehash if neccessary
if (pDb->nPager + 1 > pDb->nPgrHash) {
// TODO
}
// add to list
pPager->pNext = pDb->pgrList;
pDb->pgrList = pPager;
// add to hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
pPager->pHashNext = *ppPager;
*ppPager = pPager;
// increase the counter
pDb->nPager++;
}
void tdbEnvRemovePager(TDB *pDb, SPager *pPager) {
u32 hash;
SPager **ppPager;
// remove from the list
for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// remove from hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// decrease the counter
pDb->nPager--;
// rehash if necessary
if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) {
// TODO
}
}

178
source/libs/tdb/src/db/tdbEnv.c
View File

@ -1,178 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) {
TENV *pEnv;
int dsize;
int zsize;
int tsize;
u8 *pPtr;
int ret;
*ppEnv = NULL;
dsize = strlen(rootDir);
zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;
pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
if (pPtr == NULL) {
return -1;
}
pEnv = (TENV *)pPtr;
pPtr += sizeof(*pEnv);
// pEnv->rootDir
pEnv->rootDir = pPtr;
memcpy(pEnv->rootDir, rootDir, dsize);
pEnv->rootDir[dsize] = '\0';
pPtr = pPtr + dsize + 1;
// pEnv->jfname
pEnv->jfname = pPtr;
memcpy(pEnv->jfname, rootDir, dsize);
pEnv->jfname[dsize] = '/';
memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';
pEnv->jfd = -1;
ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache));
if (ret < 0) {
return -1;
}
pEnv->nPgrHash = 8;
tsize = sizeof(SPager *) * pEnv->nPgrHash;
pEnv->pgrHash = tdbOsMalloc(tsize);
if (pEnv->pgrHash == NULL) {
return -1;
}
memset(pEnv->pgrHash, 0, tsize);
mkdir(rootDir, 0755);
*ppEnv = pEnv;
return 0;
}
int tdbEnvClose(TENV *pEnv) {
SPager *pPager;
if (pEnv) {
for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) {
pEnv->pgrList = pPager->pNext;
tdbPagerClose(pPager);
}
tdbPCacheClose(pEnv->pCache);
tdbOsFree(pEnv->pgrHash);
tdbOsFree(pEnv);
}
return 0;
}
int tdbBegin(TENV *pEnv, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerBegin(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
int tdbCommit(TENV *pEnv, TXN *pTxn) {
SPager *pPager;
int ret;
for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
ret = tdbPagerCommit(pPager, pTxn);
if (ret < 0) {
ASSERT(0);
return -1;
}
}
return 0;
}
SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) {
u32 hash;
SPager **ppPager;
hash = tdbCstringHash(fname);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
}
return *ppPager;
}
void tdbEnvAddPager(TENV *pEnv, SPager *pPager) {
u32 hash;
SPager **ppPager;
// rehash if neccessary
if (pEnv->nPager + 1 > pEnv->nPgrHash) {
// TODO
}
// add to list
pPager->pNext = pEnv->pgrList;
pEnv->pgrList = pPager;
// add to hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
pPager->pHashNext = *ppPager;
*ppPager = pPager;
// increase the counter
pEnv->nPager++;
}
void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) {
u32 hash;
SPager **ppPager;
// remove from the list
for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// remove from hash
hash = tdbCstringHash(pPager->dbFileName);
ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash];
for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
}
ASSERT(*ppPager == pPager);
*ppPager = pPager->pNext;
// decrease the counter
pEnv->nPager--;
// rehash if necessary
if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) {
// TODO
}
}

148
source/libs/tdb/src/db/tdbTable.c Normal file
View File

@ -0,0 +1,148 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tdbInt.h"
struct STTB {
TDB *pEnv;
SBTree *pBt;
};
struct STBC {
SBTC btc;
};
int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) {
TTB *pTb;
SPager *pPager;
int ret;
char fFullName[TDB_FILENAME_LEN];
SPage *pPage;
SPgno pgno;
*ppTb = NULL;
pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb));
if (pTb == NULL) {
return -1;
}
// pTb->pEnv
pTb->pEnv = pEnv;
pPager = tdbEnvGetPager(pEnv, fname);
if (pPager == NULL) {
snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname);
ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager);
if (ret < 0) {
return -1;
}
tdbEnvAddPager(pEnv, pPager);
}
ASSERT(pPager != NULL);
// pTb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt));
if (ret < 0) {
return -1;
}
*ppTb = pTb;
return 0;
}
int tdbTbClose(TTB *pTb) {
if (pTb) {
tdbBtreeClose(pTb->pBt);
tdbOsFree(pTb);
}
return 0;
}
int tdbTbDrop(TTB *pTb) {
// TODO
return 0;
}
int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn);
}
int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pTb->pBt, pKey, kLen, pTxn); }
int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
return tdbBtreeUpsert(pTb->pBt, pKey, kLen, pVal, vLen, pTxn);
}
int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen) {
return tdbBtreeGet(pTb->pBt, pKey, kLen, ppVal, vLen);
}
int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) {
return tdbBtreePGet(pTb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen);
}
int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) {
int ret;
TBC *pTbc = NULL;
*ppTbc = NULL;
pTbc = (TBC *)tdbOsMalloc(sizeof(*pTbc));
if (pTbc == NULL) {
return -1;
}
tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn);
*ppTbc = pTbc;
return 0;
}
int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); }
int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); }
int tdbTbcMoveToLast(TBC *pTbc) { return tdbBtcMoveToLast(&pTbc->btc); }
int tdbTbcMoveToNext(TBC *pTbc) { return tdbBtcMoveToNext(&pTbc->btc); }
int tdbTbcMoveToPrev(TBC *pTbc) { return tdbBtcMoveToPrev(&pTbc->btc); }
int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) {
return tdbBtcGet(&pTbc->btc, ppKey, pkLen, ppVal, pvLen);
}
int tdbTbcDelete(TBC *pTbc) { return tdbBtcDelete(&pTbc->btc); }
int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen);
}
int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert);
}
int tdbTbcClose(TBC *pTbc) {
if (pTbc) {
tdbBtcClose(&pTbc->btc);
tdbOsFree(pTbc);
}
return 0;
}
int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); }

12
source/libs/tdb/src/inc/tdbInt.h
View File

@ -103,9 +103,9 @@ typedef struct SPage SPage;
#define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED)
// tdbEnv.c ==================================== // tdbEnv.c ====================================
void tdbEnvAddPager(TENV *pEnv, SPager *pPager); void tdbEnvAddPager(TDB *pEnv, SPager *pPager);
void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); void tdbEnvRemovePager(TDB *pEnv, SPager *pPager);
SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); SPager *tdbEnvGetPager(TDB *pEnv, const char *fname);
// tdbBtree.c ==================================== // tdbBtree.c ====================================
typedef struct SBTree SBTree; typedef struct SBTree SBTree;
@ -334,7 +334,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
return pCell; return pCell;
} }
struct STEnv { struct STDB {
char *rootDir; char *rootDir;
char *jfname; char *jfname;
int jfd; int jfd;
@ -357,8 +357,8 @@ struct SPager {
SPgno dbOrigSize; SPgno dbOrigSize;
SPage *pDirty; SPage *pDirty;
u8 inTran; u8 inTran;
SPager *pNext; // used by TENV SPager *pNext; // used by TDB
SPager *pHashNext; // used by TENV SPager *pHashNext; // used by TDB
}; };
#ifdef __cplusplus #ifdef __cplusplus

104
source/libs/tdb/test/tdbTest.cpp
View File

@ -117,8 +117,8 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
TEST(tdb_test, simple_insert1) { TEST(tdb_test, simple_insert1) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
int nData = 1000000; int nData = 1000000;
TXN txn; TXN txn;
@ -126,12 +126,12 @@ TEST(tdb_test, simple_insert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); ret = tdbOpen("tdb", 4096, 64, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
compFunc = tKeyCmpr; compFunc = tKeyCmpr;
ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
{ {
@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) {
for (int iData = 1; iData <= nData; iData++) { for (int iData = 1; iData <= nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(val, "value%d", iData); sprintf(val, "value%d", iData);
ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// if pool is full, commit the transaction and start a new one // if pool is full, commit the transaction and start a new one
@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) {
sprintf(key, "key%d", i); sprintf(key, "key%d", i);
sprintf(val, "value%d", i); sprintf(val, "value%d", i);
ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen);
ASSERT(ret == 0); ASSERT(ret == 0);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
@ -193,19 +193,19 @@ TEST(tdb_test, simple_insert1) {
} }
{ // Iterate to query the DB data { // Iterate to query the DB data
TDBC *pDBC; TBC *pDBC;
void *pKey = NULL; void *pKey = NULL;
void *pVal = NULL; void *pVal = NULL;
int vLen, kLen; int vLen, kLen;
int count = 0; int count = 0;
ret = tdbDbcOpen(pDb, &pDBC, NULL); ret = tdbTbcOpen(pDb, &pDBC, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbDbcMoveToFirst(pDBC); tdbTbcMoveToFirst(pDBC);
for (;;) { for (;;) {
ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
// std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
@ -217,28 +217,28 @@ TEST(tdb_test, simple_insert1) {
GTEST_ASSERT_EQ(count, nData); GTEST_ASSERT_EQ(count, nData);
tdbDbcClose(pDBC); tdbTbcClose(pDBC);
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
} }
} }
ret = tdbDrop(pDb); ret = tdbTbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Close a database // Close a database
tdbClose(pDb); tdbTbClose(pDb);
// Close Env // Close Env
ret = tdbEnvClose(pEnv); ret = tdbClose(pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
TEST(tdb_test, simple_insert2) { TEST(tdb_test, simple_insert2) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
tdb_cmpr_fn_t compFunc; tdb_cmpr_fn_t compFunc;
int nData = 1000000; int nData = 1000000;
TXN txn; TXN txn;
@ -246,12 +246,12 @@ TEST(tdb_test, simple_insert2) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// Open Env // Open Env
ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); ret = tdbOpen("tdb", 1024, 10, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Create a database // Create a database
compFunc = tDefaultKeyCmpr; compFunc = tDefaultKeyCmpr;
ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
{ {
@ -271,24 +271,24 @@ TEST(tdb_test, simple_insert2) {
for (int iData = 1; iData <= nData; iData++) { for (int iData = 1; iData <= nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(val, "value%d", iData); sprintf(val, "value%d", iData);
ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
{ // Iterate to query the DB data { // Iterate to query the DB data
TDBC *pDBC; TBC *pDBC;
void *pKey = NULL; void *pKey = NULL;
void *pVal = NULL; void *pVal = NULL;
int vLen, kLen; int vLen, kLen;
int count = 0; int count = 0;
ret = tdbDbcOpen(pDb, &pDBC, NULL); ret = tdbTbcOpen(pDb, &pDBC, NULL);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbDbcMoveToFirst(pDBC); tdbTbcMoveToFirst(pDBC);
for (;;) { for (;;) {
ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
if (ret < 0) break; if (ret < 0) break;
// std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
@ -300,7 +300,7 @@ TEST(tdb_test, simple_insert2) {
GTEST_ASSERT_EQ(count, nData); GTEST_ASSERT_EQ(count, nData);
tdbDbcClose(pDBC); tdbTbcClose(pDBC);
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
@ -311,29 +311,29 @@ TEST(tdb_test, simple_insert2) {
tdbCommit(pEnv, &txn); tdbCommit(pEnv, &txn);
tdbTxnClose(&txn); tdbTxnClose(&txn);
ret = tdbDrop(pDb); ret = tdbTbDrop(pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// Close a database // Close a database
tdbClose(pDb); tdbTbClose(pDb);
// Close Env // Close Env
ret = tdbEnvClose(pEnv); ret = tdbClose(pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
TEST(tdb_test, simple_delete1) { TEST(tdb_test, simple_delete1) {
int ret; int ret;
TDB *pDb; TTB *pDb;
char key[128]; char key[128];
char data[128]; char data[128];
TXN txn; TXN txn;
TENV *pEnv; TDB *pEnv;
SPoolMem *pPool; SPoolMem *pPool;
void *pKey = NULL; void *pKey = NULL;
void *pData = NULL; void *pData = NULL;
int nKey; int nKey;
TDBC *pDbc; TBC *pDbc;
int nData; int nData;
int nKV = 69; int nKV = 69;
@ -342,11 +342,11 @@ TEST(tdb_test, simple_delete1) {
pPool = openPool(); pPool = openPool();
// open env // open env
ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); ret = tdbOpen("tdb", 1024, 256, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = 0; iData < nKV; iData++) { for (int iData = 0; iData < nKV; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0);
} }
@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = nKV - 1; iData > 30; iData--) { for (int iData = nKV - 1; iData > 30; iData--) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
ret = tdbDelete(pDb, key, strlen(key), &txn); ret = tdbTbDelete(pDb, key, strlen(key), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) {
for (int iData = 0; iData < nKV; iData++) { for (int iData = 0; iData < nKV; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
if (iData <= 30) { if (iData <= 30) {
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} else { } else {
@ -391,15 +391,15 @@ TEST(tdb_test, simple_delete1) {
} }
// loop to iterate the data // loop to iterate the data
tdbDbcOpen(pDb, &pDbc, NULL); tdbTbcOpen(pDb, &pDbc, NULL);
ret = tdbDbcMoveToFirst(pDbc); ret = tdbTbcMoveToFirst(pDbc);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
pKey = NULL; pKey = NULL;
pData = NULL; pData = NULL;
for (;;) { for (;;) {
ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); ret = tdbTbcNext(pDbc, &pKey, &nKey, &pData, &nData);
if (ret < 0) break; if (ret < 0) break;
std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " ";
@ -407,20 +407,20 @@ TEST(tdb_test, simple_delete1) {
std::cout << std::endl; std::cout << std::endl;
} }
tdbDbcClose(pDbc); tdbTbcClose(pDbc);
tdbCommit(pEnv, &txn); tdbCommit(pEnv, &txn);
closePool(pPool); closePool(pPool);
tdbClose(pDb); tdbTbClose(pDb);
tdbEnvClose(pEnv); tdbClose(pEnv);
} }
TEST(tdb_test, simple_upsert1) { TEST(tdb_test, simple_upsert1) {
int ret; int ret;
TENV *pEnv; TDB *pEnv;
TDB *pDb; TTB *pDb;
int nData = 100000; int nData = 100000;
char key[64]; char key[64];
char data[64]; char data[64];
@ -431,11 +431,11 @@ TEST(tdb_test, simple_upsert1) {
taosRemoveDir("tdb"); taosRemoveDir("tdb");
// open env // open env
ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); ret = tdbOpen("tdb", 4096, 64, &pEnv);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
// open database // open database
ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); ret = tdbTbOpen("db.db", -1, -1, NULL, pEnv, &pDb);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
pPool = openPool(); pPool = openPool();
@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d", iData); sprintf(data, "data%d", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
} }
@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d-u", iData); sprintf(data, "data%d-u", iData);
ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) {
for (int iData = 0; iData < nData; iData++) { for (int iData = 0; iData < nData; iData++) {
sprintf(key, "key%d", iData); sprintf(key, "key%d", iData);
sprintf(data, "data%d-u", iData); sprintf(data, "data%d-u", iData);
ret = tdbGet(pDb, key, strlen(key), &pData, &nData); ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0);
} }
tdbClose(pDb); tdbTbClose(pDb);
tdbEnvClose(pEnv); tdbClose(pEnv);
} }

1
source/libs/transport/src/transSrv.c
View File

@ -264,6 +264,7 @@ static void uvHandleReq(SSrvConn* pConn) {
CONN_SHOULD_RELEASE(pConn, pHead); CONN_SHOULD_RELEASE(pConn, pHead);
STransMsg transMsg; STransMsg transMsg;
memset(&transMsg, 0, sizeof(transMsg));
transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.contLen = transContLenFromMsg(pHead->msgLen);
transMsg.pCont = pHead->content; transMsg.pCont = pHead->content;
transMsg.msgType = pHead->msgType; transMsg.msgType = pHead->msgType;

35
tests/system-test/7-tmq/subscribeDb.py
View File

@ -219,6 +219,41 @@ class TDTestCase:
tdSql.query("drop topic %s"%topicName1) tdSql.query("drop topic %s"%topicName1)
tdLog.info("creat the same topic name , and start to consume")
self.initConsumerTable()
tdLog.info("create topics from db")
topicName1 = 'topic_db1'
tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName']))
consumerId = 0
expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"]
topicList = topicName1
ifcheckdata = 0
ifManualCommit = 0
keyList = 'group.id:cgrp1,\
enable.auto.commit:false,\
auto.commit.interval.ms:6000,\
auto.offset.reset:earliest'
self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
pollDelay = 5
showMsg = 1
showRow = 1
self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow)
expectRows = 1
resultList = self.selectConsumeResult(expectRows)
totalConsumeRows = 0
for i in range(expectRows):
totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt))
tdLog.exit("tmq consume rows error!")
tdSql.query("drop topic %s"%topicName1)
tdLog.printNoPrefix("======== test case 1 end ...... ") tdLog.printNoPrefix("======== test case 1 end ...... ")
def tmqCase2(self, cfgPath, buildPath): def tmqCase2(self, cfgPath, buildPath):