diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 7ba6e5044c..9fa657a2a6 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -60,9 +60,9 @@ typedef struct { ReportStartup reportStartupFp; } SMsgCb; -void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb); -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg); -int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype); +void tmsgSetDefault(const SMsgCb* msgcb); +int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); +int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); diff --git a/include/common/ttypes.h b/include/common/ttypes.h index cab429d136..14428bfc43 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -179,6 +179,8 @@ typedef struct { } \ } while (0) +//TODO: use varchar(0) to represent NULL type +#define IS_NULL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_NULL) #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 162b6fb2ed..8f300c96c5 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -42,7 +42,7 @@ typedef struct SReadHandle { #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 typedef enum { - OPTR_EXEC_MODEL_BATCH = 0x1, + OPTR_EXEC_MODEL_BATCH = 0x1, OPTR_EXEC_MODEL_STREAM = 0x2, } EOPTR_EXEC_MODEL; @@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -169,7 +169,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes); +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes); #ifdef __cplusplus } diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 27dae6d210..291e08fdbf 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -208,6 +208,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ecfc991331..2a2fdea537 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -830,10 +830,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } // init hb timer - tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + if (tmq->hbTimer == NULL) { + tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + } // init auto commit timer - if (tmq->autoCommit) { + if (tmq->autoCommit && tmq->commitTimer == NULL) { tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); } @@ -1456,9 +1458,18 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - tmq_list_t* lst = tmq_list_new(); - tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); + tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp == TMQ_RESP_ERR__SUCCESS) { + // TODO: free resources + return TMQ_RESP_ERR__SUCCESS; + } else { + return TMQ_RESP_ERR__FAIL; + } + + tmq_list_t* lst = tmq_list_new(); + rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (rsp == TMQ_RESP_ERR__SUCCESS) { // TODO: free resources return TMQ_RESP_ERR__SUCCESS; diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index f69fb65f04..126a4c023a 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -17,46 +17,46 @@ #include "tmsgcb.h" #include "taoserror.h" -static SMsgCb tsDefaultMsgCb; +static SMsgCb defaultMsgCb; -void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; } +void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pMsg) { - PutToQueueFp fp = pMsgCb->queueFps[qtype]; - return (*fp)(pMsgCb->mgmt, pMsg); +int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { + PutToQueueFp fp = msgcb->queueFps[qtype]; + return (*fp)(msgcb->mgmt, pMsg); } -int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) { - GetQueueSizeFp fp = pMsgCb->qsizeFp; - return (*fp)(pMsgCb->mgmt, vgId, qtype); +int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) { + GetQueueSizeFp fp = msgcb->qsizeFp; + return (*fp)(msgcb->mgmt, vgId, qtype); } int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { - SendReqFp fp = tsDefaultMsgCb.sendReqFp; + SendReqFp fp = defaultMsgCb.sendReqFp; return (*fp)(epSet, pMsg); } void tmsgSendRsp(SRpcMsg* pMsg) { - SendRspFp fp = tsDefaultMsgCb.sendRspFp; + SendRspFp fp = defaultMsgCb.sendRspFp; return (*fp)(pMsg); } void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { - SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp; + SendRedirectRspFp fp = defaultMsgCb.sendRedirectRspFp; (*fp)(pMsg, pNewEpSet); } void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { - RegisterBrokenLinkArgFp fp = tsDefaultMsgCb.registerBrokenLinkArgFp; + RegisterBrokenLinkArgFp fp = defaultMsgCb.registerBrokenLinkArgFp; (*fp)(pMsg); } void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { - ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp; + ReleaseHandleFp fp = defaultMsgCb.releaseHandleFp; (*fp)(pHandle, type); } void tmsgReportStartup(const char* name, const char* desc) { - ReportStartup fp = tsDefaultMsgCb.reportStartupFp; + ReportStartup fp = defaultMsgCb.reportStartupFp; (*fp)(name, desc); } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index d97949d3f7..14c6250b2a 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -63,7 +63,7 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { bmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 7afa86a377..7894eb5acb 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -86,7 +86,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; SRpcMsg rpcRsp = {0}; - dTrace("send status msg to mnode, app:%p", rpcMsg.info.ahandle); + dTrace("send status msg to mnode"); SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 787c6c98c8..be8ddf1430 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -160,7 +160,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { rpcSendResponse(&rsp); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index ee65335382..98e234af3c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -29,7 +29,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - dTrace("msg:%p, get from mnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, get from mnode queue", pMsg); switch (pMsg->msgType) { case TDMT_DND_ALTER_MNODE: @@ -51,7 +51,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { mmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -73,7 +73,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { } } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -84,13 +84,20 @@ static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { return 0; } -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); } +int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); +} -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); } +int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); +} -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); } +int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); +} -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); +int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { @@ -101,25 +108,25 @@ static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); taosWriteQitem(pWorker->queue, pMsg); return 0; } -int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pRpc); +int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pMsg); } -int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pRpc); +int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pMsg); } -int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return mmPutRpcMsgToWorker(&pMgmt->readWorker, pRpc); +int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg); } -int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pRpc) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pRpc); } +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg qCfg = { diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index d5c82a043c..444c42717a 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -44,7 +44,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } @@ -60,7 +60,7 @@ static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -76,7 +76,7 @@ static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -105,7 +105,7 @@ static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SR return -1; } - dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); taosWriteQitem(pWorker->queue, pMsg); return 0; diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 10bf6ff9bf..c94cf527c7 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -44,7 +44,7 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { smSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pRpc->pCont); taosFreeQitem(pMsg); } @@ -166,7 +166,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -174,7 +174,7 @@ int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMonitorQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -187,7 +187,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -195,7 +195,7 @@ int32_t smPutNodeMsgToUniqueQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToSharedQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->sharedWorker; - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d7298804fe..9f77180cd8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -51,7 +51,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, not processed in vnode-mgmt/monitor queue", pMsg); + dError("msg:%p, not processed in vnode queue", pMsg); } if (msgType & 1u) { @@ -59,7 +59,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -67,13 +67,13 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, will be processed in vnode-query queue", pMsg); + dTrace("msg:%p, get from vnode-query queue", pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -82,13 +82,13 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg); + dTrace("msg:%p, get from vnode-fetch queue", pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -96,7 +96,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg rsp; SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); if (pArray == NULL) { @@ -108,7 +107,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SRpcMsg *pMsg = NULL; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - dTrace("msg:%p, will be processed in vnode-write queue", pMsg); + dTrace("msg:%p, get from vnode-write queue", pMsg); if (taosArrayPush(pArray, &pMsg) == NULL) { dTrace("msg:%p, failed to process since %s", pMsg, terrstr()); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); @@ -116,21 +115,12 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } for (int i = 0; i < taosArrayGetSize(pArray); i++) { - SRpcMsg *pMsg; - SRpcMsg *pRpc; + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + SRpcMsg rsp = {.info = pMsg->info, .pCont = NULL, .contLen = 0}; - pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - pRpc = pMsg; - - rsp.info = pRpc->info; - rsp.pCont = NULL; - rsp.contLen = 0; - - int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false); + int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - // rsp.code = TSDB_CODE_SYN_NOT_LEADER; - // tmsgSendRsp(&rsp); - dTrace("syncPropose not leader redirect, vgId:%d ", syncGetVgId(vnodeGetSyncHandle(pVnode->pImpl))); + dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId); rsp.code = TSDB_CODE_RPC_REDIRECT; SEpSet newEpSet; syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); @@ -160,7 +150,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -181,7 +171,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // apply data into tsdb if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { rsp.code = terrno; - dTrace("vnodeProcessWriteReq error, code:%d", terrno); + dTrace("msg:%p, process write error since %s", pMsg, terrstr()); } syncApplyMsgDestroy(pSyncApplyMsg); @@ -215,7 +205,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf SRpcMsg rsp = {0}; rsp.code = terrno; rsp.info = pMsg->info; - dTrace("vmProcessSyncQueue error, code:%d", terrno); + dTrace("msg:%p, process sync queue error since code:%s", pMsg, terrstr()); tmsgSendRsp(&rsp); } } @@ -227,18 +217,18 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - dTrace("msg:%p, will be processed in vnode-merge queue", pMsg); + dTrace("msg:%p, get from vnode-merge queue", pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -255,29 +245,29 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); + dError("vgId:%d, failed to put msg:%p into vnode-queue since %s", pHead->vgId, pMsg, terrstr()); return terrno != 0 ? terrno : -1; } switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-query queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-fetch queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-write queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-sync queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, type:%s will be written into vnode-merge queue", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; default: @@ -312,7 +302,7 @@ int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->mgmtWorker; - dTrace("msg:%p, will be put into vnode-mgmt queue, worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into vnode-mgmt worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -320,7 +310,7 @@ int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, will be put into vnode-monitor queue, worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -332,35 +322,33 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q if (pVnode == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - int32_t code = 0; + int32_t code = 0; if (pMsg != NULL) { - dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - // if (pMsg->handle != NULL) assert(pMsg->refId != 0); switch (qtype) { case WRITE_QUEUE: - dTrace("msg:%p, will be put into vnode-write queue", pMsg); + dTrace("msg:%p, create and put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case QUERY_QUEUE: - dTrace("msg:%p, will be put into vnode-query queue", pMsg); + dTrace("msg:%p, create and put into vnode-query queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); + dTrace("msg:%p, create and put into vnode-fetch queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case APPLY_QUEUE: - dTrace("msg:%p, will be put into vnode-apply queue", pMsg); + dTrace("msg:%p, create and put into vnode-apply queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pApplyQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, will be put into vnode-merge queue", pMsg); + dTrace("msg:%p, create and put into vnode-merge queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, will be put into vnode-sync queue", pMsg); + dTrace("msg:%p, create and put into vnode-sync queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; default: diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 5d9f61b846..30d7750f79 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -274,25 +274,24 @@ static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) } } -void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("net test req is received"); - SRpcMsg rsp = {.code = 0, .info = pReq->info}; - rsp.pCont = rpcMallocCont(pReq->contLen); +void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg) { + dDebug("msg:%p, net test req will be processed", pMsg); + SRpcMsg rsp = {.code = 0, .info = pMsg->info}; + rsp.pCont = rpcMallocCont(pMsg->contLen); if (rsp.pCont == NULL) { rsp.code = TSDB_CODE_OUT_OF_MEMORY; } else { - rsp.contLen = pReq->contLen; + rsp.contLen = pMsg->contLen; } rpcSendResponse(&rsp); } -void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("server startup status req is received"); - +void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { + dDebug("msg:%p, server startup status req will be processed", pMsg); SServerStatusRsp statusRsp = {0}; dmGetServerStartupStatus(pDnode, &statusRsp); - SRpcMsg rspMsg = {.info = pReq->info}; + SRpcMsg rspMsg = {.info = pMsg->info}; int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); if (rspLen < 0) { rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index f23db5da16..ecfa37725a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -37,6 +37,7 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) { dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); return -1; } + dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->proc.shm.id, shmsize); return 0; } @@ -60,7 +61,7 @@ static int32_t dmNewProc(SMgmtWrapper *pWrapper, EDndNodeType ntype) { taosIgnSignal(SIGCHLD); pWrapper->proc.pid = pid; - dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid); + dInfo("node:%s, continue running in new pid:%d", pWrapper->name, pid); return 0; } @@ -77,7 +78,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); if (pWrapper->ntype == DNODE || InChildProc(pWrapper)) { - tmsgSetDefaultMsgCb(&input.msgCb); + tmsgSetDefault(&input.msgCb); } if (OnlyInSingleProc(pWrapper)) { @@ -177,11 +178,11 @@ void dmCloseNode(SMgmtWrapper *pWrapper) { if (OnlyInParentProc(pWrapper)) { int32_t pid = pWrapper->proc.pid; if (pid > 0 && taosProcExist(pid)) { - dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pid); + dInfo("node:%s, send kill signal to the child pid:%d", pWrapper->name, pid); taosKillProc(pid); - dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pid); + dInfo("node:%s, wait for child pid:%d to stop", pWrapper->name, pid); taosWaitProc(pid); - dInfo("node:%s, child process:%d is stopped", pWrapper->name, pid); + dInfo("node:%s, child pid:%d is stopped", pWrapper->name, pid); } } @@ -255,7 +256,7 @@ static void dmWatchNodes(SDnode *pDnode) { if (!OnlyInParentProc(pWrapper)) continue; if (proc->pid <= 0 || !taosProcExist(proc->pid)) { - dError("node:%s, process:%d is killed and needs to restart", pWrapper->name, proc->pid); + dError("node:%s, pid:%d is killed and needs to restart", pWrapper->name, proc->pid); dmCloseProcRpcHandles(&pWrapper->proc); dmNewProc(pWrapper, ntype); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 0a8b5135aa..757f609731 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -162,7 +162,7 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg return 0; } -static int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { +static inline int32_t dmPopFromProcQueue(SProcQueue *queue, SRpcMsg **ppMsg, EProcFuncType *pFuncType) { tsem_wait(&queue->sem); taosThreadMutexLock(&queue->mutex); @@ -412,7 +412,7 @@ void dmCleanupProc(struct SMgmtWrapper *pWrapper) { SProc *proc = &pWrapper->proc; if (proc->name == NULL) return; - dDebug("node:%s, start to clean up proc", pWrapper->name); + dDebug("node:%s, start to cleanup proc", pWrapper->name); dmStopProc(proc); dmCleanupProcQueue(proc->cqueue); dmCleanupProcQueue(proc->pqueue); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index c9100aab9d..b4c8a0807c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -43,8 +43,8 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { return -1; } + dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); pMsg->info.wrapper = pWrapper; - dTrace("msg:%p, will be processed by %s, handle:%p", pMsg, pWrapper->name, pMsg->info.handle); return (*msgFp)(pWrapper->pMgmt, pMsg); } @@ -56,8 +56,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; SMgmtWrapper *pWrapper = NULL; - dTrace("msg:%s is received, handle:%p cont:%p len:%d code:0x%04x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), - pRpc->info.handle, pRpc->pCont, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); + dTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), + pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.noResp = 0; pRpc->info.persistHandle = 0; pRpc->info.wrapper = NULL; @@ -344,66 +344,6 @@ void dmCleanupClient(SDnode *pDnode) { } } -static inline int32_t dmGetHideUserAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - int32_t code = 0; - char pass[TSDB_PASSWORD_LEN + 1] = {0}; - - if (strcmp(user, INTERNAL_USER) == 0) { - taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - } else { - code = -1; - } - - if (code == 0) { - memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 1; - *encrypt = 0; - *ckey = 0; - } - - return code; -} - -static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, - char *ckey) { - if (dmGetHideUserAuth(user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - - SAuthReq authReq = {0}; - tstrncpy(authReq.user, user, TSDB_USER_LEN); - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); - tSerializeSAuthReq(pReq, contLen, &authReq); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .info.ahandle = (void *)9528}; - SRpcMsg rpcRsp = {0}; - SEpSet epSet = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); - dmGetMnodeEpSet(&pDnode->data, &epSet); - dmSendRecv(&epSet, &rpcMsg, &rpcRsp); - - if (rpcRsp.code != 0) { - terrno = rpcRsp.code; - dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); - } else { - SAuthRsp authRsp = {0}; - tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); - memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); - memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); - *spi = authRsp.spi; - *encrypt = authRsp.encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, - authRsp.encrypt); - } - - rpcFreeCont(rpcRsp.pCont); - return rpcRsp.code; -} - int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; @@ -416,7 +356,6 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = (RpcAfp)dmRetrieveUserAuthInfo; rpcInit.parent = pDnode; pTrans->serverRpc = rpcOpen(&rpcInit); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 503c0f404a..274876567e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -627,21 +627,26 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); - SArray *tmp = pOldConsumer->rebNewTopics; - pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; - pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->rebRemovedTopics; - pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; - pNewConsumer->rebRemovedTopics = tmp; + if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { + pOldConsumer->status = MQ_CONSUMER_STATUS__READY; + } else { + SArray *tmp = pOldConsumer->rebNewTopics; + pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; + pNewConsumer->rebNewTopics = tmp; - tmp = pOldConsumer->assignedTopics; - pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; - pNewConsumer->assignedTopics = tmp; + tmp = pOldConsumer->rebRemovedTopics; + pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; + pNewConsumer->rebRemovedTopics = tmp; - pOldConsumer->subscribeTime = pNewConsumer->upTime; + tmp = pOldConsumer->assignedTopics; + pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; + pNewConsumer->assignedTopics = tmp; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pOldConsumer->subscribeTime = pNewConsumer->upTime; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 5cf2a36731..7e99c5583d 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -197,8 +197,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto CONN_OVER; } if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->conn.user, connReq.passwd, - pUser->pass); + mError("user:%s, failed to auth while acquire user, input:%s", pReq->conn.user, connReq.passwd); code = TSDB_CODE_RPC_AUTH_FAILURE; goto CONN_OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7bd1dd80fb..c9b68f27e7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -842,7 +842,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%04x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, code & 0xFFFF, pTrans->stage, pTrans->rpcInfo.ahandle); SRpcMsg rspMsg = { .info = pTrans->rpcInfo, @@ -899,7 +899,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) { } } - mDebug("trans:%d, action:%d response is received, code:0x%04x, accept:0x%04x", transId, action, pRsp->code, + mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%04x", transId, action, pRsp->code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -1031,7 +1031,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA mDebug("trans:%d, all %d actions execute successfully", pTrans->id, numOfActions); return 0; } else { - mError("trans:%d, all %d actions executed, code:0x%04x", pTrans->id, numOfActions, errCode & 0XFFFF); + mError("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errCode & 0XFFFF); mndTransResetActions(pMnode, pTrans, pArray); terrno = errCode; return errCode; @@ -1222,7 +1222,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); } - mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); + mDebug("trans:%d, finished, code:0x%x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); return continueExec; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3dfba4eca7..5326376bda 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -343,19 +343,20 @@ void mndStop(SMnode *pMnode) { return mndCleanupTimer(pMnode); } int32_t mndProcessMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; void *ahandle = pMsg->info.ahandle; - mTrace("msg:%p, will be processed, type:%s app:%p", pMsg, TMSG_INFO(pMsg->msgType), ahandle); - if (IsReq(pMsg) && !mndIsMaster(pMnode)) { - terrno = TSDB_CODE_APP_NOT_READY; - mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - return -1; - } + if (IsReq(pMsg)) { + if (!mndIsMaster(pMnode)) { + terrno = TSDB_CODE_APP_NOT_READY; + mDebug("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + return -1; + } - if (IsReq(pMsg) && (pMsg->contLen == 0 || pMsg->pCont == NULL)) { - terrno = TSDB_CODE_INVALID_MSG_LEN; - mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); - return -1; + if (pMsg->contLen == 0 || pMsg->pCont == NULL) { + terrno = TSDB_CODE_INVALID_MSG_LEN; + mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + return -1; + } } MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 82acd7fddc..c4ed48fe60 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -56,7 +56,7 @@ class MndTestTrans2 : public ::testing::Test { msgCb.sendReqFp = sendReq; msgCb.sendRspFp = sendRsp; msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack - tmsgSetDefaultMsgCb(&msgCb); + tmsgSetDefault(&msgCb); SMnodeOpt opt = {0}; opt.deploy = 1; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f7eac3a9d9..ecb022426b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -210,7 +210,7 @@ struct SMetaReader { }; struct SMTbCursor { - TDBC *pDbc; + TBC *pDbc; void *pKey; void *pVal; int kLen; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index d3abc95da9..693f4a0a2b 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -22,8 +22,8 @@ extern "C" { #endif -typedef struct SMetaIdx SMetaIdx; -typedef struct SMetaDB SMetaDB; +typedef struct SMetaIdx SMetaIdx; +typedef struct SMetaDB SMetaDB; // metaDebug ================== // clang-format off @@ -63,16 +63,16 @@ struct SMeta { char* path; SVnode* pVnode; - TENV* pEnv; + TDB* pEnv; TXN txn; - TDB* pTbDb; - TDB* pSkmDb; - TDB* pUidIdx; - TDB* pNameIdx; - TDB* pCtbIdx; - TDB* pTagIdx; - TDB* pTtlIdx; - TDB* pSmaIdx; + TTB* pTbDb; + TTB* pSkmDb; + TTB* pUidIdx; + TTB* pNameIdx; + TTB* pCtbIdx; + TTB* pTagIdx; + TTB* pTtlIdx; + TTB* pSmaIdx; SMetaIdx* pIdx; }; @@ -118,7 +118,7 @@ typedef struct { int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); -int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); +int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 2efe600b3d..0601df61e7 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -46,7 +46,7 @@ struct SSmaEnv { TXN txn; void *pPool; // SPoolMem SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -93,16 +93,16 @@ typedef struct SDBFile SDBFile; struct SDBFile { int32_t fid; - TDB *pDB; + TTB *pDB; char *path; }; int32_t tdSmaBeginCommit(SSmaEnv *pEnv); int32_t tdSmaEndCommit(SSmaEnv *pEnv); -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path); -int32_t smaCloseDBEnv(TENV *pEnv); -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF); +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path); +int32_t smaCloseDBEnv(TDB *pEnv); +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF); int32_t smaCloseDBF(SDBFile *pDBF); int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn); void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9727d9df9f..6a5ad63822 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -96,8 +96,8 @@ STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy) SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); SArray* metaGetSmaTbUids(SMeta* pMeta); -int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); -int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); +int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); +int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); @@ -117,6 +117,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqCommit(STQ*); +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 127eb43f2b..135f44e939 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -50,63 +50,63 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { taosMkDir(pMeta->path); // open env - ret = tdbEnvOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); + ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb - ret = tdbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); + ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb - ret = tdbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); + ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx - ret = tdbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); + ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx - ret = tdbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); + ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx - ret = tdbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx - ret = tdbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); + ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTtlIdx - ret = tdbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); + ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSmaIdx - ret = tdbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); + ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); if (ret < 0) { metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; @@ -125,15 +125,15 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { _err: if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); return -1; @@ -142,15 +142,15 @@ _err: int metaClose(SMeta *pMeta) { if (pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pSmaIdx) tdbClose(pMeta->pSmaIdx); - if (pMeta->pTtlIdx) tdbClose(pMeta->pTtlIdx); - if (pMeta->pTagIdx) tdbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbClose(pMeta->pCtbIdx); - if (pMeta->pNameIdx) tdbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbEnvClose(pMeta->pEnv); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); metaDestroyLock(pMeta); taosMemoryFree(pMeta); } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f02b6402c4..2bcb68c82a 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -35,7 +35,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u STbDbKey tbDbKey = {.version = version, .uid = uid}; // query table.db - if (tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; goto _err; } @@ -58,7 +58,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { int64_t version; // query uid.idx - if (tdbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -72,7 +72,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { tb_uid_t uid; // query name.idx - if (tdbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } @@ -100,9 +100,9 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { metaReaderInit(&pTbCur->mr, pMeta, 0); - tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); + tdbTbcOpen(pMeta->pUidIdx, &pTbCur->pDbc, NULL); - tdbDbcMoveToFirst(pTbCur->pDbc); + tdbTbcMoveToFirst(pTbCur->pDbc); return pTbCur; } @@ -113,7 +113,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { tdbFree(pTbCur->pVal); metaReaderClear(&pTbCur->mr); if (pTbCur->pDbc) { - tdbDbcClose(pTbCur->pDbc); + tdbTbcClose(pTbCur->pDbc); } taosMemoryFree(pTbCur); } @@ -125,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) { STbCfg tbCfg; for (;;) { - ret = tdbDbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); + ret = tdbTbcNext(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen); if (ret < 0) { return -1; } @@ -159,7 +159,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo pKey = &skmDbKey; kLen = sizeof(skmDbKey); metaRLock(pMeta); - ret = tdbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); + ret = tdbTbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen); metaULock(pMeta); if (ret < 0) { return NULL; @@ -184,7 +184,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo struct SMCtbCursor { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t suid; void *pKey; void *pVal; @@ -207,7 +207,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { pCtbCur->suid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pCtbCur); @@ -217,9 +217,9 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid ctbIdxKey.suid = uid; ctbIdxKey.uid = INT64_MIN; - tdbDbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); + tdbTbcMoveTo(pCtbCur->pCur, &ctbIdxKey, sizeof(ctbIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pCtbCur->pCur); + tdbTbcMoveToNext(pCtbCur->pCur); } return pCtbCur; @@ -229,7 +229,7 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) { if (pCtbCur) { if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta); if (pCtbCur->pCur) { - tdbDbcClose(pCtbCur->pCur); + tdbTbcClose(pCtbCur->pCur); tdbFree(pCtbCur->pKey); tdbFree(pCtbCur->pVal); @@ -243,7 +243,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int ret; SCtbIdxKey *pCtbIdxKey; - ret = tdbDbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + ret = tdbTbcNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); if (ret < 0) { return 0; } @@ -299,7 +299,7 @@ int metaGetTbNum(SMeta *pMeta) { typedef struct { SMeta *pMeta; - TDBC *pCur; + TBC *pCur; tb_uid_t uid; void *pKey; void *pVal; @@ -323,7 +323,7 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { pSmaCur->uid = uid; metaRLock(pMeta); - ret = tdbDbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); + ret = tdbTbcOpen(pMeta->pSmaIdx, &pSmaCur->pCur, NULL); if (ret < 0) { metaULock(pMeta); taosMemoryFree(pSmaCur); @@ -333,9 +333,9 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { // move to the suid smaIdxKey.uid = uid; smaIdxKey.smaUid = INT64_MIN; - tdbDbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); + tdbTbcMoveTo(pSmaCur->pCur, &smaIdxKey, sizeof(smaIdxKey), &c); if (c > 0) { - tdbDbcMoveToNext(pSmaCur->pCur); + tdbTbcMoveToNext(pSmaCur->pCur); } return pSmaCur; @@ -345,7 +345,7 @@ void metaCloseSmaCursor(SMSmaCursor *pSmaCur) { if (pSmaCur) { if (pSmaCur->pMeta) metaULock(pSmaCur->pMeta); if (pSmaCur->pCur) { - tdbDbcClose(pSmaCur->pCur); + tdbTbcClose(pSmaCur->pCur); tdbFree(pSmaCur->pKey); tdbFree(pSmaCur->pVal); @@ -359,7 +359,7 @@ tb_uid_t metaSmaCursorNext(SMSmaCursor *pSmaCur) { int ret; SSmaIdxKey *pSmaIdxKey; - ret = tdbDbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); + ret = tdbTbcNext(pSmaCur->pCur, &pSmaCur->pKey, &pSmaCur->kLen, &pSmaCur->pVal, &pSmaCur->vLen); if (ret < 0) { return 0; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 1fd81bc2cb..75595d83a6 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -130,17 +130,17 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 273f7885e6..619458af24 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TDBC *pNameIdxc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pCtbIdxc = NULL; + TBC *pNameIdxc = NULL; + TBC *pUidIdxc = NULL; + TBC *pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -82,43 +82,43 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int c, ret; // prepare uid idx cursor - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c != 0) { terrno = TSDB_CODE_VND_TB_NOT_EXIST; - tdbDbcClose(pUidIdxc); + tdbTbcClose(pUidIdxc); goto _err; } // prepare name idx cursor - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); if (ret < 0 || c != 0) { ASSERT(0); } - tdbDbcDelete(pUidIdxc); - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pUidIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pNameIdxc); // loop to drop each child table - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) { - tdbDbcClose(pCtbIdxc); + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (ret < 0 || (c < 0 && tdbTbcMoveToNext(pCtbIdxc) < 0)) { + tdbTbcClose(pCtbIdxc); goto _exit; } for (;;) { - tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); + tdbTbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL); pCtbIdxKey = (SCtbIdxKey *)pKey; if (pCtbIdxKey->suid > pReq->suid) break; // drop the child table (TODO) - if (tdbDbcMoveToNext(pCtbIdxc) < 0) break; + if (tdbTbcMoveToNext(pCtbIdxc) < 0) break; } _exit: @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TDBC *pUidIdxc = NULL; - TDBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -143,14 +143,14 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -158,11 +158,11 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = *(int64_t *)pData; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(ret == 0 && c == 0); - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ASSERT(ret == 0); tDecoderInit(&dc, pData, nData); @@ -191,12 +191,12 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaSaveToTbDb(pMeta, &nStbEntry); // update uid index - tdbDbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaULock(pMeta); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; } @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { - TDBC *pTbDbc = NULL; - TDBC *pUidIdxc = NULL; - TDBC *pNameIdxc = NULL; + TBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -271,15 +271,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int c = 0, ret; // search & delete the name idx - tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { - tdbDbcClose(pNameIdxc); + tdbTbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); + if (ret < 0 || !tdbTbcIsValid(pNameIdxc) || c) { + tdbTbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; } - ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pNameIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -287,36 +287,36 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { uid = *(tb_uid_t *)pData; - tdbDbcDelete(pNameIdxc); - tdbDbcClose(pNameIdxc); + tdbTbcDelete(pNameIdxc); + tdbTbcClose(pNameIdxc); // search & delete uid idx - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; } tver = *(int64_t *)pData; - tdbDbcDelete(pUidIdxc); - tdbDbcClose(pUidIdxc); + tdbTbcDelete(pUidIdxc); + tdbTbcClose(pUidIdxc); // search and get meta entry - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + ret = tdbTbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); if (ret < 0) { ASSERT(0); return -1; @@ -345,21 +345,21 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { taosMemoryFree(pDataCopy); tDecoderClear(&coder); - tdbDbcClose(pTbDbc); + tdbTbcClose(pTbDbc); if (type == TSDB_CHILD_TABLE) { // remove the pCtbIdx - TDBC *pCtbIdxc = NULL; - tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); - ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); + ret = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c); if (ret < 0 || c != 0) { ASSERT(0); return -1; } - tdbDbcDelete(pCtbIdxc); - tdbDbcClose(pCtbIdxc); + tdbTbcDelete(pCtbIdxc); + tdbTbcClose(pCtbIdxc); // remove tags from pTagIdx (todo) } else if (type == TSDB_NORMAL_TABLE) { @@ -389,7 +389,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl int c; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -400,22 +400,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry SDecoder dc = {0}; @@ -442,7 +442,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } entry.version = version; - int tlen; + int tlen; + SSchema *pNewSchema = NULL; switch (pAlterTbReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: if (pColumn) { @@ -451,8 +452,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl } pSchema->sver++; pSchema->nCols++; - pSchema->pSchema = - taosMemoryRealloc(entry.ntbEntry.schema.pSchema, sizeof(SSchema) * entry.ntbEntry.schema.nCols); + pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); + memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); + pSchema->pSchema = pNewSchema; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].bytes = pAlterTbReq->bytes; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].type = pAlterTbReq->type; pSchema->pSchema[entry.ntbEntry.schema.nCols - 1].flags = pAlterTbReq->flags; @@ -505,21 +507,22 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // save to table db metaSaveToTbDb(pMeta, &entry); - tdbDbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); + tdbTbcUpsert(pUidIdxc, &entry.uid, sizeof(tb_uid_t), &version, sizeof(version), 0); metaSaveToSkmDb(pMeta, &entry); metaULock(pMeta); + if (pNewSchema) taosMemoryFree(pNewSchema); tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: tDecoderClear(&dc); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -536,7 +539,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA int nData = 0; // search name index - ret = tdbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; @@ -547,24 +550,24 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA pVal = NULL; // search uid index - TDBC *pUidIdxc = NULL; + TBC *pUidIdxc = NULL; - tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); - tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); - tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = *(int64_t *)pData; // search table.db - TDBC *pTbDbc = NULL; + TBC *pTbDbc = NULL; SDecoder dc = {0}; /* get ctbEntry */ - tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); - tdbDbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); - tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData); + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); memcpy(ctbEntry.pBuf, pData, nData); @@ -573,9 +576,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc); /* get stbEntry*/ - tdbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); - tdbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), - (void **)&stbEntry.pBuf, &nVal); + tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); + tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), + (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); tDecoderInit(&dc, stbEntry.pBuf, nVal); metaDecodeEntry(&dc, &stbEntry); @@ -632,19 +635,19 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaSaveToTbDb(pMeta, &ctbEntry); // save to uid.idx - tdbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return 0; _err: if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbDbcClose(pTbDbc); - tdbDbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); return -1; } @@ -708,7 +711,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { goto _err; } @@ -721,11 +724,11 @@ _err: } static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -748,12 +751,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60; ttlKey.uid = pME->uid; - return tdbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, @@ -801,10 +804,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; - tdbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &stbEntry); @@ -817,7 +820,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } - tdbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); @@ -859,7 +862,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaTDBImpl.c b/source/dnode/vnode/src/sma/smaTDBImpl.c index 6e57656802..cac986d053 100644 --- a/source/dnode/vnode/src/sma/smaTDBImpl.c +++ b/source/dnode/vnode/src/sma/smaTDBImpl.c @@ -17,12 +17,12 @@ #include "sma.h" -int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { smaError("failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t smaOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t smaCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,21 +54,21 @@ static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int return 0; } -static int32_t smaOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { tdb_cmpr_fn_t compFunc; // Create a database compFunc = tdSmaKeyCmpr; - if (tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { + if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) { return -1; } return 0; } -static int32_t smaCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t smaOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -99,7 +99,7 @@ int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, in int32_t ret; printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn); - ret = tdbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { smaError("failed to upsert tsma data into db, ret = %d", ret); return -1; @@ -112,7 +112,7 @@ void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_ void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { smaError("failed to get tsma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 1d54d75ad5..f771e73c8a 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -55,7 +55,6 @@ typedef enum { SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma } ESmaStorageLevel; - // static func static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); @@ -69,18 +68,15 @@ static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t f static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, - TXN *txn); + TXN *txn); // expired window - -static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version); +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); // read data - // implementation /** @@ -157,7 +153,6 @@ static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { return false; } - /** * @brief Approximate value for week/month/year. * @@ -239,9 +234,8 @@ static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, return interval; } - static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, - int8_t intervalUnit) { + int8_t intervalUnit) { pSmaH->pSma = pSma; pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); pSmaH->pDataBlocks = pDataBlocks; @@ -493,11 +487,12 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { smaCloseDBF(&tSmaH.dFile); } tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, + smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + " queryKey:%" PRIi64, SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tdDestroyTSmaWriteH(&tSmaH); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; @@ -523,9 +518,8 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { tdResetExpiredWindow(pSma, pStat, indexUid, skey); } else { smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, - SMA_VID(pSma), skey, tlen, indexUid); + SMA_VID(pSma), skey, tlen, indexUid); } - } } tdSmaEndCommit(pEnv); // TODO: not commit for every insert @@ -557,7 +551,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); @@ -600,12 +594,12 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { // error handling tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), - skey, indexUid); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + indexUid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), - skey, indexUid); + skey, indexUid); // TODO: use a standalone interface to received state upate notification from stream computing module. /** * @brief state @@ -666,8 +660,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); taosSsleep(1); if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, - refVal); + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); break; }; } @@ -730,17 +723,17 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdUnRefSmaStat(pSma, pStat); terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, - tstrerror(terrno), smaStat); + tstrerror(terrno), smaStat); return TSDB_CODE_FAILED; } if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), - querySKey, indexUid); + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); } else { smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, - indexUid); + indexUid); } STSma *pTSma = pItem->pTSma; @@ -755,7 +748,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query tdInitTSmaFile(&tReadH, indexUid, querySKey); smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, - SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); + SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; @@ -766,18 +759,18 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query int64_t queryGroupId = 0; tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), - tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); void *result = NULL; int32_t valueSize = 0; if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", - SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); smaCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } - #endif +#endif #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ for (uint32_t v = 0; v < valueSize; v += 8) { @@ -878,7 +871,7 @@ static SSmaStatItem *tdNewSmaStatItem(int8_t state) { } static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, - int64_t version) { + int64_t version) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (!pItem) { // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later @@ -923,17 +916,15 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde taosMemoryFreeClear(pItem->pTSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_FAILED; } smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, - winSKey); + winSKey); return TSDB_CODE_SUCCESS; } - - /** * @brief Update expired window according to msg from stream computing module. * @@ -1035,7 +1026,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) } } else { smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", - SMA_VID(pSma), pTSma->indexUid, winSKey); + SMA_VID(pSma), pTSma->indexUid, winSKey); } } } @@ -1044,4 +1035,3 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) return TSDB_CODE_SUCCESS; } - diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6cc986d54b..80aa350e4d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -101,6 +101,21 @@ static void tdSRowDemo() { taosMemoryFree(pTSChema); } +int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList) { + void* pIter = NULL; + STqExec* pExec = NULL; + while (1) { + pIter = taosHashIterate(pTq->execs, pIter); + if (pIter == NULL) break; + pExec = (STqExec*)pIter; + for (int32_t i = 0; i < 5; i++) { + int32_t code = qUpdateQualifiedTableId(pExec->task[i], tbUidList, true); + ASSERT(code == 0); + } + } + return 0; +} + int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index dc782cc022..18cf18dbad 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -53,7 +53,7 @@ struct SSmaEnv { TXN txn; SPoolMem *pPool; SDiskID did; - TENV *dbEnv; // TODO: If it's better to put it in smaIndex level? + TDB *dbEnv; // TODO: If it's better to put it in smaIndex level? char *path; // relative path SSmaStat *pStat; }; @@ -876,7 +876,7 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t ke TXN *txn) { SDBFile *pDBFile = &pSmaH->dFile; - // TODO: insert tsma data blocks into B+Tree(TDB) + // TODO: insert tsma data blocks into B+Tree(TTB) if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { tsdbWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c index bed61a186f..a553f32bee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbTDBImpl.c @@ -17,12 +17,12 @@ #include "tsdb.h" -int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { +int32_t tsdbOpenDBEnv(TDB **ppEnv, const char *path) { int ret = 0; if (path == NULL) return -1; - ret = tdbEnvOpen(path, 4096, 256, ppEnv); // use as param + ret = tdbOpen(path, 4096, 256, ppEnv); // use as param if (ret != 0) { tsdbError("Failed to create tsdb db env, ret = %d", ret); @@ -32,7 +32,7 @@ int32_t tsdbOpenDBEnv(TENV **ppEnv, const char *path) { return 0; } -int32_t tsdbCloseDBEnv(TENV *pEnv) { return tdbEnvClose(pEnv); } +int32_t tsdbCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); } static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { const SSmaKey *pKey1 = (const SSmaKey *)arg1; @@ -54,20 +54,20 @@ static inline int tsdbSmaKeyCmpr(const void *arg1, int len1, const void *arg2, i return 0; } -static int32_t tsdbOpenDBDb(TDB **ppDB, TENV *pEnv, const char *pFName) { +static int32_t tsdbOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) { int ret; tdb_cmpr_fn_t compFunc; // Create a database compFunc = tsdbSmaKeyCmpr; - ret = tdbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); + ret = tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB); return 0; } -static int32_t tsdbCloseDBDb(TDB *pDB) { return tdbClose(pDB); } +static int32_t tsdbCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); } -int32_t tsdbOpenDBF(TENV *pEnv, SDBFile *pDBF) { +int32_t tsdbOpenDBF(TDB *pEnv, SDBFile *pDBF) { // TEnv is shared by a group of SDBFile if (!pEnv || !pDBF) { terrno = TSDB_CODE_INVALID_PTR; @@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) { int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) { int32_t ret; - ret = tdbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); + ret = tdbTbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn); if (ret < 0) { tsdbError("Failed to create insert sma data into db, ret = %d", ret); return -1; @@ -110,7 +110,7 @@ void *tsdbGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32 void *pVal = NULL; int ret; - ret = tdbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); + ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen); if (ret < 0) { tsdbError("Failed to get sma data from db, ret = %d", ret); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 371ade4f76..6fc9f2998b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -333,6 +333,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, SVCreateTbRsp cRsp = {0}; char tbName[TSDB_TABLE_FNAME_LEN]; STbUidStore *pStore = NULL; + SArray *tbUids = NULL; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -348,7 +349,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); - if (rsp.pArray == NULL) { + tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); + if (rsp.pArray == NULL || tbUids == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -376,6 +378,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } else { cRsp.code = TSDB_CODE_SUCCESS; tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); + taosArrayPush(tbUids, &pCreateReq->uid); } taosArrayPush(rsp.pArray, &cRsp); @@ -383,6 +386,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tDecoderClear(&decoder); + tqUpdateTbUidList(pVnode->pTq, tbUids); tdUpdateTbUidList(pVnode->pSma, pStore); tdUidStoreFree(pStore); @@ -402,6 +406,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, _exit: taosArrayDestroy(rsp.pArray); + taosArrayDestroy(tbUids); tDecoderClear(&decoder); tEncoderClear(&encoder); return rcode; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bf178612ba..e010363a1e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -378,6 +378,13 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_DATAREADER, } EStreamScanMode; +typedef struct SCatchSupporter { + SHashObj* pWindowHashTable; // quick locate the window object for each window + SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file + int32_t keySize; + int64_t* pKeyBuf; +} SCatchSupporter; + typedef struct SStreamBlockScanInfo { SArray* pBlockLists; // multiple SSDatablock. SSDataBlock* pRes; // result SSDataBlock @@ -400,6 +407,8 @@ typedef struct SStreamBlockScanInfo { EStreamScanMode scanMode; SOperatorInfo* pOperatorDumy; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. + SCatchSupporter childAggSup; + SArray* childIds; } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -460,6 +469,16 @@ typedef struct SIntervalAggOperatorInfo { bool invertible; } SIntervalAggOperatorInfo; +typedef struct SStreamFinalIntervalOperatorInfo { + SOptrBasicInfo binfo; // basic info + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + SAggSupporter aggSup; // aggregate supporter + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; +} SStreamFinalIntervalOperatorInfo; + typedef struct SAggOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; @@ -696,6 +715,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, @@ -771,9 +793,8 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); - -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput); +int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, + const char* pKey, const char* pDir); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 320450eb6e..66073b70eb 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -126,7 +126,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; // traverse to the stream scanner node to add this table id @@ -141,12 +141,12 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); - for(int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%"PRIu64" code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; } @@ -160,7 +160,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA metaReaderClear(&mr); - qDebug(" %d qualified child tables added into stream scanner", (int32_t) taosArrayGetSize(qa)); + qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 354f4d8752..d4d8696aba 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -133,6 +133,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; + cleanUpUdfs(); qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); return pTaskInfo->code; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d0a1840d72..e42ced4669 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -342,28 +342,6 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } -void doClearWindow(SIntervalAggOperatorInfo* pInfo, char* pData, int16_t bytes, - uint64_t groupId, int32_t numOfOutput) { - SAggSupporter* pSup = &pInfo->aggSup; - SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); - SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); - SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].resultInfo = getResultCell(pResult, i, pInfo->binfo.rowCellInfoOffset); - struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; - if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) { - continue; - } - pResInfo->initialized = false; - if (pCtx[i].functionId != -1) { - pCtx[i].fpSet.init(&pCtx[i], pResInfo); - } - } -} - /** * the struct of key in hash table * +----------+---------------+ @@ -5321,3 +5299,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } + +int32_t initCatchSupporter(SCatchSupporter* pCatchSup, size_t rowSize, size_t keyBufSize, + const char* pKey, const char* pDir) { + pCatchSup->keySize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); + pCatchSup->pKeyBuf = taosMemoryCalloc(1, pCatchSup->keySize); + int32_t pageSize = rowSize * 32; + int32_t bufSize = pageSize * 4096; + createDiskbasedBuf(&pCatchSup->pDataBuf, pageSize, bufSize, pKey, pDir); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pCatchSup->pWindowHashTable = taosHashInit(10000, hashFn, true, HASH_NO_LOCK);; + return TSDB_CODE_SUCCESS; +} + \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eaa1197a84..2e3f8c8044 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -36,6 +36,11 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) +typedef struct SWindowPosition { + int32_t pageId; + int32_t rowId; +} SWindowPosition; + static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); @@ -668,12 +673,103 @@ static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool inverti } pDataBlock->info.rows = size; pDataBlock->info.type = STREAM_REPROCESS; + blockDataUpdateTsWindow(pDataBlock); taosArrayClear(pInfo->tsArray); return pDataBlock; } return NULL; } +void static setSupKeyBuf(SCatchSupporter* pSup, int64_t groupId, int64_t childId, TSKEY ts) { + int64_t* pKey = (int64_t*)pSup->pKeyBuf; + pKey[0] = groupId; + pKey[1] = childId; + pKey[2] = ts; +} + +static int32_t catchWidonwInfo(SSDataBlock* pDataBlock, SCatchSupporter* pSup, + int32_t pageId, int32_t tsIndex, int64_t childId) { + SColumnInfoData* pColDataInfo = taosArrayGet(pDataBlock->pDataBlock, tsIndex); + TSKEY* tsCols = (int64_t*)pColDataInfo->pData; + for (int32_t i = 0; i < pDataBlock->info.rows; i++) { + setSupKeyBuf(pSup, pDataBlock->info.groupId, childId, tsCols[i]); + SWindowPosition* p1 = (SWindowPosition*)taosHashGet(pSup->pWindowHashTable, + pSup->pKeyBuf, pSup->keySize); + if (p1 == NULL) { + SWindowPosition pos = {.pageId = pageId, .rowId = i}; + int32_t code = taosHashPut(pSup->pWindowHashTable, pSup->pKeyBuf, pSup->keySize, &pos, + sizeof(SWindowPosition)); + if (code != TSDB_CODE_SUCCESS ) { + return code; + } + } else { + p1->pageId = pageId; + p1->rowId = i; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t catchDatablock(SSDataBlock* pDataBlock, SCatchSupporter* pSup, + int32_t tsIndex, int64_t childId) { + int32_t start = 0; + int32_t stop = 0; + int32_t pageSize = getBufPageSize(pSup->pDataBuf); + while(start < pDataBlock->info.rows) { + blockDataSplitRows(pDataBlock, pDataBlock->info.hasVarCol, start, &stop, pageSize); + SSDataBlock* pDB = blockDataExtractBlock(pDataBlock, start, stop - start + 1); + if (pDB == NULL) { + return terrno; + } + int32_t pageId = -1; + void* pPage = getNewBufPage(pSup->pDataBuf, pDataBlock->info.groupId, &pageId); + if (pPage == NULL) { + blockDataDestroy(pDB); + return terrno; + } + int32_t size = blockDataGetSize(pDB) + sizeof(int32_t) + pDB->info.numOfCols * sizeof(int32_t); + assert(size <= pageSize); + blockDataToBuf(pPage, pDB); + setBufPageDirty(pPage, true); + releaseBufPage(pSup->pDataBuf, pPage); + blockDataDestroy(pDB); + start = stop + 1; + int32_t code = catchWidonwInfo(pDB, pSup, pageId, tsIndex, childId); + if (code != TSDB_CODE_SUCCESS ) { + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* getDataFromCatch(SStreamBlockScanInfo* pInfo) { + SSDataBlock* pBlock = pInfo->pUpdateRes; + if (pInfo->updateResIndex < pBlock->info.rows) { + blockDataCleanup(pInfo->pRes); + SCatchSupporter* pCSup = &pInfo->childAggSup; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; + int32_t size = taosArrayGetSize(pInfo->childIds); + for (int32_t i = 0; i < size; i++) { + int64_t id = *(int64_t *)taosArrayGet(pInfo->childIds, i); + setSupKeyBuf(pCSup, pBlock->info.groupId, id, + tsCols[pInfo->updateResIndex]); + SWindowPosition* pos = (SWindowPosition*)taosHashGet(pCSup->pWindowHashTable, + pCSup->pKeyBuf, pCSup->keySize); + void* buf = getBufPage(pCSup->pDataBuf, pos->pageId); + SSDataBlock* pDB = createOneDataBlock(pInfo->pRes, false); + blockDataFromBuf(pDB, buf); + SSDataBlock* pSub = blockDataExtractBlock(pDB, pos->rowId, 1); + blockDataMerge(pInfo->pRes, pSub, NULL); + blockDataDestroy(pDB); + blockDataDestroy(pSub); + } + pInfo->updateResIndex++; + return pInfo->pRes; + } + return NULL; +} + static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -687,6 +783,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { size_t total = taosArrayGetSize(pInfo->pBlockLists); if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { + SSDataBlock* pDB = getDataFromCatch(pInfo); + if (pDB != NULL) { + return pDB; + } else { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } + } + if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); pOperator->status = OP_EXEC_DONE; @@ -694,7 +799,17 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } int32_t current = pInfo->validBlockIndex++; - return taosArrayGetP(pInfo->pBlockLists, current); + SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + if (pBlock->info.type == STREAM_REPROCESS) { + pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; + } else { + int32_t code = catchDatablock(pBlock, &pInfo->childAggSup, pInfo->primaryTsIndex, 0); + if (code != TDB_CODE_SUCCESS) { + pTaskInfo->code = code; + longjmp(pTaskInfo->env, code); + } + } + return pBlock; } else { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); @@ -768,6 +883,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } rows = pBlockInfo->rows; doFilter(pInfo->pCondition, pInfo->pRes, NULL); + blockDataUpdateTsWindow(pInfo->pRes); break; } @@ -855,6 +971,10 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR pInfo->pOperatorDumy = pOperatorDumy; pInfo->interval = pSTInfo->interval; + size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); + initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize, + "StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blocking = false; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 479ce394b1..62732aa1f9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -8,6 +8,8 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; +static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator); + /* * There are two cases to handle: * @@ -473,8 +475,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, int32_t prevPosition, SIntervalAggOperatorInfo* pInfo) { - int32_t order = pInfo->order; + TSKEY* primaryKeys, int32_t prevPosition, int32_t order) { bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; @@ -723,7 +724,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; } @@ -1031,18 +1032,41 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } } -static void doClearWindows(SIntervalAggOperatorInfo* pInfo, int32_t numOfOutput, SSDataBlock* pBlock) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + +void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, + int16_t bytes, uint64_t groupId, int32_t numOfOutput) { + SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, + GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRow* pResult = getResultRowByPos(pSup->pResultBuf, p1); + SqlFunctionCtx* pCtx = pBinfo->pCtx; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset); + struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (fmIsWindowPseudoColumnFunc(pCtx[i].functionId)) { + continue; + } + pResInfo->initialized = false; + if (pCtx[i].functionId != -1) { + pCtx[i].fpSet.init(&pCtx[i], pResInfo); + } + } +} + +static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, + SInterval* pIntrerval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], &pInfo->interval, - pInfo->interval.precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pIntrerval, + pIntrerval->precision, NULL); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - doClearWindow(pInfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); + doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); } } @@ -1084,7 +1108,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); } if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(pInfo, pOperator->numOfExprs, pBlock); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, + pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock); continue; } pInfo->order = TSDB_ORDER_ASC; @@ -1097,8 +1122,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - // TODO: remove for stream - /*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/ pOperator->status = OP_RES_TO_RETURN; return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; @@ -1116,6 +1139,12 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { cleanupAggSup(&pInfo->aggSup); } +void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + cleanupAggSup(&pInfo->aggSup); +} + bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { if (!fmIsInvertible(pFCtx[i].functionId)) { @@ -1185,6 +1214,63 @@ _error: return NULL; } +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { + SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, + keyBufSize, pTaskInfo->id.str); + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamFinalIntervalOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, + destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, + NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyStreamFinalIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, @@ -1548,3 +1634,91 @@ _error: pTaskInfo->code = code; return NULL; } + +static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardStep = 0; + + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, + &pInfo->interval, pInfo->interval.precision, NULL); + while (1) { + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); + forwardStep = + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + pInfo->order, false); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); + if (startPos < 0) { + break; + } + } + return pUpdated; +} + +static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { + SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = NULL; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } else if (pOperator->status == OP_RES_TO_RETURN) { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + while (1) { + publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); + if (pBlock->info.type == STREAM_REPROCESS) { + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, + pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock); + continue; + } + pUpdated = doHashInterval(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + } + + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + pOperator->status = OP_RES_TO_RETURN; + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 748fb60ef9..a20d0e4718 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -74,6 +74,7 @@ int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bcb4c5c585..5358930df0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -47,8 +47,10 @@ static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t le } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } else if (IS_NULL_TYPE(paraType)) { + paraType = TSDB_DATA_TYPE_BIGINT; } pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; @@ -62,7 +64,7 @@ static int32_t translateInNumOutDou(SFunctionNode* pFunc, char* pErrBuf, int32_t } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -115,18 +117,19 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType)) { + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } uint8_t resType = 0; - if (IS_SIGNED_NUMERIC_TYPE(paraType) || paraType == TSDB_DATA_TYPE_BOOL) { + if (IS_SIGNED_NUMERIC_TYPE(paraType) || TSDB_DATA_TYPE_BOOL == paraType || IS_NULL_TYPE(paraType)) { resType = TSDB_DATA_TYPE_BIGINT; } else if (IS_UNSIGNED_NUMERIC_TYPE(paraType)) { resType = TSDB_DATA_TYPE_UBIGINT; } else if (IS_FLOAT_TYPE(paraType)) { resType = TSDB_DATA_TYPE_DOUBLE; } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -872,7 +875,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, - .finalizeFunc = functionFinalize + .finalizeFunc = lastFinalize }, { .name = "diff", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f28c5f1eaa..d54cc96611 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -255,7 +255,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -331,8 +331,18 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { int32_t countFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = getNumofElem(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); - *((int64_t*)buf) += numOfElem; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (IS_NULL_TYPE(type)) { + //select count(NULL) returns 0 + numOfElem = 1; + *((int64_t*)buf) = 0; + } else { + *((int64_t*)buf) += numOfElem; + } SET_VAL(pResInfo, numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -378,11 +388,17 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; - SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; - int32_t type = pInput->pData[0]->info.type; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; + int32_t type = pInput->pData[0]->info.type; SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _sum_over; + } + if (pInput->colDataAggIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; ASSERT(numOfElem >= 0); @@ -427,6 +443,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { } } +_sum_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -526,6 +543,12 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _avg_over; + } + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; @@ -617,6 +640,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { break; } +_avg_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -783,6 +807,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SMinmaxResInfo *pBuf = GET_ROWCELL_INTERBUF(pResInfo); + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElems = 1; + goto _min_max_over; + } + // data in current data block are qualified to the query if (pInput->colDataAggIsSet) { numOfElems = pInput->numOfRows - pAgg->numOfNull; @@ -1183,6 +1213,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } } +_min_max_over: return numOfElems; } @@ -1215,9 +1246,9 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { float v = *(double*) &pRes->v; - colDataAppend(pCol, currentRow, (const char*)&v, false); + colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes); } else { - colDataAppend(pCol, currentRow, (const char*)&pRes->v, false); + colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); } setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); @@ -1287,6 +1318,12 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; + if (IS_NULL_TYPE(type)) { + GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 1; + goto _stddev_over; + } + switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; @@ -1384,6 +1421,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { break; } +_stddev_over: // data in the check operation are all null, not output SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); return TSDB_CODE_SUCCESS; @@ -1943,6 +1981,19 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +int32_t lastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + + char* in = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + + return pResInfo->numOfRes; +} + bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 068998e469..5f20d2e50a 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1364,9 +1364,12 @@ void releaseUdfFuncHandle(char* udfName) { SUdfcFuncStub key = {0}; strcpy(key.udfName, udfName); SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - ASSERT(foundStub); - --foundStub->refCount; - ASSERT(foundStub->refCount>=0); + if (!foundStub) { + return; + } + if (foundStub->refCount > 0) { + --foundStub->refCount; + } uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); } @@ -1377,7 +1380,7 @@ int32_t cleanUpUdfs() { while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); if (stub->refCount == 0) { - fnInfo("tear down udf. udf name: %s, handle: %p", stub->udfName, stub->handle); + fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", @@ -1530,12 +1533,15 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, } SUdfcUvSession *session = handle; code = doCallUdfScalarFunc(handle, input, numOfCols, output); - if (session->outputType != output->columnData->info.type - || session->outputLen != output->columnData->info.bytes) { - fnError("udfc scalar function calculate error, session type: %d(%d), output type: %d(%d)", - session->outputType, session->outputLen, - output->columnData->info.type, output->columnData->info.bytes); + if (output->columnData == NULL) { + fnError("udfc scalar function calculate error. no column data"); code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } else { + if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { + fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, + session->outputLen, output->columnData->info.type, output->columnData->info.bytes); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } } releaseUdfFuncHandle(udfName); return code; @@ -1565,7 +1571,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); - taosMemoryFree(task->session); + taosMemoryFree(session); taosMemoryFree(task); return err; @@ -1573,7 +1579,6 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { //memory layout |---SUdfAggRes----|-----final result-----|---inter result----| typedef struct SUdfAggRes { - SUdfcUvSession *session; int8_t finalResNum; int8_t interResNum; char* finalResBuf; @@ -1606,7 +1611,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - udfRes->session = (SUdfcUvSession *)handle; SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); @@ -1621,22 +1625,26 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult releaseUdfFuncHandle(pCtx->udfName); return false; } + releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&buf); return true; } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; - int32_t numOfCols = pInput->numOfInputCols; - - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SUdfcUvSession *session = udfRes->session; - if (session == NULL) { - return TSDB_CODE_UDF_NO_FUNC_HANDLE; + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + SInputColumnInfoData* pInput = &pCtx->input; + int32_t numOfCols = pInput->numOfInputCols; int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; @@ -1664,7 +1672,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { .numOfResult = udfRes->interResNum}; SUdfInterBuf newState = {0}; - int32_t udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); + udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); if (udfCode != 0) { fnError("udfAggProcess error. code: %d", udfCode); newState.numOfResult = 0; @@ -1684,19 +1692,21 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { blockDataDestroy(inputBlock); taosArrayDestroy(tempBlock.pDataBlock); - if (udfCode != 0) { - releaseUdfFuncHandle(pCtx->udfName); - } + releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); return udfCode; } int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SUdfcUvSession *session = udfRes->session; - if (session == NULL) { - return TSDB_CODE_UDF_NO_FUNC_HANDLE; + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 8763a8ab3f..802cdf7aad 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -316,46 +316,57 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { case TSDB_DATA_TYPE_TIMESTAMP: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int64_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_TINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int8_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_SMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint16_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_INT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(int32_t*)src, *dst, -1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint32_t*)src, *dst, 1); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%" PRIu64, *(uint64_t*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UBIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); indexInt2str(*(uint64_t*)src, *dst, 1); + tlen = strlen(*dst); case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_DOUBLE: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(double*)src); + tlen = strlen(*dst); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index f0c6455442..ffb102ef2e 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -66,6 +66,17 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp indexMultiTermDestroy(terms); } + +static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, + int8_t operType = DEL_VALUE) { + SIndexTerm* term = + indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + tIndexJsonPut(index, terms, (int64_t)tableId); + + indexMultiTermDestroy(terms); +} static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, void* data, int dlen, int8_t filterType, SArray** result) { std::string colName(colNam); @@ -580,38 +591,46 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) { } TEST_F(JsonEnv, testWriteJsonTfileAndCache_DOUBLE) { { - double val = 10.0; - std::string colName("test1"); + double val = 10.0; for (int i = 0; i < 1000; i++) { - WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i); } } { - double val = 2.0; - std::string colName("test1"); + double val = 2.0; for (int i = 0; i < 1000; i++) { - WriteData(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); + WriteData(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), i + 1000); } } { SArray* res = NULL; std::string colName("test1"); double val = 1.9; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); EXPECT_EQ(2000, taosArrayGetSize(res)); } { - SArray* res = NULL; - std::string colName("test1"); - double val = 2.1; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); EXPECT_EQ(1000, taosArrayGetSize(res)); } { - std::string colName("test1"); - SArray* res = NULL; - double val = 2.1; - Search(index, colName, TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + SArray* res = NULL; + double val = 2.1; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_GREATER_EQUAL, &res); + EXPECT_EQ(1000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_EQUAL, &res); + EXPECT_EQ(2000, taosArrayGetSize(res)); + } + { + SArray* res = NULL; + double val = 10.0; + Search(index, "test1", TSDB_DATA_TYPE_DOUBLE, &val, sizeof(val), QUERY_LESS_THAN, &res); EXPECT_EQ(1000, taosArrayGetSize(res)); } } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 01490030f2..405fb1c5a0 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -7,7 +7,7 @@ target_sources(tdb "src/db/tdbUtil.c" "src/db/tdbBtree.c" "src/db/tdbDb.c" - "src/db/tdbEnv.c" + "src/db/tdbTable.c" "src/db/tdbTxn.c" "src/db/tdbPage.c" "src/db/tdbOs.c" diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 8ba66f1d26..521b61e3c8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -25,40 +25,40 @@ extern "C" { typedef int (*tdb_cmpr_fn_t)(const void *pKey1, int kLen1, const void *pKey2, int kLen2); // exposed types -typedef struct STEnv TENV; -typedef struct STDB TDB; -typedef struct STDBC TDBC; -typedef struct STxn TXN; - -// TENV -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv); -int tdbEnvClose(TENV *pEnv); -int tdbBegin(TENV *pEnv, TXN *pTxn); -int tdbCommit(TENV *pEnv, TXN *pTxn); +typedef struct STDB TDB; +typedef struct STTB TTB; +typedef struct STBC TBC; +typedef struct STxn TXN; // TDB -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb); +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb); int tdbClose(TDB *pDb); -int tdbDrop(TDB *pDb); -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn); -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +int tdbBegin(TDB *pDb, TXN *pTxn); +int tdbCommit(TDB *pDb, TXN *pTxn); -// TDBC -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); -int tdbDbcClose(TDBC *pDbc); -int tdbDbcIsValid(TDBC *pDbc); -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); -int tdbDbcMoveToFirst(TDBC *pDbc); -int tdbDbcMoveToLast(TDBC *pDbc); -int tdbDbcMoveToNext(TDBC *pDbc); -int tdbDbcMoveToPrev(TDBC *pDbc); -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); -int tdbDbcDelete(TDBC *pDbc); -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert); +// TTB +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb); +int tdbTbClose(TTB *pTb); +int tdbTbDrop(TTB *pTb); +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn); +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn); +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); + +// TBC +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn); +int tdbTbcClose(TBC *pTbc); +int tdbTbcIsValid(TBC *pTbc); +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c); +int tdbTbcMoveToFirst(TBC *pTbc); +int tdbTbcMoveToLast(TBC *pTbc); +int tdbTbcMoveToNext(TBC *pTbc); +int tdbTbcMoveToPrev(TBC *pTbc); +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen); +int tdbTbcDelete(TBC *pTbc); +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert); // TXN #define TDB_TXN_WRITE 0x1 diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index fdad797333..912c1ecaaf 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,134 +15,164 @@ #include "tdbInt.h" -struct STDB { - TENV *pEnv; - SBTree *pBt; -}; - -struct STDBC { - SBTC btc; -}; - -int tdbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb) { - TDB *pDb; - SPager *pPager; - int ret; - char fFullName[TDB_FILENAME_LEN]; - SPage *pPage; - SPgno pgno; +int tdbOpen(const char *rootDir, int szPage, int pages, TDB **ppDb) { + TDB *pDb; + int dsize; + int zsize; + int tsize; + u8 *pPtr; + int ret; *ppDb = NULL; - pDb = (TDB *)tdbOsCalloc(1, sizeof(*pDb)); - if (pDb == NULL) { + dsize = strlen(rootDir); + zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; + + pPtr = (uint8_t *)tdbOsCalloc(1, zsize); + if (pPtr == NULL) { return -1; } - // pDb->pEnv - pDb->pEnv = pEnv; + pDb = (TDB *)pPtr; + pPtr += sizeof(*pDb); + // pDb->rootDir + pDb->rootDir = pPtr; + memcpy(pDb->rootDir, rootDir, dsize); + pDb->rootDir[dsize] = '\0'; + pPtr = pPtr + dsize + 1; + // pDb->jfname + pDb->jfname = pPtr; + memcpy(pDb->jfname, rootDir, dsize); + pDb->jfname[dsize] = '/'; + memcpy(pDb->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); + pDb->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - pPager = tdbEnvGetPager(pEnv, fname); - if (pPager == NULL) { - snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); - ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); - if (ret < 0) { - return -1; - } + pDb->jfd = -1; - tdbEnvAddPager(pEnv, pPager); - } - - ASSERT(pPager != NULL); - - // pDb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pDb->pBt)); + ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache)); if (ret < 0) { return -1; } + pDb->nPgrHash = 8; + tsize = sizeof(SPager *) * pDb->nPgrHash; + pDb->pgrHash = tdbOsMalloc(tsize); + if (pDb->pgrHash == NULL) { + return -1; + } + memset(pDb->pgrHash, 0, tsize); + + mkdir(rootDir, 0755); + *ppDb = pDb; return 0; } int tdbClose(TDB *pDb) { + SPager *pPager; + if (pDb) { - tdbBtreeClose(pDb->pBt); + for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { + pDb->pgrList = pPager->pNext; + tdbPagerClose(pPager); + } + + tdbPCacheClose(pDb->pCache); + tdbOsFree(pDb->pgrHash); tdbOsFree(pDb); } + return 0; } -int tdbDrop(TDB *pDb) { - // TODO - return 0; -} +int tdbBegin(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; -int tdbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { - return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn); -} - -int tdbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); } - -int tdbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { - return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn); -} - -int tdbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { - return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); -} - -int tdbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { - return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); -} - -int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn) { - int ret; - TDBC *pDbc = NULL; - - *ppDbc = NULL; - pDbc = (TDBC *)tdbOsMalloc(sizeof(*pDbc)); - if (pDbc == NULL) { - return -1; - } - - tdbBtcOpen(&pDbc->btc, pDb->pBt, pTxn); - - *ppDbc = pDbc; - return 0; -} - -int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pDbc->btc, pKey, kLen, c); } - -int tdbDbcMoveToFirst(TDBC *pDbc) { return tdbBtcMoveToFirst(&pDbc->btc); } - -int tdbDbcMoveToLast(TDBC *pDbc) { return tdbBtcMoveToLast(&pDbc->btc); } - -int tdbDbcMoveToNext(TDBC *pDbc) { return tdbBtcMoveToNext(&pDbc->btc); } - -int tdbDbcMoveToPrev(TDBC *pDbc) { return tdbBtcMoveToPrev(&pDbc->btc); } - -int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { - return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen); -} - -int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); } - -int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); -} - -int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { - return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert); -} - -int tdbDbcClose(TDBC *pDbc) { - if (pDbc) { - tdbBtcClose(&pDbc->btc); - tdbOsFree(pDbc); + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerBegin(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } } return 0; } -int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file +int tdbCommit(TDB *pDb, TXN *pTxn) { + SPager *pPager; + int ret; + + for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { + ret = tdbPagerCommit(pPager, pTxn); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + + return 0; +} + +SPager *tdbEnvGetPager(TDB *pDb, const char *fname) { + u32 hash; + SPager **ppPager; + + hash = tdbCstringHash(fname); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { + } + + return *ppPager; +} + +void tdbEnvAddPager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // rehash if neccessary + if (pDb->nPager + 1 > pDb->nPgrHash) { + // TODO + } + + // add to list + pPager->pNext = pDb->pgrList; + pDb->pgrList = pPager; + + // add to hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + pPager->pHashNext = *ppPager; + *ppPager = pPager; + + // increase the counter + pDb->nPager++; +} + +void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { + u32 hash; + SPager **ppPager; + + // remove from the list + for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // remove from hash + hash = tdbCstringHash(pPager->dbFileName); + ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; + for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { + } + ASSERT(*ppPager == pPager); + *ppPager = pPager->pNext; + + // decrease the counter + pDb->nPager--; + + // rehash if necessary + if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) { + // TODO + } +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c deleted file mode 100644 index c0c1343a4f..0000000000 --- a/source/libs/tdb/src/db/tdbEnv.c +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -int tdbEnvOpen(const char *rootDir, int szPage, int pages, TENV **ppEnv) { - TENV *pEnv; - int dsize; - int zsize; - int tsize; - u8 *pPtr; - int ret; - - *ppEnv = NULL; - - dsize = strlen(rootDir); - zsize = sizeof(*pEnv) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3; - - pPtr = (uint8_t *)tdbOsCalloc(1, zsize); - if (pPtr == NULL) { - return -1; - } - - pEnv = (TENV *)pPtr; - pPtr += sizeof(*pEnv); - // pEnv->rootDir - pEnv->rootDir = pPtr; - memcpy(pEnv->rootDir, rootDir, dsize); - pEnv->rootDir[dsize] = '\0'; - pPtr = pPtr + dsize + 1; - // pEnv->jfname - pEnv->jfname = pPtr; - memcpy(pEnv->jfname, rootDir, dsize); - pEnv->jfname[dsize] = '/'; - memcpy(pEnv->jfname + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME)); - pEnv->jfname[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0'; - - pEnv->jfd = -1; - - ret = tdbPCacheOpen(szPage, pages, &(pEnv->pCache)); - if (ret < 0) { - return -1; - } - - pEnv->nPgrHash = 8; - tsize = sizeof(SPager *) * pEnv->nPgrHash; - pEnv->pgrHash = tdbOsMalloc(tsize); - if (pEnv->pgrHash == NULL) { - return -1; - } - memset(pEnv->pgrHash, 0, tsize); - - mkdir(rootDir, 0755); - - *ppEnv = pEnv; - return 0; -} - -int tdbEnvClose(TENV *pEnv) { - SPager *pPager; - - if (pEnv) { - for (pPager = pEnv->pgrList; pPager; pPager = pEnv->pgrList) { - pEnv->pgrList = pPager->pNext; - tdbPagerClose(pPager); - } - - tdbPCacheClose(pEnv->pCache); - tdbOsFree(pEnv->pgrHash); - tdbOsFree(pEnv); - } - - return 0; -} - -int tdbBegin(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerBegin(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -int tdbCommit(TENV *pEnv, TXN *pTxn) { - SPager *pPager; - int ret; - - for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) { - ret = tdbPagerCommit(pPager, pTxn); - if (ret < 0) { - ASSERT(0); - return -1; - } - } - - return 0; -} - -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname) { - u32 hash; - SPager **ppPager; - - hash = tdbCstringHash(fname); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) { - } - - return *ppPager; -} - -void tdbEnvAddPager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // rehash if neccessary - if (pEnv->nPager + 1 > pEnv->nPgrHash) { - // TODO - } - - // add to list - pPager->pNext = pEnv->pgrList; - pEnv->pgrList = pPager; - - // add to hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - pPager->pHashNext = *ppPager; - *ppPager = pPager; - - // increase the counter - pEnv->nPager++; -} - -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager) { - u32 hash; - SPager **ppPager; - - // remove from the list - for (ppPager = &pEnv->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // remove from hash - hash = tdbCstringHash(pPager->dbFileName); - ppPager = &pEnv->pgrHash[hash % pEnv->nPgrHash]; - for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { - } - ASSERT(*ppPager == pPager); - *ppPager = pPager->pNext; - - // decrease the counter - pEnv->nPager--; - - // rehash if necessary - if (pEnv->nPgrHash > 8 && pEnv->nPager < pEnv->nPgrHash / 2) { - // TODO - } -} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c new file mode 100644 index 0000000000..ce7da0b471 --- /dev/null +++ b/source/libs/tdb/src/db/tdbTable.c @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdbInt.h" + +struct STTB { + TDB *pEnv; + SBTree *pBt; +}; + +struct STBC { + SBTC btc; +}; + +int tdbTbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { + TTB *pTb; + SPager *pPager; + int ret; + char fFullName[TDB_FILENAME_LEN]; + SPage *pPage; + SPgno pgno; + + *ppTb = NULL; + + pTb = (TTB *)tdbOsCalloc(1, sizeof(*pTb)); + if (pTb == NULL) { + return -1; + } + + // pTb->pEnv + pTb->pEnv = pEnv; + + pPager = tdbEnvGetPager(pEnv, fname); + if (pPager == NULL) { + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->rootDir, fname); + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + + tdbEnvAddPager(pEnv, pPager); + } + + ASSERT(pPager != NULL); + + // pTb->pBt + ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt)); + if (ret < 0) { + return -1; + } + + *ppTb = pTb; + return 0; +} + +int tdbTbClose(TTB *pTb) { + if (pTb) { + tdbBtreeClose(pTb->pBt); + tdbOsFree(pTb); + } + return 0; +} + +int tdbTbDrop(TTB *pTb) { + // TODO + return 0; +} + +int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) { + return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn); +} + +int tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pTb->pBt, pKey, kLen, pTxn); } + +int tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) { + return tdbBtreeUpsert(pTb->pBt, pKey, kLen, pVal, vLen, pTxn); +} + +int tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pTb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + return tdbBtreePGet(pTb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); +} + +int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) { + int ret; + TBC *pTbc = NULL; + + *ppTbc = NULL; + pTbc = (TBC *)tdbOsMalloc(sizeof(*pTbc)); + if (pTbc == NULL) { + return -1; + } + + tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn); + + *ppTbc = pTbc; + return 0; +} + +int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); } + +int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); } + +int tdbTbcMoveToLast(TBC *pTbc) { return tdbBtcMoveToLast(&pTbc->btc); } + +int tdbTbcMoveToNext(TBC *pTbc) { return tdbBtcMoveToNext(&pTbc->btc); } + +int tdbTbcMoveToPrev(TBC *pTbc) { return tdbBtcMoveToPrev(&pTbc->btc); } + +int tdbTbcGet(TBC *pTbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen) { + return tdbBtcGet(&pTbc->btc, ppKey, pkLen, ppVal, pvLen); +} + +int tdbTbcDelete(TBC *pTbc) { return tdbBtcDelete(&pTbc->btc); } + +int tdbTbcNext(TBC *pTbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + return tdbBtreeNext(&pTbc->btc, ppKey, kLen, ppVal, vLen); +} + +int tdbTbcUpsert(TBC *pTbc, const void *pKey, int nKey, const void *pData, int nData, int insert) { + return tdbBtcUpsert(&pTbc->btc, pKey, nKey, pData, nData, insert); +} + +int tdbTbcClose(TBC *pTbc) { + if (pTbc) { + tdbBtcClose(&pTbc->btc); + tdbOsFree(pTbc); + } + + return 0; +} + +int tdbTbcIsValid(TBC *pTbc) { return tdbBtcIsValid(&pTbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index c00706ce0c..bb59a7fc61 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -103,9 +103,9 @@ typedef struct SPage SPage; #define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED) // tdbEnv.c ==================================== -void tdbEnvAddPager(TENV *pEnv, SPager *pPager); -void tdbEnvRemovePager(TENV *pEnv, SPager *pPager); -SPager *tdbEnvGetPager(TENV *pEnv, const char *fname); +void tdbEnvAddPager(TDB *pEnv, SPager *pPager); +void tdbEnvRemovePager(TDB *pEnv, SPager *pPager); +SPager *tdbEnvGetPager(TDB *pEnv, const char *fname); // tdbBtree.c ==================================== typedef struct SBTree SBTree; @@ -334,7 +334,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { return pCell; } -struct STEnv { +struct STDB { char *rootDir; char *jfname; int jfd; @@ -357,8 +357,8 @@ struct SPager { SPgno dbOrigSize; SPage *pDirty; u8 inTran; - SPager *pNext; // used by TENV - SPager *pHashNext; // used by TENV + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB }; #ifdef __cplusplus diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3c8bcee3f7..a161f1c589 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -117,8 +117,8 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in TEST(tdb_test, simple_insert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -126,12 +126,12 @@ TEST(tdb_test, simple_insert1) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -152,7 +152,7 @@ TEST(tdb_test, simple_insert1) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one @@ -181,7 +181,7 @@ TEST(tdb_test, simple_insert1) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbGet(pDb, key, strlen(key), &pVal, &vLen); + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); ASSERT(ret == 0); GTEST_ASSERT_EQ(ret, 0); @@ -193,19 +193,19 @@ TEST(tdb_test, simple_insert1) { } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -217,28 +217,28 @@ TEST(tdb_test, simple_insert1) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); } } - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_insert2) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; TXN txn; @@ -246,12 +246,12 @@ TEST(tdb_test, simple_insert2) { taosRemoveDir("tdb"); // Open Env - ret = tdbEnvOpen("tdb", 1024, 10, &pEnv); + ret = tdbOpen("tdb", 1024, 10, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database compFunc = tDefaultKeyCmpr; - ret = tdbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -271,24 +271,24 @@ TEST(tdb_test, simple_insert2) { for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); GTEST_ASSERT_EQ(ret, 0); } { // Iterate to query the DB data - TDBC *pDBC; + TBC *pDBC; void *pKey = NULL; void *pVal = NULL; int vLen, kLen; int count = 0; - ret = tdbDbcOpen(pDb, &pDBC, NULL); + ret = tdbTbcOpen(pDb, &pDBC, NULL); GTEST_ASSERT_EQ(ret, 0); - tdbDbcMoveToFirst(pDBC); + tdbTbcMoveToFirst(pDBC); for (;;) { - ret = tdbDbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; @@ -300,7 +300,7 @@ TEST(tdb_test, simple_insert2) { GTEST_ASSERT_EQ(count, nData); - tdbDbcClose(pDBC); + tdbTbcClose(pDBC); tdbFree(pKey); tdbFree(pVal); @@ -311,29 +311,29 @@ TEST(tdb_test, simple_insert2) { tdbCommit(pEnv, &txn); tdbTxnClose(&txn); - ret = tdbDrop(pDb); + ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); // Close a database - tdbClose(pDb); + tdbTbClose(pDb); // Close Env - ret = tdbEnvClose(pEnv); + ret = tdbClose(pEnv); GTEST_ASSERT_EQ(ret, 0); } TEST(tdb_test, simple_delete1) { int ret; - TDB *pDb; + TTB *pDb; char key[128]; char data[128]; TXN txn; - TENV *pEnv; + TDB *pEnv; SPoolMem *pPool; void *pKey = NULL; void *pData = NULL; int nKey; - TDBC *pDbc; + TBC *pDbc; int nData; int nKV = 69; @@ -342,11 +342,11 @@ TEST(tdb_test, simple_delete1) { pPool = openPool(); // open env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); @@ -356,7 +356,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -365,7 +365,7 @@ TEST(tdb_test, simple_delete1) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(data, pData, nData), 0); } @@ -374,7 +374,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbDelete(pDb, key, strlen(key), &txn); + ret = tdbTbDelete(pDb, key, strlen(key), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -382,7 +382,7 @@ TEST(tdb_test, simple_delete1) { for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); if (iData <= 30) { GTEST_ASSERT_EQ(ret, 0); } else { @@ -391,15 +391,15 @@ TEST(tdb_test, simple_delete1) { } // loop to iterate the data - tdbDbcOpen(pDb, &pDbc, NULL); + tdbTbcOpen(pDb, &pDbc, NULL); - ret = tdbDbcMoveToFirst(pDbc); + ret = tdbTbcMoveToFirst(pDbc); GTEST_ASSERT_EQ(ret, 0); pKey = NULL; pData = NULL; for (;;) { - ret = tdbDbcNext(pDbc, &pKey, &nKey, &pData, &nData); + ret = tdbTbcNext(pDbc, &pKey, &nKey, &pData, &nData); if (ret < 0) break; std::cout.write((char *)pKey, nKey) /* << " " << kLen */ << " "; @@ -407,20 +407,20 @@ TEST(tdb_test, simple_delete1) { std::cout << std::endl; } - tdbDbcClose(pDbc); + tdbTbcClose(pDbc); tdbCommit(pEnv, &txn); closePool(pPool); - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } TEST(tdb_test, simple_upsert1) { int ret; - TENV *pEnv; - TDB *pDb; + TDB *pEnv; + TTB *pDb; int nData = 100000; char key[64]; char data[64]; @@ -431,11 +431,11 @@ TEST(tdb_test, simple_upsert1) { taosRemoveDir("tdb"); // open env - ret = tdbEnvOpen("tdb", 4096, 64, &pEnv); + ret = tdbOpen("tdb", 4096, 64, &pEnv); GTEST_ASSERT_EQ(ret, 0); // open database - ret = tdbOpen("db.db", -1, -1, NULL, pEnv, &pDb); + ret = tdbTbOpen("db.db", -1, -1, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); pPool = openPool(); @@ -446,7 +446,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -454,7 +454,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } @@ -463,7 +463,7 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); GTEST_ASSERT_EQ(ret, 0); } @@ -473,11 +473,11 @@ TEST(tdb_test, simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbGet(pDb, key, strlen(key), &pData, &nData); + ret = tdbTbGet(pDb, key, strlen(key), &pData, &nData); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(memcmp(pData, data, nData), 0); } - tdbClose(pDb); - tdbEnvClose(pEnv); + tdbTbClose(pDb); + tdbClose(pEnv); } \ No newline at end of file diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index d1bd78f809..84adeb4bf6 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -264,6 +264,7 @@ static void uvHandleReq(SSrvConn* pConn) { CONN_SHOULD_RELEASE(pConn, pHead); STransMsg transMsg; + memset(&transMsg, 0, sizeof(transMsg)); transMsg.contLen = transContLenFromMsg(pHead->msgLen); transMsg.pCont = pHead->content; transMsg.msgType = pHead->msgType; diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index d2cccd0532..b536a70515 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -219,6 +219,41 @@ class TDTestCase: tdSql.query("drop topic %s"%topicName1) + tdLog.info("creat the same topic name , and start to consume") + self.initConsumerTable() + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 5 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + tdLog.printNoPrefix("======== test case 1 end ...... ") def tmqCase2(self, cfgPath, buildPath):