From 6460514147e3fc8dcea76cff887c9d5732cf912f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Jul 2022 13:22:21 +0800 Subject: [PATCH 1/4] enh: grant refactor and grant check --- include/common/tcommon.h | 20 --------- include/common/tgrant.h | 47 +++++++++++++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 23 +++++++--- source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 9 ++-- source/dnode/mnode/impl/src/mndDnode.c | 9 ++-- source/dnode/mnode/impl/src/mndUser.c | 9 ++-- source/dnode/vnode/inc/vnode.h | 1 + 8 files changed, 78 insertions(+), 41 deletions(-) create mode 100644 include/common/tgrant.h diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5aee218ddc..444a17c7f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -268,26 +268,6 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; -//====================================================================================================================== -// for grant -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantCheck(EGrantType grant); - #ifdef __cplusplus } #endif diff --git a/include/common/tgrant.h b/include/common/tgrant.h new file mode 100644 index 0000000000..ad23c661b1 --- /dev/null +++ b/include/common/tgrant.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_GRANT_H_ +#define _TD_COMMON_GRANT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef enum { + TSDB_GRANT_ALL, + TSDB_GRANT_TIME, + TSDB_GRANT_USER, + TSDB_GRANT_DB, + TSDB_GRANT_TIMESERIES, + TSDB_GRANT_DNODE, + TSDB_GRANT_ACCT, + TSDB_GRANT_STORAGE, + TSDB_GRANT_SPEED, + TSDB_GRANT_QUERY_TIME, + TSDB_GRANT_CONNS, + TSDB_GRANT_STREAMS, + TSDB_GRANT_CPU_CORES, +} EGrantType; + +int32_t grantCheck(EGrantType grant); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 06c782e217..93df7f8ab2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,9 +153,15 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_GRANT_EXPIRED; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); @@ -166,9 +172,14 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); + if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_VND_NO_WRITE_AUTH; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteQ, pMsg); + } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8ad1ac56e4..f72b69a7de 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,7 @@ #include "tcache.h" #include "tdatablock.h" #include "tglobal.h" +#include "tgrant.h" #include "tqueue.h" #include "ttime.h" #include "version.h" diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e3b0f3c157..b4b1b383e7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -509,11 +509,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DB); - // if (code != 0) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d4c043028d..f26c1a7d32 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,11 +621,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DNODE); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5c361ed28d..0452659d47 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -363,11 +363,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - // code = grantCheck(TSDB_GRANT_USER); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { + code = terrno; + goto _OVER; + } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6b5f795eb0..8191501469 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -27,6 +27,7 @@ #include "wal.h" #include "tcommon.h" +#include "tgrant.h" #include "tfs.h" #include "tmsg.h" #include "trow.h" From b52c734f885461d0b6831db2387af8f96435b175 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 18 Jul 2022 13:31:13 +0800 Subject: [PATCH 2/4] fix: free vMetaRsp.pSchemas --- source/dnode/vnode/src/meta/metaQuery.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a7fef795ee..776a3c5b7f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -205,7 +205,7 @@ _query: } tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, &schema); + tDecodeSSchemaWrapperEx(&dc, &schema); pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -470,9 +470,9 @@ int64_t metaGetTbNum(SMeta *pMeta) { } // N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta) { +int64_t metaGetTimeSeriesNum(SMeta *pMeta) { // TODO - return 400; + return 400; } typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b49e12fc08..e5322f1bd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -630,6 +630,9 @@ _exit: tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); tEncoderClear(&ec); + if (vMetaRsp.pSchemas) { + taosMemoryFree(vMetaRsp.pSchemas); + } return 0; } From 158c6ae3e419a0dc8f1de9a615078057e2609805 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 13:51:31 +0800 Subject: [PATCH 3/4] fix(tmq): table not found --- source/dnode/vnode/src/tq/tqExec.c | 13 ++++++++++--- source/libs/executor/src/executorMain.c | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index eb17f06cc9..618b32239a 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -64,9 +64,16 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - pRsp->rspOffset = *pOffset; - return 0; + if (pOffset->type == TMQ_OFFSET__LOG) { + pRsp->rspOffset = *pOffset; + return 0; + } else { + tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + if (qStreamPrepareScan(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + return 0; + } + } } int32_t rowCnt = 0; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 37d2521e5d..9d3d62cabf 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -332,6 +332,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); uid = pTableInfo->uid; ts = INT64_MIN; + } else { + return -1; } } /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ From fc67efb5aa9860d9a102197b9d8f4e67ef2eafb8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 13:52:33 +0800 Subject: [PATCH 4/4] fix(query): fix memory leak. --- source/libs/executor/inc/executil.h | 4 ++-- source/libs/executor/src/executil.c | 8 ++++++-- source/libs/executor/src/executorimpl.c | 6 +++++- source/libs/executor/src/timewindowoperator.c | 15 ++++++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f9aba30a46..330ccd68a9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,9 +96,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); + +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 17484ec84b..f10dbf5349 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -72,8 +72,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); - taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; + for(int32_t i = 0; i < taosArrayGetSize(pGroupResInfo->pRows); ++i) { + SResKeyPos* pRes = taosArrayGetP(pGroupResInfo->pRows, i); + taosMemoryFree(pRes); + } + + pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); pGroupResInfo->index = 0; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fe90e58527..8381fae019 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -377,6 +377,10 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } +void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { + colDataDestroy(pColData); +} + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { @@ -3737,7 +3741,7 @@ void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroyEx(pInfo->groupResInfo.pRows, freeItem); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fa29fca74..0feb4430bc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1513,12 +1513,25 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } +static void freeItem(void* param) { + SGroupKeys *pKey = (SGroupKeys*) param; + taosMemoryFree(pKey->pData); +} + void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + pInfo->pPrevValues = NULL; + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); }